You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@marvin.apache.org by we...@apache.org on 2020/10/09 14:27:40 UTC
[incubator-marvin] 01/05: Minor Bug fixes and Notebook security fix
This is an automated email from the ASF dual-hosted git repository.
weichen pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/incubator-marvin.git
commit 26df30d291da415c007430bb67ab5c166cab5c6f
Author: cardosolucas <ca...@gmail.com>
AuthorDate: Wed Sep 23 14:14:07 2020 -0300
Minor Bug fixes and Notebook security fix
---
marvin-cli/Makefile | 1 +
marvin-cli/README.md | 2 +
marvin-cli/bin/marvin | 6 ++
.../marvin_cli/communication/remote_calls.py | 5 +-
marvin-cli/marvin_cli/management/engine.py | 49 ++++++++++----
marvin-cli/marvin_cli/management/notebook.py | 5 +-
marvin-cli/marvin_cli/utils/docker.py | 12 +++-
marvin-cli/marvin_cli/utils/misc.py | 54 +++++++++++++--
marvin-cli/setup.py | 4 +-
.../tests/communication/test_remote_calls.py | 2 +-
marvin-cli/tests/utils/test_misc.py | 6 +-
.../daemon_base/command_server.py | 3 +-
.../notebook_extensions/jupyter_notebook_config.py | 76 ++++++++++++++--------
.../marvin_python_daemon/management/engine.py | 9 +--
.../marvin_python_daemon/management/notebook.py | 7 +-
python-daemon/tests/management/test_notebook.py | 32 +--------
16 files changed, 172 insertions(+), 101 deletions(-)
diff --git a/marvin-cli/Makefile b/marvin-cli/Makefile
index 4f52365..4178d2b 100644
--- a/marvin-cli/Makefile
+++ b/marvin-cli/Makefile
@@ -41,6 +41,7 @@ clean-build:
rm -rf .cache
rm -rf .eggs
rm -rf dist
+ rm -rf benchmarks
clean-reports:
rm -rf coverage_report/
diff --git a/marvin-cli/README.md b/marvin-cli/README.md
index 592f8b6..0d7a26b 100644
--- a/marvin-cli/README.md
+++ b/marvin-cli/README.md
@@ -61,6 +61,8 @@ Commands:
test Run tests.
test-tdd Watch for changes to run tests automatically.
test-tox Run tests using Tox environment.
+ system-kill Kill all marvin cli background process.
+
```
> Marvin is a project started at B2W Digital offices and released open source on September 2017.
diff --git a/marvin-cli/bin/marvin b/marvin-cli/bin/marvin
index 7ac2853..f639271 100644
--- a/marvin-cli/bin/marvin
+++ b/marvin-cli/bin/marvin
@@ -18,6 +18,7 @@
import os
import click
from marvin_cli.utils.misc import package_to_name
+from marvin_cli.utils.misc import kill_persisted_process
from marvin_cli.utils.config import parse_ini
from marvin_cli.utils.config import read_cli_conf, generate_default_conf
from marvin_cli.utils.docker import search_engine_container
@@ -117,6 +118,11 @@ def cli(ctx):
'executor_url': config['executor_url']
}
+@cli.command("system-kill", help="Kill all marvin cli background process.")
+def kill():
+ kill_persisted_process()
+
+
commands = {}
commands.update(cli_engine.commands)
commands.update(cli_notebook.commands)
diff --git a/marvin-cli/marvin_cli/communication/remote_calls.py b/marvin-cli/marvin_cli/communication/remote_calls.py
index 4ec433a..b754dd5 100644
--- a/marvin-cli/marvin_cli/communication/remote_calls.py
+++ b/marvin-cli/marvin_cli/communication/remote_calls.py
@@ -77,10 +77,9 @@ class RemoteCalls:
def stop_grpc(self):
self.stop_command('GRPC')
- def run_notebook(self, port, enable_security):
+ def run_notebook(self, port):
parameters = {
- 'port': port,
- 'enable_security': str(enable_security)
+ 'port': port
}
self.call_command('NOTEBOOK', parameters)
diff --git a/marvin-cli/marvin_cli/management/engine.py b/marvin-cli/marvin_cli/management/engine.py
index e813aa8..8b51135 100644
--- a/marvin-cli/marvin_cli/management/engine.py
+++ b/marvin-cli/marvin_cli/management/engine.py
@@ -15,17 +15,21 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import click
import os
import sys
+import json
import time
import wget
-import subprocess
+import click
+import pickle
import pathlib
+import subprocess
from cookiecutter.main import cookiecutter
+from shutil import which
from ..communication.remote_calls import RemoteCalls
from ..utils.misc import package_folder, extract_folder, get_version
from ..utils.misc import call_logs, package_to_name, get_executor_path_or_download
+from ..utils.misc import generate_timestamp, persist_process
from ..utils.git import git_init, bump_version
from ..utils.log import get_logger
@@ -84,8 +88,8 @@ def import_project(file, dest):
extract_folder(file, dest)
@cli.command("engine-dryrun", help="Run engines in a standalone way.")
-@click.option('--host', '-h', prompt='gRPC host', help='gRPC Host Address', default='localhost')
-@click.option('--port', '-p', prompt='gRPC port', help='gRPC Port', default='50057')
+@click.option('--host', '-gh', help='gRPC Host Address', default='localhost')
+@click.option('--port', '-gp', help='gRPC Port', default='50057')
@click.option(
'--action',
'-a',
@@ -98,8 +102,8 @@ def dryrun(host, port, action, profiling):
rc.run_dryrun(action, profiling)
@cli.command("engine-grpcserver", help="Run gRPC of given actions.")
-@click.option('--host', '-h', prompt='gRPC host', help='gRPC Host Address', default='localhost')
-@click.option('--port', '-p', prompt='gRPC port', help='gRPC Port', default='50057')
+@click.option('--host', '-gh', help='gRPC Host Address', default='localhost')
+@click.option('--port', '-gp', help='gRPC Port', default='50057')
@click.option(
'--action',
'-a',
@@ -119,13 +123,18 @@ def grpc(host, port, action, max_workers, max_rpc_workers):
logger.info("gRPC server terminated!")
@cli.command("engine-logs", help="Show daemon execution.")
+@click.option('--follow', '-f', is_flag=True)
+@click.option('--tail', '-t', default=True, is_flag=True)
+@click.option('--buffer', '-b', default=20)
@click.pass_context
-def docker_logs(ctx):
- call_logs(ctx.obj['package_name'])
+def docker_logs(ctx, follow, tail, buffer):
+ p_logs = call_logs(ctx.obj['package_name'], follow, buffer)
+ if follow:
+ persist_process(p_logs)
@cli.command("engine-httpserver", help="Run executor HTTP server.")
-@click.option('--grpchost', '-gh', prompt='gRPC host', help='gRPC Host Address', default='localhost')
-@click.option('--grpcport', '-gp', prompt='gRPC port', help='gRPC Port', default='50057')
+@click.option('--grpchost', '-gh', help='gRPC Host Address', default='localhost')
+@click.option('--grpcport', '-gp', help='gRPC Port', default='50057')
@click.option('--host', '-h', prompt='API host', help='REST API Host', default='localhost')
@click.option('--port', '-p', prompt='API port', help='REST API Port', default='8000')
@click.option('--protocol', '-pr', help='Marvin protocol to be loaded during initialization.', default='')
@@ -139,9 +148,10 @@ def docker_logs(ctx):
@click.option('--max-rpc-workers', '-rw', help='Max gRPC Workers', default=None)
@click.option('--executor-path', '-e', help='Marvin engine executor jar path', type=click.Path(exists=True))
@click.option('--extra-executor-parameters', '-jvm', help='Use to send extra JVM parameters to engine executor process')
+@click.option('--benchmark', '-b', default=False, is_flag=True, help='Run benchmark.')
@click.pass_context
def http(ctx, grpchost, grpcport, host, port, protocol, action, max_workers,
- max_rpc_workers, executor_path, extra_executor_parameters):
+ max_rpc_workers, executor_path, extra_executor_parameters, benchmark):
rc = RemoteCalls(grpchost, grpcport)
@@ -152,6 +162,8 @@ def http(ctx, grpchost, grpcport, host, port, protocol, action, max_workers,
print("Could not start grpc server!")
sys.exit(1)
+ bench_thread = None
+
try:
if not executor_path:
executor_path = get_executor_path_or_download(ctx.obj['executor_url'])
@@ -168,13 +180,19 @@ def http(ctx, grpchost, grpcport, host, port, protocol, action, max_workers,
command_list.append('-jar')
command_list.append(executor_path)
- print(command_list)
+ if benchmark:
+ logger.info("Init benchmark...")
+ timestamp = generate_timestamp()
+ bench_thread = benchmark_thread(ctx.obj['package_name'], timestamp)
+ bench_thread.start()
httpserver = subprocess.Popen(command_list)
except:
- print("Could not start http server!")
+ logger.error("Could not start http server!")
rc.stop_grpc()
+ if benchmark:
+ bench_thread.terminate()
sys.exit(1)
try:
@@ -186,6 +204,9 @@ def http(ctx, grpchost, grpcport, host, port, protocol, action, max_workers,
rc.stop_grpc()
httpserver.terminate() if httpserver else None
logger.info("Http and grpc servers terminated!")
+ if benchmark:
+ bench_thread.terminate()
+ logger.info("Benchmark terminated!")
sys.exit(0)
@@ -199,4 +220,4 @@ def http(ctx, grpchost, grpcport, host, port, protocol, action, max_workers,
type=click.Choice(['major', 'minor', 'patch']),
help='The part of the version to increase.')
def bumpversion(verbose, dry_run, part):
- bump_version(part, verbose, dry_run)
+ bump_version(part, verbose, dry_run)
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/management/notebook.py b/marvin-cli/marvin_cli/management/notebook.py
index b3c5c4c..844dbad 100644
--- a/marvin-cli/marvin_cli/management/notebook.py
+++ b/marvin-cli/marvin_cli/management/notebook.py
@@ -26,10 +26,9 @@ def cli():
@click.option('--host', '-h', prompt='gRPC host', help='gRPC Host Address', default='localhost')
@click.option('--port', '-p', prompt='gRPC port', help='gRPC Port', default='50057')
@click.option('--notebook-port', '-np', prompt='Notebook port', help='Notebook port', default='8888')
-@click.option('--enable-security', '-s', default=False, is_flag=True, help='Enable notebook security.')
-def notebook(host, port, notebook_port, enable_security):
+def notebook(host, port, notebook_port):
rc = RemoteCalls()
- rc.run_notebook(notebook_port, enable_security)
+ rc.run_notebook(notebook_port)
@cli.command("lab", help="Run custom engine Jupyter Lab.")
@click.option('--host', '-h', prompt='gRPC host', help='gRPC Host Address', default='localhost')
diff --git a/marvin-cli/marvin_cli/utils/docker.py b/marvin-cli/marvin_cli/utils/docker.py
index 269c3a9..5a605b7 100644
--- a/marvin-cli/marvin_cli/utils/docker.py
+++ b/marvin-cli/marvin_cli/utils/docker.py
@@ -98,4 +98,14 @@ def create_daemon_container(engine_package, engine_name):
detach=True
)
- logger.info("Creating engine docker container ... Done!")
\ No newline at end of file
+ logger.info("Creating engine docker container ... Done!")
+
+def get_stats(engine_package):
+ client = _get_client()
+ if search_engine_container(engine_package):
+ container_name = "marvin-cont-" + package_to_name(engine_package)
+ container = client.containers.get(container_name)
+ return container.stats(stream=False)
+ else:
+ logger.error("Engine container was not found!")
+ return None
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/utils/misc.py b/marvin-cli/marvin_cli/utils/misc.py
index ad40429..84d7974 100644
--- a/marvin-cli/marvin_cli/utils/misc.py
+++ b/marvin-cli/marvin_cli/utils/misc.py
@@ -19,6 +19,12 @@ import os
import subprocess
import tarfile
import wget
+import glob
+import pickle
+import datetime
+from .log import get_logger
+
+logger = get_logger('misc')
def make_tarfile(output_filename, source_dir):
with tarfile.open(output_filename, "w:gz") as tar:
@@ -48,10 +54,44 @@ def extract_folder(input, output):
tf = tarfile.open(input)
tf.extractall(output)
-def call_logs(package):
+def call_logs(package, follow, buffer):
container_name = 'marvin-cont-' + package_to_name(package)
- subprocess.Popen(['xterm', '-e', 'docker', 'logs', '-f', container_name])
-
+ p_return = None
+ if follow:
+ p_return = subprocess.Popen(['docker', 'logs', '--follow', container_name], stdout=subprocess.PIPE)
+ else:
+ p_return = subprocess.Popen(['docker', 'logs', '--tail', str(buffer), container_name], stdout=subprocess.PIPE)
+
+ return p_return
+
+def create_or_return_tmp_dir():
+ tmp_path = '/tmp/marvin'
+ if not os.path.exists(tmp_path):
+ os.makedirs(tmp_path)
+ return tmp_path
+
+def kill_persisted_process():
+ base_path = create_or_return_tmp_dir()
+ for obj_file in glob.glob('{0}/*.mproc'.format(base_path)):
+ pid = int(
+ obj_file[(len(base_path) + 1):(len('.mproc') * -1)]
+ )
+ try:
+ os.kill(pid, 9)
+ logger.info("PID {0} now killed!".format(pid))
+ except ProcessLookupError:
+ logger.info("PID {0} already killed!".format(pid))
+ os.remove(obj_file)
+
+
+def persist_process(obj):
+ filepath = os.path.join(create_or_return_tmp_dir(),
+ '{0}.mproc'.format(obj.pid))
+ logger.info("Creating {0}...".format(filepath))
+ with open(filepath, 'w'):
+ pass
+
+
def get_executor_path_or_download(executor_url):
#get filename from url
_executor_name = executor_url.split('/').pop(-1)
@@ -59,6 +99,10 @@ def get_executor_path_or_download(executor_url):
executor_path = os.path.join(os.environ['MARVIN_DATA_PATH'], _executor_name)
if not os.path.exists(executor_path):
- wget(executor_url, out=executor_path)
+ logger.info("Downloading engine executor in {0}...".format(executor_path))
+ wget.download(executor_url, out=executor_path)
+
+ return executor_path
- return executor_path
\ No newline at end of file
+def generate_timestamp():
+ return datetime.datetime.now().timestamp()
\ No newline at end of file
diff --git a/marvin-cli/setup.py b/marvin-cli/setup.py
index b8dd8fa..7d38de9 100644
--- a/marvin-cli/setup.py
+++ b/marvin-cli/setup.py
@@ -53,7 +53,9 @@ REQUIREMENTS_EXTERNAL = [
'grpcio',
'grpcio-tools',
'click',
- 'cookiecutter'
+ 'cookiecutter',
+ 'docker-compose',
+ 'matplotlib'
]
# This is normally an empty list
diff --git a/marvin-cli/tests/communication/test_remote_calls.py b/marvin-cli/tests/communication/test_remote_calls.py
index d3564dd..47e0eae 100644
--- a/marvin-cli/tests/communication/test_remote_calls.py
+++ b/marvin-cli/tests/communication/test_remote_calls.py
@@ -72,7 +72,7 @@ def test_run_dryrun(call_mocked):
@mock.patch('marvin_cli.communication.remote_calls.RemoteCalls.call_command')
def test_run_notebook(call_mocked):
rc = RemoteCalls()
- rc.run_notebook(True, True)
+ rc.run_notebook(True)
call_mocked.assert_called()
@mock.patch('marvin_cli.communication.remote_calls.RemoteCalls.call_command')
diff --git a/marvin-cli/tests/utils/test_misc.py b/marvin-cli/tests/utils/test_misc.py
index c10ca59..b52f188 100644
--- a/marvin-cli/tests/utils/test_misc.py
+++ b/marvin-cli/tests/utils/test_misc.py
@@ -59,10 +59,10 @@ def test_extract_folder(open_mocked):
@mock.patch('marvin_cli.utils.misc.subprocess.Popen')
def test_call_logs(popen_mocked):
- call_logs(mocked_package)
- popen_mocked.assert_called_with(['xterm', '-e', 'docker', 'logs', '-f', 'marvin-cont-mocked'])
+ call_logs(mocked_package, True, 0)
+ popen_mocked.assert_called_with(['docker', 'logs', '--follow', 'marvin-cont-mocked'], stdout=-1)
-@mock.patch('marvin_cli.utils.misc.wget')
+@mock.patch('marvin_cli.utils.misc.wget.download')
def test_get_executor_path_or_download(wget_mocked):
get_executor_path_or_download(mocked_url)
wget_mocked.assert_called_with(mocked_url, out=os.path.join(os.environ['MARVIN_DATA_PATH'], 'marvin.jar'))
\ No newline at end of file
diff --git a/python-daemon/marvin_python_daemon/daemon_base/command_server.py b/python-daemon/marvin_python_daemon/daemon_base/command_server.py
index 2dfb7bf..ebe4202 100644
--- a/python-daemon/marvin_python_daemon/daemon_base/command_server.py
+++ b/python-daemon/marvin_python_daemon/daemon_base/command_server.py
@@ -48,9 +48,8 @@ def call_grpc(config, parameters):
def call_notebook(config, parameters):
- security = strtobool(parameters['enable_security'])
- notebook(config, bool(security), parameters['port'])
+ notebook(config, parameters['port'])
def call_lab(config, parameters):
diff --git a/python-daemon/marvin_python_daemon/extras/notebook_extensions/jupyter_notebook_config.py b/python-daemon/marvin_python_daemon/extras/notebook_extensions/jupyter_notebook_config.py
index eb9429e..db11ca2 100644
--- a/python-daemon/marvin_python_daemon/extras/notebook_extensions/jupyter_notebook_config.py
+++ b/python-daemon/marvin_python_daemon/extras/notebook_extensions/jupyter_notebook_config.py
@@ -18,12 +18,16 @@
def marvin_code_export(model, **kwargs):
- import autopep8
- import inspect
+ import os
import re
+ import inspect
+ import autopep8
+ from marvin_python_daemon.common.log import get_logger
from marvin_python_daemon.common.config import Config
- print("Executing the marvin export hook script...")
+ logger = get_logger('marvin.savehook')
+
+ logger.info("Executing the marvin export hook script...")
if model['type'] != 'notebook':
return
@@ -39,11 +43,16 @@ def marvin_code_export(model, **kwargs):
'marvin_metrics': re.compile(r"(\bmarvin_metrics\b)")
}
- batch_exec_pattern = re.compile(
- "(def\s+execute\s*\(\s*self\s*,\s*params\s*,\s*\*\*kwargs\s*\)\s*:)")
- online_exec_pattern = re.compile(
- "(def\s+execute\s*\(\s*self\s*,\s*input_message\s*,\s*params\s*,\s*\*\*kwargs\s*\)\s*:)")
-
+ class_path = {
+ "acquisitor": "data_handler{0}acquisitor_and_cleaner.py".format(os.path.sep),
+ "tpreparator": "data_handler{0}training_preparator.py".format(os.path.sep),
+ "trainer": "training{0}trainer.py".format(os.path.sep),
+ "evaluator": "training{0}metrics_evaluator.py".format(os.path.sep),
+ "ppreparator": "prediction{0}prediction_preparator.py".format(os.path.sep),
+ "predictor": "prediction{0}predictor.py".format(os.path.sep),
+ "feedback": "prediction{0}feedback.py".format(os.path.sep),
+ }
+
CLAZZES = {
"acquisitor": "AcquisitorAndCleaner",
"tpreparator": "TrainingPreparator",
@@ -54,6 +63,11 @@ def marvin_code_export(model, **kwargs):
"feedback": "Feedback"
}
+ batch_exec_pattern = re.compile(
+ "(def\s+execute\s*\(\s*self\s*,\s*params\s*,\s*\*\*kwargs\s*\)\s*:)")
+ online_exec_pattern = re.compile(
+ "(def\s+execute\s*\(\s*self\s*,\s*input_message\s*,\s*params\s*,\s*\*\*kwargs\s*\)\s*:)")
+
for cell in cells:
if cell['cell_type'] == 'code' and cell["metadata"].get("marvin_cell", False):
source = cell["source"]
@@ -61,9 +75,14 @@ def marvin_code_export(model, **kwargs):
source, options={'max_line_length': 160})
marvin_action = cell["metadata"]["marvin_cell"]
- marvin_action_clazz = getattr(__import__(
- Config.get("package")), CLAZZES[marvin_action])
- source_path = inspect.getsourcefile(marvin_action_clazz)
+ source_path_list = []
+ site_package_clazz = getattr(__import__(Config.get("package")),
+ CLAZZES[marvin_action])
+
+ #append paths of instaled and source engine package code
+ source_path_list.append(inspect.getsourcefile(site_package_clazz))
+ source_path_list.append(os.path.join(os.getcwd(),
+ Config.get("package"), class_path[marvin_action]))
fnew_source_lines = []
for new_line in new_source.split("\n"):
@@ -93,23 +112,24 @@ def marvin_code_export(model, **kwargs):
exec_pattern = batch_exec_pattern
fnew_source = "".join(fnew_source_lines)
-
- with open(source_path, 'r+') as fp:
- lines = fp.readlines()
- fp.seek(0)
- for line in lines:
- if re.findall(exec_pattern, line):
- fp.write(line)
- fp.write(fnew_source)
- fp.truncate()
-
- break
- else:
- fp.write(line)
-
- print("File {} updated!".format(source_path))
-
- print("Finished the marvin export hook script...")
+
+ for source_path in source_path_list:
+ with open(source_path, 'r+') as fp:
+ lines = fp.readlines()
+ fp.seek(0)
+ for line in lines:
+ if re.findall(exec_pattern, line):
+ fp.write(line)
+ fp.write(fnew_source)
+ fp.truncate()
+
+ break
+ else:
+ fp.write(line)
+
+ logger.info("File {0} updated!".format(source_path))
+
+ logger.info("Finished the marvin export hook script...")
c.FileContentsManager.pre_save_hook = marvin_code_export
diff --git a/python-daemon/marvin_python_daemon/management/engine.py b/python-daemon/marvin_python_daemon/management/engine.py
index 425e802..799c3a1 100644
--- a/python-daemon/marvin_python_daemon/management/engine.py
+++ b/python-daemon/marvin_python_daemon/management/engine.py
@@ -52,7 +52,6 @@ ARTIFACTS = {
}
def dryrun(config, action, profiling):
-
# setting spark configuration directory
os.environ["SPARK_CONF_DIR"] = os.path.join(
os.environ["SPARK_HOME"], "conf")
@@ -87,17 +86,15 @@ class MarvinDryRun(object):
self.feedback_messages = messages[1]
self.pmessages = []
self.package_name = config['marvin_package']
- self.kwargs = None
def execute(self, clazz, params, profiling_enabled=False):
self.print_start_step(clazz)
_Step = dynamic_import("{}.{}".format(self.package_name, clazz))
- if not self.kwargs:
- self.kwargs = generate_kwargs(self.package_name, _Step, params)
+ kwargs = generate_kwargs(self.package_name, _Step, params)
- step = _Step(**self.kwargs)
+ step = _Step(**kwargs)
def call_online_actions(step, msg, msg_idx):
if profiling_enabled:
@@ -186,7 +183,7 @@ def generate_kwargs(package_name, clazz, params=None, initial_dataset='initialda
_artifact_folder = package_name.replace(
'marvin_', '').replace('_engine', '')
_artifacts_to_load = ARTIFACTS[clazz.__name__]
-
+ logger.debug("clazz: {0}, artifacts to load: {1}".format(clazz, str(_artifacts_to_load)))
if params:
kwargs["params"] = params
if dataset in _artifacts_to_load:
diff --git a/python-daemon/marvin_python_daemon/management/notebook.py b/python-daemon/marvin_python_daemon/management/notebook.py
index bf53228..473a6a6 100644
--- a/python-daemon/marvin_python_daemon/management/notebook.py
+++ b/python-daemon/marvin_python_daemon/management/notebook.py
@@ -25,8 +25,9 @@ from ..common.log import get_logger
logger = get_logger('management.notebook')
-
-def notebook(config, enable_security, port):
+def notebook(config, port):
+ logger.info(os.path.join(os.environ["MARVIN_DAEMON_PATH"],
+ 'extras', 'notebook_extensions', 'jupyter_notebook_config.py'))
notebookdir = os.path.join(config['base_path'], 'notebooks')
command = [
"SPARK_CONF_DIR={0} YARN_CONF_DIR={0}".format(
@@ -39,8 +40,6 @@ def notebook(config, enable_security, port):
'--config', os.path.join(os.environ["MARVIN_DAEMON_PATH"],
'extras', 'notebook_extensions', 'jupyter_notebook_config.py')
]
-
- command.append("--NotebookApp.token=") if enable_security else None
command.append("--allow-root")
return_code = os.system(' '.join(command))
diff --git a/python-daemon/tests/management/test_notebook.py b/python-daemon/tests/management/test_notebook.py
index 2cb7e9b..34fd384 100644
--- a/python-daemon/tests/management/test_notebook.py
+++ b/python-daemon/tests/management/test_notebook.py
@@ -42,7 +42,7 @@ def test_notebook(system_mocked, sys_mocked):
spark_conf = '/opt/spark/conf'
system_mocked.return_value = 1
- notebook(config, enable_security, port)
+ notebook(config, port)
system_mocked.assert_called_once_with("SPARK_CONF_DIR={0} YARN_CONF_DIR={0} jupyter notebook --notebook-dir /tmp/notebooks --ip 0.0.0.0 --port 8888 --no-browser --config ".format(os.path.join(os.environ["SPARK_HOME"], "conf")) +
os.environ["MARVIN_ENGINE_PATH"] + '/marvin_python_daemon/extras/notebook_extensions/jupyter_notebook_config.py --allow-root')
@@ -50,20 +50,6 @@ def test_notebook(system_mocked, sys_mocked):
@mock.patch('marvin_python_daemon.management.notebook.sys')
@mock.patch('marvin_python_daemon.management.notebook.os.system')
-def test_notebook_with_security(system_mocked, sys_mocked):
- config = mocked_config
- port = '8888'
- enable_security = True
- system_mocked.return_value = 1
-
- notebook(config, enable_security, port)
-
- system_mocked.assert_called_once_with("SPARK_CONF_DIR={0} YARN_CONF_DIR={0} jupyter notebook --notebook-dir /tmp/notebooks --ip 0.0.0.0 --port 8888 --no-browser --config ".format(os.path.join(os.environ["SPARK_HOME"], "conf")) +
- os.environ["MARVIN_ENGINE_PATH"] + '/marvin_python_daemon/extras/notebook_extensions/jupyter_notebook_config.py --NotebookApp.token= --allow-root')
-
-
-@mock.patch('marvin_python_daemon.management.notebook.sys')
-@mock.patch('marvin_python_daemon.management.notebook.os.system')
def test_jupyter_lab(system_mocked, sys_mocked):
config = mocked_config
port = '8888'
@@ -71,21 +57,7 @@ def test_jupyter_lab(system_mocked, sys_mocked):
spark_conf = '/opt/spark/conf'
system_mocked.return_value = 1
- lab(config, enable_security, port)
+ lab(config, port)
system_mocked.assert_called_once_with(
"SPARK_CONF_DIR={0} YARN_CONF_DIR={0} jupyter-lab --notebook-dir /tmp/notebooks --ip 0.0.0.0 --port 8888 --no-browser --allow-root".format(os.path.join(os.environ["SPARK_HOME"], "conf")))
-
-
-@mock.patch('marvin_python_daemon.management.notebook.sys')
-@mock.patch('marvin_python_daemon.management.notebook.os.system')
-def test_jupyter_lab_with_security(system_mocked, sys_mocked):
- config = mocked_config
- port = '8888'
- enable_security = True
- system_mocked.return_value = 1
-
- lab(config, enable_security, port)
-
- system_mocked.assert_called_once_with(
- "SPARK_CONF_DIR={0} YARN_CONF_DIR={0} jupyter-lab --notebook-dir /tmp/notebooks --ip 0.0.0.0 --port 8888 --no-browser --NotebookApp.token= --allow-root".format(os.path.join(os.environ["SPARK_HOME"], "conf")))