You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by dm...@apache.org on 2019/07/12 13:43:14 UTC
[incubator-dlab] branch DLAB-433 updated: [DLAB-433] Configured
container to use proxy for SSH
This is an automated email from the ASF dual-hosted git repository.
dmysakovets pushed a commit to branch DLAB-433
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/DLAB-433 by this push:
new db4ee8b [DLAB-433] Configured container to use proxy for SSH
db4ee8b is described below
commit db4ee8b4b07bc1404e203205719cea02099cca0a
Author: Dyoma33 <de...@gmail.com>
AuthorDate: Fri Jul 12 16:42:43 2019 +0300
[DLAB-433] Configured container to use proxy for SSH
---
infrastructure-provisioning/src/general/lib/os/fab.py | 9 +++++++--
.../src/general/scripts/aws/edge_prepare.py | 2 +-
.../src/general/scripts/aws/jupyter-docker_configure.py | 3 +++
.../src/general/scripts/os/jupyter_container_start.py | 10 +++++++---
.../src/jupyter-docker/Dockerfile_jupyter | 12 ++++++++----
.../src/jupyter-docker/{jupyter_run.sh => scripts/build.sh} | 10 +---------
.../jupyter-docker/scripts/configure_jupyter-docker_node.py | 3 ++-
.../src/jupyter-docker/{ => scripts}/jupyter_run.sh | 0
.../src/jupyter-docker/{jupyter_run.sh => scripts/start.sh} | 13 ++++---------
9 files changed, 33 insertions(+), 29 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index d80a7dd..5bda129 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -223,20 +223,23 @@ def configure_docker(os_user, http_file, https_file):
print('Failed to configure Docker:', str(err))
sys.exit(1)
-def ensure_jupyter_docker_files(os_user, jupyter_dir, jupyter_conf_file, docker_jupyter_conf, exploratory_name):
+def ensure_jupyter_docker_files(os_user, jupyter_dir, jupyter_conf_file, docker_jupyter_conf, exploratory_name, edge_ip):
if not exists(jupyter_dir):
try:
sudo('mkdir {}'.format(jupyter_dir))
# put(templates_dir + 'pyspark_local_template.json', '/tmp/pyspark_local_template.json')
# put(templates_dir + 'py3spark_local_template.json', '/tmp/py3spark_local_template.json')
put('/root/Dockerfile_jupyter', '/tmp/Dockerfile_jupyter')
- put('/root/jupyter_run.sh', '/tmp/jupyter_run.sh')
+ put('/root/scripts/jupyter_run.sh', '/tmp/jupyter_run.sh')
# sudo('\cp /tmp/pyspark_local_template.json ' + jupyter_dir + 'pyspark_local_template.json')
# sudo('\cp /tmp/py3spark_local_template.json ' + jupyter_dir + 'py3spark_local_template.json')
# sudo('sed -i \'s/3.5/3.6/g\' {}py3spark_local_template.json'.format(jupyter_dir))
sudo('mv /tmp/jupyter_run.sh {}jupyter_run.sh'.format(jupyter_dir))
sudo('mv /tmp/Dockerfile_jupyter {}Dockerfile_jupyter'.format(jupyter_dir))
+ sudo('mv /tmp/build.sh {}build.sh'.format(jupyter_dir))
+ sudo('mv /tmp/start.sh {}start.sh'.format(jupyter_dir))
sudo('sed -i \'s/nb_user/{}/g\' {}Dockerfile_jupyter'.format(os_user, jupyter_dir))
+ sudo('sed -i \'s/nb_user/{}/g\' {}start.sh'.format(os_user, jupyter_dir))
# sudo('sed -i \'s/jup_version/{}/g\' {}Dockerfile_jupyter'.format(jupyter_version, jupyter_dir))
# sudo('sed -i \'s/hadoop_version/{}/g\' {}Dockerfile_jupyter'.format(os.environ['notebook_hadoop_version'], jupyter_dir))
# sudo('sed -i \'s/tornado_version/{}/g\' {}Dockerfile_jupyter'.format(os.environ['notebook_tornado_version'], jupyter_dir))
@@ -253,6 +256,8 @@ def ensure_jupyter_docker_files(os_user, jupyter_dir, jupyter_conf_file, docker_
sudo('''echo "c.NotebookApp.token = u''" >> {}'''.format(jupyter_conf_file))
sudo('echo \'c.KernelSpecManager.ensure_native_kernel = False\' >> {}'.format(jupyter_conf_file))
sudo('chown dlab-user:dlab-user /opt')
+ sudo('echo -e "Host git.epam.com\n HostName git.epam.com\n ProxyCommand nc -X connect -x {}:3128 %h %p\n" > /home/{}/.ssh/config'.format(edge_ip, os_user))
+ sudo('echo -e "Host github.com\n HostName github.com\n ProxyCommand nc -X connect -x {}:3128 %h %p" >> /home/{}/.ssh/config'.format(edge_ip, os_user))
# sudo('touch {}'.format(spark_script))
# sudo('echo "#!/bin/bash" >> {}'.format(spark_script))
# sudo(
diff --git a/infrastructure-provisioning/src/general/scripts/aws/edge_prepare.py b/infrastructure-provisioning/src/general/scripts/aws/edge_prepare.py
index ef7660a..1cf7048 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/edge_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/edge_prepare.py
@@ -211,7 +211,7 @@ if __name__ == "__main__":
{
"PrefixListIds": [],
"FromPort": 22,
- "IpRanges": [{"CidrIp": edge_conf['private_subnet_cidr']}],
+ "IpRanges": [{"CidrIp": edge_conf['all_ip_cidr']}],
"ToPort": 22, "IpProtocol": "tcp", "UserIdGroupPairs": []
},
{
diff --git a/infrastructure-provisioning/src/general/scripts/aws/jupyter-docker_configure.py b/infrastructure-provisioning/src/general/scripts/aws/jupyter-docker_configure.py
index abf899d..85ee26e 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/jupyter-docker_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/jupyter-docker_configure.py
@@ -76,6 +76,7 @@ if __name__ == "__main__":
else:
edge_instance_ip = get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Public')
keyfile_name = "{}{}.pem".format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
+ edge_ip = get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Private')
try:
if os.environ['conf_os_family'] == 'debian':
@@ -143,6 +144,7 @@ if __name__ == "__main__":
print('[CONFIGURE JUPYTER-DOCKER NOTEBOOK INSTANCE]')
params = "--hostname {} " \
"--keyfile {} " \
+ "--edge_ip {}" \
"--region {} " \
"--spark_version {} " \
"--hadoop_version {} " \
@@ -152,6 +154,7 @@ if __name__ == "__main__":
"--exploratory_name {}".\
format(instance_hostname,
keyfile_name,
+ edge_ip,
os.environ['aws_region'],
os.environ['notebook_spark_version'],
os.environ['notebook_hadoop_version'],
diff --git a/infrastructure-provisioning/src/general/scripts/os/jupyter_container_start.py b/infrastructure-provisioning/src/general/scripts/os/jupyter_container_start.py
index 0bb28f6..7f1af69 100644
--- a/infrastructure-provisioning/src/general/scripts/os/jupyter_container_start.py
+++ b/infrastructure-provisioning/src/general/scripts/os/jupyter_container_start.py
@@ -38,9 +38,13 @@ jupyter_dir = '/home/' + args.os_user + '/.jupyter/'
def start_jupyter_container(jupyter_dir):
try:
with cd('{}'.format(jupyter_dir)):
- run('docker volume create -d local-persist -o mountpoint=/home/{} --name=jup_volume'.format(args.os_user))
- run('docker build --file /home/{}/.jupyter/Dockerfile_jupyter -t jupyter-lab .'.format(args.os_user))
- run('docker run -d --restart unless-stopped -p 8888:8888 -v jup_volume:/opt/legion/repository jupyter-lab:latest')
+ run('docker volume create -d local-persist \
+ -o mountpoint=/home/{} --name=jup_volume'.format(args.os_user))
+ run('docker build -file Dockerfile_jupyter -t jupyter-lab .'.format(args.os_user))
+ run('docker run -d --restart unless-stopped -p 8888:8888 \
+ -v jup_volume:/opt/legion/repository \
+ -v /home/{}/.ssh/:/home/nb_user/.ssh/ \
+ jupyter-lab:latest'.format(args.os_user))
except: sys.exit(1)
if __name__ == "__main__":
diff --git a/infrastructure-provisioning/src/jupyter-docker/Dockerfile_jupyter b/infrastructure-provisioning/src/jupyter-docker/Dockerfile_jupyter
index f13add5..68ac706 100644
--- a/infrastructure-provisioning/src/jupyter-docker/Dockerfile_jupyter
+++ b/infrastructure-provisioning/src/jupyter-docker/Dockerfile_jupyter
@@ -19,9 +19,9 @@
#
# ******************************************************************************
-FROM nexus.cc.epm.kharlamov.biz:443/legion/python-toolchain:1.0.0-20190610131523.402.da560a8
+FROM nexus.cc.epm.kharlamov.biz:443/legion/python-toolchain:1.0.0-20190709105425.523.e22cd7d
-ARG NB_USER="nb_user"
+ARG NB_USER="dlab-user"
RUN useradd -u 1001 -ms /bin/bash $NB_USER
@@ -36,10 +36,14 @@ RUN sed -i 's|CONF_PATH|/etc/jupyter/jupyter_notebook_config.py|' /jupyter_run.
&& chown -R 1001:1001 /etc/jupyter \
&& chown -R 1001:1001 /opt/legion
-RUN jupyter labextension disable jupyter.extensions.legion:local
+RUN apt update && apt install -y vim netcat-openbsd
USER $NB_USER
-RUN jupyter serverextension enable --py jupyter_legion
+RUN jupyter serverextension enable --py jupyterlab_git && \
+ jupyter serverextension enable --py legion.jupyterlab && \
+ echo "ENABLED PLUGINS:" && \
+ jupyter serverextension list
+
ENTRYPOINT ["/jupyter_run.sh", "-d"]
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/jupyter-docker/jupyter_run.sh b/infrastructure-provisioning/src/jupyter-docker/scripts/build.sh
similarity index 86%
copy from infrastructure-provisioning/src/jupyter-docker/jupyter_run.sh
copy to infrastructure-provisioning/src/jupyter-docker/scripts/build.sh
index 55c2245..8665337 100644
--- a/infrastructure-provisioning/src/jupyter-docker/jupyter_run.sh
+++ b/infrastructure-provisioning/src/jupyter-docker/scripts/build.sh
@@ -21,12 +21,4 @@
#
# ******************************************************************************
-jupyter lab --config CONF_PATH
-
-if [[$1 == '-d']];then
- while true; do sleep 1000; done
-fi
-
-if [[$1 == "-bash"]];then
- /bin/bash
-fi
\ No newline at end of file
+docker build -t jupyter-lab -f Dockerfile_jupyter .
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/jupyter-docker/scripts/configure_jupyter-docker_node.py b/infrastructure-provisioning/src/jupyter-docker/scripts/configure_jupyter-docker_node.py
index fcb7335..b22e81d 100644
--- a/infrastructure-provisioning/src/jupyter-docker/scripts/configure_jupyter-docker_node.py
+++ b/infrastructure-provisioning/src/jupyter-docker/scripts/configure_jupyter-docker_node.py
@@ -32,6 +32,7 @@ import os
parser = argparse.ArgumentParser()
parser.add_argument('--hostname', type=str, default='')
parser.add_argument('--keyfile', type=str, default='')
+parser.add_argument('--edge_ip', type=str, default='')
parser.add_argument('--region', type=str, default='')
parser.add_argument('--spark_version', type=str, default='')
parser.add_argument('--hadoop_version', type=str, default='')
@@ -105,7 +106,7 @@ if __name__ == "__main__":
# CONFIGURE JUPYTER FILES
print("Configure jupyter files")
- ensure_jupyter_docker_files(args.os_user, jupyter_dir, jupyter_conf_file, docker_jupyter_conf, args.exploratory_name)
+ ensure_jupyter_docker_files(args.os_user, jupyter_dir, jupyter_conf_file, docker_jupyter_conf, args.exploratory_name, args.edge_ip)
# INSTALL UNGIT
print("Install nodejs")
diff --git a/infrastructure-provisioning/src/jupyter-docker/jupyter_run.sh b/infrastructure-provisioning/src/jupyter-docker/scripts/jupyter_run.sh
similarity index 100%
copy from infrastructure-provisioning/src/jupyter-docker/jupyter_run.sh
copy to infrastructure-provisioning/src/jupyter-docker/scripts/jupyter_run.sh
diff --git a/infrastructure-provisioning/src/jupyter-docker/jupyter_run.sh b/infrastructure-provisioning/src/jupyter-docker/scripts/start.sh
similarity index 85%
rename from infrastructure-provisioning/src/jupyter-docker/jupyter_run.sh
rename to infrastructure-provisioning/src/jupyter-docker/scripts/start.sh
index 55c2245..2a6e089 100644
--- a/infrastructure-provisioning/src/jupyter-docker/jupyter_run.sh
+++ b/infrastructure-provisioning/src/jupyter-docker/scripts/start.sh
@@ -21,12 +21,7 @@
#
# ******************************************************************************
-jupyter lab --config CONF_PATH
-
-if [[$1 == '-d']];then
- while true; do sleep 1000; done
-fi
-
-if [[$1 == "-bash"]];then
- /bin/bash
-fi
\ No newline at end of file
+docker run -d --restart unless-stopped -p 8888:8888 \
+ -v jup_volume:/opt/legion/repository \
+ -v /home/nb_user/.ssh/:/home/nb_user/.ssh/ \
+ jupyter-lab:latest
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org