You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by dm...@apache.org on 2019/11/29 12:43:57 UTC
[incubator-dlab] branch DLAB-1326 created (now 8b5ff32)
This is an automated email from the ASF dual-hosted git repository.
dmysakovets pushed a change to branch DLAB-1326
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git.
at 8b5ff32 [DLAB-1326] Implemented proxy configuration for docker
This branch includes the following new commits:
new 8b5ff32 [DLAB-1326] Implemented proxy configuration for docker
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org
[incubator-dlab] 01/01: [DLAB-1326] Implemented proxy configuration
for docker
Posted by dm...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dmysakovets pushed a commit to branch DLAB-1326
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 8b5ff32d3f5891e45966e6a651d71e7503891dfa
Author: Demyan Mysakovets <de...@gmail.com>
AuthorDate: Fri Nov 29 14:43:38 2019 +0200
[DLAB-1326] Implemented proxy configuration for docker
---
.../src/general/lib/os/fab.py | 48 ++++------------------
.../general/scripts/aws/jupyterlab_configure.py | 15 +++++++
.../general/scripts/gcp/jupyterlab_configure.py | 16 ++++++++
.../src/general/scripts/gcp/superset_configure.py | 15 +++++++
...iner_start.py => configure_proxy_for_docker.py} | 42 ++++++++-----------
.../scripts/os/jupyterlab_container_start.py | 4 +-
6 files changed, 71 insertions(+), 69 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index bbb5e39..8447a41 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -193,31 +193,16 @@ def configure_jupyter(os_user, jupyter_conf_file, templates_dir, jupyter_version
print('Error:', str(err))
sys.exit(1)
-def configure_docker(os_user, http_file, https_file):
+def configure_docker(os_user, ):
try:
if not exists('/home/' + os_user + '/.ensure_dir/docker_ensured'):
+ docker_version = os.environ['ssn_docker_version']
sudo('curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -')
sudo('add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) \
stable"')
sudo('apt-get update')
sudo('apt-cache policy docker-ce')
- sudo('apt-get install -y docker-ce')
- sudo('mkdir -p /etc/systemd/system/docker.service.d')
- sudo('touch {}'.format(http_file))
- sudo('echo -e \'[Service] \nEnvironment=\"HTTP_PROXY=\'$http_proxy\'\"\' > {}'.format(http_file))
- sudo('touch {}'.format(https_file))
- sudo('echo -e \'[Service] \nEnvironment=\"HTTPS_PROXY=\'$http_proxy\'\"\' > {}'.format(https_file))
- sudo('mkdir /home/{}/.docker'.format(os_user))
- sudo('touch /home/{}/.docker/config.json'.format(os_user))
- sudo('echo -e \'{\n "proxies":\n {\n "default":\n {\n "httpProxy":"\'$http_proxy\'",\n "httpsProxy":"\'$http_proxy\'"\n }\n }\n}\' > /home/dlab-user/.docker/config.json')
- sudo('usermod -a -G docker ' + os_user)
- sudo('update-rc.d docker defaults')
- sudo('update-rc.d docker enable')
- sudo('wget https://raw.githubusercontent.com/CWSpear/local-persist/master/scripts/install.sh && chmod +x install.sh')
- sudo('sed -i "66s/curl/sudo curl/g" install.sh')
- sudo('sed -i "s/sudo curl/sudo -E curl/g" install.sh')
- run('sudo -E ./install.sh')
- sudo('rm install.sh')
+ sudo('apt-get install -y docker-ce={}~ce-0~ubuntu'.format(docker_version))
sudo('touch /home/{}/.ensure_dir/docker_ensured'.format(os_user))
except Exception as err:
print('Failed to configure Docker:', str(err))
@@ -834,33 +819,14 @@ def update_hosts_file(os_user):
print('Failed to update hosts file', str(err))
sys.exit(1)
-def ensure_docker_compose(dlab_path, os_user):
+def ensure_docker_compose(os_user):
try:
- if not exists(dlab_path + 'tmp/docker_daemon_ensured'):
- docker_version = os.environ['ssn_docker_version']
- sudo('curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -')
- sudo('add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) \
- stable"')
- sudo('apt-get update')
- sudo('apt-cache policy docker-ce')
- sudo('apt-get install -y docker-ce={}~ce-0~ubuntu'.format(docker_version))
- print('Proxy for docker configuring')
- sudo('mkdir -p /etc/systemd/system/docker.service.d')
- with cd('/etc/systemd/system/docker.service.d'):
- sudo('echo -e \'[Service] \nEnvironment=\"HTTP_PROXY=\'$http_proxy\'\"\' > http-proxy.conf')
- sudo('echo -e \'[Service] \nEnvironment=\"HTTPS_PROXY=\'$http_proxy\'\"\' > https-proxy.conf')
- sudo('mkdir -p /home/' + os_user + '/.docker')
- with cd('/home/' + os_user + '/.docker'):
- sudo('echo -e \'{\n "proxies":\n {\n "default":\n {\n "httpProxy":"\'$http_proxy\'",\n "httpsProxy":"\'$http_proxy\'"\n }\n }\n}\' > /home/' + os_user + '/.docker/config.json')
- sudo('usermod -a -G docker ' + os_user)
- sudo('update-rc.d docker defaults')
- sudo('update-rc.d docker enable')
- sudo('mkdir -p ' + dlab_path + 'tmp')
- sudo('touch ' + dlab_path + 'tmp/docker_daemon_ensured')
- if not exists(dlab_path + 'tmp/docker_compose_ensured'):
+ configure_docker(os_user)
+ if not exists('/home/{}/.ensure_dir/docker_compose_ensured'.format(os_user)):
docker_compose_version = "1.24.1"
sudo('curl -L https://github.com/docker/compose/releases/download/{}/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose'.format(docker_compose_version))
sudo('chmod +x /usr/local/bin/docker-compose')
+ sudo('touch /home/{}/.ensure_dir/docker_compose_ensured'.format(os_user))
sudo('systemctl daemon-reload')
sudo('systemctl restart docker')
return True
diff --git a/infrastructure-provisioning/src/general/scripts/aws/jupyterlab_configure.py b/infrastructure-provisioning/src/general/scripts/aws/jupyterlab_configure.py
index 1b36187..157e140 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/jupyterlab_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/jupyterlab_configure.py
@@ -290,6 +290,21 @@ if __name__ == "__main__":
sys.exit(1)
try:
+ print('[CONFIGURING PROXY FOR DOCKER]')
+ logging.info('[CONFIGURING PROXY FOR DOCKER]')
+ params = "--os_user {} ".format(notebook_config['dlab_ssh_user'])
+ try:
+ local("~/scripts/configure_proxy_for_docker.py {}".format(params))
+ except:
+ traceback.print_exc()
+ raise Exception
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ append_result("Failed to configure proxy for docker.", str(err))
+ remove_ec2(notebook_config['tag_name'], notebook_config['instance_name'])
+ sys.exit(1)
+
+ try:
print('[STARTING JUPYTER CONTAINER]')
logging.info('[STARTING JUPYTER CONTAINER]')
params = "--hostname {} " \
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/jupyterlab_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/jupyterlab_configure.py
index 9246352..0bf279c 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/jupyterlab_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/jupyterlab_configure.py
@@ -280,6 +280,22 @@ if __name__ == "__main__":
sys.exit(1)
try:
+ print('[CONFIGURING PROXY FOR DOCKER]')
+ logging.info('[CONFIGURING PROXY FOR DOCKER]')
+ params = "--os_user {} ".format(notebook_config['dlab_ssh_user'])
+ try:
+ local("~/scripts/configure_proxy_for_docker.py {}".format(params))
+ except:
+ traceback.print_exc()
+ raise Exception
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ append_result("Failed to configure proxy for docker.", str(err))
+ remove_ec2(notebook_config['tag_name'], notebook_config['instance_name'])
+ sys.exit(1)
+
+
+ try:
print('[STARTING JUPYTER CONTAINER]')
logging.info('[STARTING JUPYTER CONTAINER]')
params = "--hostname {} " \
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/superset_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/superset_configure.py
index b3ae8da..6c19177 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/superset_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/superset_configure.py
@@ -277,6 +277,21 @@ if __name__ == "__main__":
GCPActions().remove_instance(notebook_config['instance_name'], notebook_config['zone'])
sys.exit(1)
+ try:
+ print('[CONFIGURING PROXY FOR DOCKER]')
+ logging.info('[CONFIGURING PROXY FOR DOCKER]')
+ params = "--os_user {} ".format(notebook_config['dlab_ssh_user'])
+ try:
+ local("~/scripts/configure_proxy_for_docker.py {}".format(params))
+ except:
+ traceback.print_exc()
+ raise Exception
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ append_result("Failed to configure proxy for docker.", str(err))
+ remove_ec2(notebook_config['tag_name'], notebook_config['instance_name'])
+ sys.exit(1)
+
# generating output information
ip_address = GCPMeta().get_private_ip_address(notebook_config['instance_name'])
superset_ip_url = "http://" + ip_address + ":8088/{}/".format(notebook_config['exploratory_name'])
diff --git a/infrastructure-provisioning/src/general/scripts/os/jupyterlab_container_start.py b/infrastructure-provisioning/src/general/scripts/os/configure_proxy_for_docker.py
similarity index 56%
copy from infrastructure-provisioning/src/general/scripts/os/jupyterlab_container_start.py
copy to infrastructure-provisioning/src/general/scripts/os/configure_proxy_for_docker.py
index e7853ee..18791a2 100644
--- a/infrastructure-provisioning/src/general/scripts/os/jupyterlab_container_start.py
+++ b/infrastructure-provisioning/src/general/scripts/os/configure_proxy_for_docker.py
@@ -21,44 +21,36 @@
#
# ******************************************************************************
-import sys
-import os
-from dlab.notebook_lib import *
-from dlab.fab import *
from fabric.api import *
+import sys
+
parser = argparse.ArgumentParser()
-parser.add_argument('--hostname', type=str, default='')
-parser.add_argument('--keyfile', type=str, default='')
parser.add_argument('--os_user', type=str, default='')
args = parser.parse_args()
-jupyterlab_dir = '/home/' + args.os_user + '/.jupyterlab/'
-
-def start_jupyterlab_container(jupyterlab_dir):
- try:
- with cd('{}'.format(jupyterlab_dir)):
- run('docker volume create -d local-persist \
- -o mountpoint=/home/{} --name=jup_volume'.format(args.os_user))
- run('docker build --file Dockerfile_jupyterlab -t jupyter-lab .'.format(args.os_user))
- container_id = run('docker ps | awk \'NR==2{print $1}\'')
- if container_id != '':
- run('docker stop ' + container_id)
- run('docker run -d --restart unless-stopped -p 8888:8888 \
- -v jup_volume:/opt/legion/repository \
- -v /home/{0}/.ssh/:/home/{0}/.ssh/ \
- jupyter-lab:latest'.format(args.os_user))
- except: sys.exit(1)
+http_file = '/etc/systemd/system/docker.service.d/http-proxy.conf'
+https_file = '/etc/systemd/system/docker.service.d/https-proxy.conf'
if __name__ == "__main__":
print("Configure connections")
env['connection_attempts'] = 100
env.key_filename = [args.keyfile]
env.host_string = args.os_user + '@' + args.hostname
- print("Starting Jupyter container")
+ print("Configuring proxy for docker")
try:
- start_jupyterlab_container(jupyterlab_dir)
+ sudo('mkdir -p /etc/systemd/system/docker.service.d')
+ sudo('touch {}'.format(http_file))
+ sudo('echo -e \'[Service] \nEnvironment=\"HTTP_PROXY=\'$http_proxy\'\"\' > {}'.format(http_file))
+ sudo('touch {}'.format(https_file))
+ sudo('echo -e \'[Service] \nEnvironment=\"HTTPS_PROXY=\'$http_proxy\'\"\' > {}'.format(https_file))
+ sudo('mkdir /home/{}/.docker'.format(os_user))
+ sudo('touch /home/{}/.docker/config.json'.format(os_user))
+ sudo(
+ 'echo -e \'{\n "proxies":\n {\n "default":\n {\n "httpProxy":"\'$http_proxy\'",\n "httpsProxy":"\'$http_proxy\'"\n }\n }\n}\' > /home/dlab-user/.docker/config.json')
+ sudo('usermod -a -G docker ' + os_user)
+ sudo('update-rc.d docker defaults')
+ sudo('update-rc.d docker enable')
except Exception as err:
print('Error: {0}'.format(err))
sys.exit(1)
-
diff --git a/infrastructure-provisioning/src/general/scripts/os/jupyterlab_container_start.py b/infrastructure-provisioning/src/general/scripts/os/jupyterlab_container_start.py
index e7853ee..c4ff97b 100644
--- a/infrastructure-provisioning/src/general/scripts/os/jupyterlab_container_start.py
+++ b/infrastructure-provisioning/src/general/scripts/os/jupyterlab_container_start.py
@@ -38,14 +38,12 @@ jupyterlab_dir = '/home/' + args.os_user + '/.jupyterlab/'
def start_jupyterlab_container(jupyterlab_dir):
try:
with cd('{}'.format(jupyterlab_dir)):
- run('docker volume create -d local-persist \
- -o mountpoint=/home/{} --name=jup_volume'.format(args.os_user))
run('docker build --file Dockerfile_jupyterlab -t jupyter-lab .'.format(args.os_user))
container_id = run('docker ps | awk \'NR==2{print $1}\'')
if container_id != '':
run('docker stop ' + container_id)
run('docker run -d --restart unless-stopped -p 8888:8888 \
- -v jup_volume:/opt/legion/repository \
+ -v /home/{0}:/opt/legion/repository \
-v /home/{0}/.ssh/:/home/{0}/.ssh/ \
jupyter-lab:latest'.format(args.os_user))
except: sys.exit(1)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org