You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by dm...@apache.org on 2019/11/29 12:43:58 UTC
[incubator-dlab] 01/01: [DLAB-1326] Implemented proxy configuration
for docker
This is an automated email from the ASF dual-hosted git repository.
dmysakovets pushed a commit to branch DLAB-1326
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 8b5ff32d3f5891e45966e6a651d71e7503891dfa
Author: Demyan Mysakovets <de...@gmail.com>
AuthorDate: Fri Nov 29 14:43:38 2019 +0200
[DLAB-1326] Implemented proxy configuration for docker
---
.../src/general/lib/os/fab.py | 48 ++++------------------
.../general/scripts/aws/jupyterlab_configure.py | 15 +++++++
.../general/scripts/gcp/jupyterlab_configure.py | 16 ++++++++
.../src/general/scripts/gcp/superset_configure.py | 15 +++++++
...iner_start.py => configure_proxy_for_docker.py} | 42 ++++++++-----------
.../scripts/os/jupyterlab_container_start.py | 4 +-
6 files changed, 71 insertions(+), 69 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index bbb5e39..8447a41 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -193,31 +193,16 @@ def configure_jupyter(os_user, jupyter_conf_file, templates_dir, jupyter_version
print('Error:', str(err))
sys.exit(1)
-def configure_docker(os_user, http_file, https_file):
+def configure_docker(os_user, ):
try:
if not exists('/home/' + os_user + '/.ensure_dir/docker_ensured'):
+ docker_version = os.environ['ssn_docker_version']
sudo('curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -')
sudo('add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) \
stable"')
sudo('apt-get update')
sudo('apt-cache policy docker-ce')
- sudo('apt-get install -y docker-ce')
- sudo('mkdir -p /etc/systemd/system/docker.service.d')
- sudo('touch {}'.format(http_file))
- sudo('echo -e \'[Service] \nEnvironment=\"HTTP_PROXY=\'$http_proxy\'\"\' > {}'.format(http_file))
- sudo('touch {}'.format(https_file))
- sudo('echo -e \'[Service] \nEnvironment=\"HTTPS_PROXY=\'$http_proxy\'\"\' > {}'.format(https_file))
- sudo('mkdir /home/{}/.docker'.format(os_user))
- sudo('touch /home/{}/.docker/config.json'.format(os_user))
- sudo('echo -e \'{\n "proxies":\n {\n "default":\n {\n "httpProxy":"\'$http_proxy\'",\n "httpsProxy":"\'$http_proxy\'"\n }\n }\n}\' > /home/dlab-user/.docker/config.json')
- sudo('usermod -a -G docker ' + os_user)
- sudo('update-rc.d docker defaults')
- sudo('update-rc.d docker enable')
- sudo('wget https://raw.githubusercontent.com/CWSpear/local-persist/master/scripts/install.sh && chmod +x install.sh')
- sudo('sed -i "66s/curl/sudo curl/g" install.sh')
- sudo('sed -i "s/sudo curl/sudo -E curl/g" install.sh')
- run('sudo -E ./install.sh')
- sudo('rm install.sh')
+ sudo('apt-get install -y docker-ce={}~ce-0~ubuntu'.format(docker_version))
sudo('touch /home/{}/.ensure_dir/docker_ensured'.format(os_user))
except Exception as err:
print('Failed to configure Docker:', str(err))
@@ -834,33 +819,14 @@ def update_hosts_file(os_user):
print('Failed to update hosts file', str(err))
sys.exit(1)
-def ensure_docker_compose(dlab_path, os_user):
+def ensure_docker_compose(os_user):
try:
- if not exists(dlab_path + 'tmp/docker_daemon_ensured'):
- docker_version = os.environ['ssn_docker_version']
- sudo('curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -')
- sudo('add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) \
- stable"')
- sudo('apt-get update')
- sudo('apt-cache policy docker-ce')
- sudo('apt-get install -y docker-ce={}~ce-0~ubuntu'.format(docker_version))
- print('Proxy for docker configuring')
- sudo('mkdir -p /etc/systemd/system/docker.service.d')
- with cd('/etc/systemd/system/docker.service.d'):
- sudo('echo -e \'[Service] \nEnvironment=\"HTTP_PROXY=\'$http_proxy\'\"\' > http-proxy.conf')
- sudo('echo -e \'[Service] \nEnvironment=\"HTTPS_PROXY=\'$http_proxy\'\"\' > https-proxy.conf')
- sudo('mkdir -p /home/' + os_user + '/.docker')
- with cd('/home/' + os_user + '/.docker'):
- sudo('echo -e \'{\n "proxies":\n {\n "default":\n {\n "httpProxy":"\'$http_proxy\'",\n "httpsProxy":"\'$http_proxy\'"\n }\n }\n}\' > /home/' + os_user + '/.docker/config.json')
- sudo('usermod -a -G docker ' + os_user)
- sudo('update-rc.d docker defaults')
- sudo('update-rc.d docker enable')
- sudo('mkdir -p ' + dlab_path + 'tmp')
- sudo('touch ' + dlab_path + 'tmp/docker_daemon_ensured')
- if not exists(dlab_path + 'tmp/docker_compose_ensured'):
+ configure_docker(os_user)
+ if not exists('/home/{}/.ensure_dir/docker_compose_ensured'.format(os_user)):
docker_compose_version = "1.24.1"
sudo('curl -L https://github.com/docker/compose/releases/download/{}/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose'.format(docker_compose_version))
sudo('chmod +x /usr/local/bin/docker-compose')
+ sudo('touch /home/{}/.ensure_dir/docker_compose_ensured'.format(os_user))
sudo('systemctl daemon-reload')
sudo('systemctl restart docker')
return True
diff --git a/infrastructure-provisioning/src/general/scripts/aws/jupyterlab_configure.py b/infrastructure-provisioning/src/general/scripts/aws/jupyterlab_configure.py
index 1b36187..157e140 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/jupyterlab_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/jupyterlab_configure.py
@@ -290,6 +290,21 @@ if __name__ == "__main__":
sys.exit(1)
try:
+ print('[CONFIGURING PROXY FOR DOCKER]')
+ logging.info('[CONFIGURING PROXY FOR DOCKER]')
+ params = "--os_user {} ".format(notebook_config['dlab_ssh_user'])
+ try:
+ local("~/scripts/configure_proxy_for_docker.py {}".format(params))
+ except:
+ traceback.print_exc()
+ raise Exception
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ append_result("Failed to configure proxy for docker.", str(err))
+ remove_ec2(notebook_config['tag_name'], notebook_config['instance_name'])
+ sys.exit(1)
+
+ try:
print('[STARTING JUPYTER CONTAINER]')
logging.info('[STARTING JUPYTER CONTAINER]')
params = "--hostname {} " \
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/jupyterlab_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/jupyterlab_configure.py
index 9246352..0bf279c 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/jupyterlab_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/jupyterlab_configure.py
@@ -280,6 +280,22 @@ if __name__ == "__main__":
sys.exit(1)
try:
+ print('[CONFIGURING PROXY FOR DOCKER]')
+ logging.info('[CONFIGURING PROXY FOR DOCKER]')
+ params = "--os_user {} ".format(notebook_config['dlab_ssh_user'])
+ try:
+ local("~/scripts/configure_proxy_for_docker.py {}".format(params))
+ except:
+ traceback.print_exc()
+ raise Exception
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ append_result("Failed to configure proxy for docker.", str(err))
+ remove_ec2(notebook_config['tag_name'], notebook_config['instance_name'])
+ sys.exit(1)
+
+
+ try:
print('[STARTING JUPYTER CONTAINER]')
logging.info('[STARTING JUPYTER CONTAINER]')
params = "--hostname {} " \
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/superset_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/superset_configure.py
index b3ae8da..6c19177 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/superset_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/superset_configure.py
@@ -277,6 +277,21 @@ if __name__ == "__main__":
GCPActions().remove_instance(notebook_config['instance_name'], notebook_config['zone'])
sys.exit(1)
+ try:
+ print('[CONFIGURING PROXY FOR DOCKER]')
+ logging.info('[CONFIGURING PROXY FOR DOCKER]')
+ params = "--os_user {} ".format(notebook_config['dlab_ssh_user'])
+ try:
+ local("~/scripts/configure_proxy_for_docker.py {}".format(params))
+ except:
+ traceback.print_exc()
+ raise Exception
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ append_result("Failed to configure proxy for docker.", str(err))
+ remove_ec2(notebook_config['tag_name'], notebook_config['instance_name'])
+ sys.exit(1)
+
# generating output information
ip_address = GCPMeta().get_private_ip_address(notebook_config['instance_name'])
superset_ip_url = "http://" + ip_address + ":8088/{}/".format(notebook_config['exploratory_name'])
diff --git a/infrastructure-provisioning/src/general/scripts/os/jupyterlab_container_start.py b/infrastructure-provisioning/src/general/scripts/os/configure_proxy_for_docker.py
similarity index 56%
copy from infrastructure-provisioning/src/general/scripts/os/jupyterlab_container_start.py
copy to infrastructure-provisioning/src/general/scripts/os/configure_proxy_for_docker.py
index e7853ee..18791a2 100644
--- a/infrastructure-provisioning/src/general/scripts/os/jupyterlab_container_start.py
+++ b/infrastructure-provisioning/src/general/scripts/os/configure_proxy_for_docker.py
@@ -21,44 +21,36 @@
#
# ******************************************************************************
-import sys
-import os
-from dlab.notebook_lib import *
-from dlab.fab import *
from fabric.api import *
+import sys
+
parser = argparse.ArgumentParser()
-parser.add_argument('--hostname', type=str, default='')
-parser.add_argument('--keyfile', type=str, default='')
parser.add_argument('--os_user', type=str, default='')
args = parser.parse_args()
-jupyterlab_dir = '/home/' + args.os_user + '/.jupyterlab/'
-
-def start_jupyterlab_container(jupyterlab_dir):
- try:
- with cd('{}'.format(jupyterlab_dir)):
- run('docker volume create -d local-persist \
- -o mountpoint=/home/{} --name=jup_volume'.format(args.os_user))
- run('docker build --file Dockerfile_jupyterlab -t jupyter-lab .'.format(args.os_user))
- container_id = run('docker ps | awk \'NR==2{print $1}\'')
- if container_id != '':
- run('docker stop ' + container_id)
- run('docker run -d --restart unless-stopped -p 8888:8888 \
- -v jup_volume:/opt/legion/repository \
- -v /home/{0}/.ssh/:/home/{0}/.ssh/ \
- jupyter-lab:latest'.format(args.os_user))
- except: sys.exit(1)
+http_file = '/etc/systemd/system/docker.service.d/http-proxy.conf'
+https_file = '/etc/systemd/system/docker.service.d/https-proxy.conf'
if __name__ == "__main__":
print("Configure connections")
env['connection_attempts'] = 100
env.key_filename = [args.keyfile]
env.host_string = args.os_user + '@' + args.hostname
- print("Starting Jupyter container")
+ print("Configuring proxy for docker")
try:
- start_jupyterlab_container(jupyterlab_dir)
+ sudo('mkdir -p /etc/systemd/system/docker.service.d')
+ sudo('touch {}'.format(http_file))
+ sudo('echo -e \'[Service] \nEnvironment=\"HTTP_PROXY=\'$http_proxy\'\"\' > {}'.format(http_file))
+ sudo('touch {}'.format(https_file))
+ sudo('echo -e \'[Service] \nEnvironment=\"HTTPS_PROXY=\'$http_proxy\'\"\' > {}'.format(https_file))
+ sudo('mkdir /home/{}/.docker'.format(os_user))
+ sudo('touch /home/{}/.docker/config.json'.format(os_user))
+ sudo(
+ 'echo -e \'{\n "proxies":\n {\n "default":\n {\n "httpProxy":"\'$http_proxy\'",\n "httpsProxy":"\'$http_proxy\'"\n }\n }\n}\' > /home/dlab-user/.docker/config.json')
+ sudo('usermod -a -G docker ' + os_user)
+ sudo('update-rc.d docker defaults')
+ sudo('update-rc.d docker enable')
except Exception as err:
print('Error: {0}'.format(err))
sys.exit(1)
-
diff --git a/infrastructure-provisioning/src/general/scripts/os/jupyterlab_container_start.py b/infrastructure-provisioning/src/general/scripts/os/jupyterlab_container_start.py
index e7853ee..c4ff97b 100644
--- a/infrastructure-provisioning/src/general/scripts/os/jupyterlab_container_start.py
+++ b/infrastructure-provisioning/src/general/scripts/os/jupyterlab_container_start.py
@@ -38,14 +38,12 @@ jupyterlab_dir = '/home/' + args.os_user + '/.jupyterlab/'
def start_jupyterlab_container(jupyterlab_dir):
try:
with cd('{}'.format(jupyterlab_dir)):
- run('docker volume create -d local-persist \
- -o mountpoint=/home/{} --name=jup_volume'.format(args.os_user))
run('docker build --file Dockerfile_jupyterlab -t jupyter-lab .'.format(args.os_user))
container_id = run('docker ps | awk \'NR==2{print $1}\'')
if container_id != '':
run('docker stop ' + container_id)
run('docker run -d --restart unless-stopped -p 8888:8888 \
- -v jup_volume:/opt/legion/repository \
+ -v /home/{0}:/opt/legion/repository \
-v /home/{0}/.ssh/:/home/{0}/.ssh/ \
jupyter-lab:latest'.format(args.os_user))
except: sys.exit(1)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org