You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2021/08/04 12:20:58 UTC
[incubator-datalab] branch DATALAB-408 created (now f159de7)
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a change to branch DATALAB-408
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git.
at f159de7 [DATALAB-408]: usage of docker images from nexus during ssn deployment
This branch includes the following new commits:
new f159de7 [DATALAB-408]: usage of docker images from nexus during ssn deployment
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 01/01: [DATALAB-408]: usage of docker images
from nexus during ssn deployment
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-408
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit f159de70afa70126963e900079baec2582a6831b
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Wed Aug 4 15:20:42 2021 +0300
[DATALAB-408]: usage of docker images from nexus during ssn deployment
---
.../scripts/deploy_datalab.py | 16 ++++--
.../src/ssn/scripts/configure_docker.py | 58 ++++++++++++++--------
2 files changed, 49 insertions(+), 25 deletions(-)
diff --git a/infrastructure-provisioning/scripts/deploy_datalab.py b/infrastructure-provisioning/scripts/deploy_datalab.py
index e3bdd4c..45fde90 100644
--- a/infrastructure-provisioning/scripts/deploy_datalab.py
+++ b/infrastructure-provisioning/scripts/deploy_datalab.py
@@ -206,11 +206,21 @@ def generate_docker_command():
def build_docker_images(args):
- # Building base and ssn docker images
- subprocess.run('cd {2}; sudo docker build --build-arg OS={0} --build-arg SRC_PATH="infrastructure-provisioning/src/" --file '
+ if args.conf_repository_user and args.conf_repository_pass and args.conf_repository_address:
+ subprocess.run( 'sudo docker login -u {0} -p {1} {2}:8083'
+ .format(args.conf_repository_user, args.conf_repository_pass, args.conf_repository_address), shell=True, check=True)
+ subprocess.run('docker pull {}:8083/docker.datalab-base-{}'.format(args.conf_repository_address, args.cloud_provider), shell=True, check=True)
+ subprocess.run('docker image tag {}:8083/docker.datalab-base-{} docker.datalab-base'.format(args.conf_repository_address, args.cloud_provider), shell=True, check=True)
+ subprocess.run('docker image rm {}:8083/docker.datalab-base-{}'.format(args.conf_repository_address, args.cloud_provider), shell=True, check=True)
+ subprocess.run('docker pull {}:8083/docker.datalab-ssn-{}'.format(args.conf_repository_address, args.cloud_provider), shell=True, check=True)
+ subprocess.run('docker image tag {}:8083/docker.datalab-ssn-{} docker.datalab-ssn'.format(args.conf_repository_address, args.cloud_provider), shell=True, check=True)
+ subprocess.run('docker image rm {}:8083/docker.datalab-ssn-{}'.format(args.conf_repository_address, args.cloud_provider), shell=True, check=True)
+ else:
+ # Building base and ssn docker images
+ subprocess.run('cd {2}; sudo docker build --build-arg OS={0} --build-arg SRC_PATH="infrastructure-provisioning/src/" --file '
'infrastructure-provisioning/src/general/files/{1}/'
'base_Dockerfile -t docker.datalab-base .'.format(args.conf_os_family, args.conf_cloud_provider, args.workspace_path), shell=True, check=True)
- subprocess.run('cd {2}; sudo docker build --build-arg OS={0} --file infrastructure-provisioning/src/general/files/{1}/'
+ subprocess.run('cd {2}; sudo docker build --build-arg OS={0} --file infrastructure-provisioning/src/general/files/{1}/'
'ssn_Dockerfile -t docker.datalab-ssn .'.format(args.conf_os_family, args.conf_cloud_provider, args.workspace_path), shell=True, check=True)
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_docker.py b/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
index aa20a68..14db2a7 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
@@ -113,28 +113,42 @@ def login_in_gcr(os_user, gcr_creds, odahu_image, datalab_path, cloud_provider):
def build_docker_images(image_list, region, datalab_path):
try:
- host_string = '{}@{}'.format(args.os_user, args.hostname)
- if os.environ['conf_cloud_provider'] == 'azure':
- conn.local('scp -i {} /root/azure_auth.json {}:{}sources/infrastructure-provisioning/src/base/'
- 'azure_auth.json'.format(args.keyfile, host_string, args.datalab_path))
- conn.sudo('cp {0}sources/infrastructure-provisioning/src/base/azure_auth.json '
- '/home/{1}/keys/azure_auth.json'.format(args.datalab_path, args.os_user))
- if region == 'cn-north-1':
- add_china_repository(datalab_path)
- for image in image_list:
- name = image['name']
- tag = image['tag']
- conn.sudo('cp {0}sources/infrastructure-provisioning/src/general/files/{1}/{2}_description.json '
- '{0}sources/infrastructure-provisioning/src/{2}/description.json'.format(args.datalab_path, args.cloud_provider, name))
- if name == 'base':
- conn.sudo("bash -c 'cd {4}sources/infrastructure-provisioning/src/; docker build --build-arg OS={2} "
- "--build-arg SRC_PATH=\"\" --file general/files/{3}/{0}_Dockerfile -t docker.datalab-{0}:{1} "
- ".'".format(name, tag, args.os_family, args.cloud_provider, args.datalab_path))
- else:
- conn.sudo("bash -c 'cd {4}sources/infrastructure-provisioning/src/; docker build --build-arg OS={2} "
- "--file general/files/{3}/{0}_Dockerfile -t docker.datalab-{0}:{1} .'".format(name, tag, args.os_family, args.cloud_provider, args.datalab_path))
- conn.sudo('rm -f {}sources/infrastructure-provisioning/src/base/azure_auth.json'.format(args.datalab_path))
- return True
+ if 'conf_repository_user' in os.environ and 'conf_repository_pass' in os.environ and 'conf_repository_address' in os.environ:
+ conn.sudo('sudo docker login -u {0} -p {1} {2}:8083'
+ .format(os.environ['conf_repository_user'], os.environ['conf_repository_pass'], os.environ['conf_repository_address']))
+ for image in image_list:
+ name = image['name']
+ tag = image['tag']
+ conn.sudo('docker pull {}:8083/docker.datalab-{}-{}:{}'
+ .format(os.environ['conf_repository_address'], os.environ['conf_cloud_provider'], name, tag))
+ conn.sudo('docker image tag {0}:8083/docker.datalab-{1}-{2}:{3} docker.datalab-{1}-{2}:{3}'
+ .format(os.environ['conf_repository_address'], os.environ['conf_cloud_provider'], name, tag))
+ conn.sudo('docker image rm {}:8083/docker.datalab-{}-{}:{}'
+ .format(os.environ['conf_repository_address'], os.environ['conf_cloud_provider'], name, tag))
+ return True
+ else:
+ host_string = '{}@{}'.format(args.os_user, args.hostname)
+ if os.environ['conf_cloud_provider'] == 'azure':
+ conn.local('scp -i {} /root/azure_auth.json {}:{}sources/infrastructure-provisioning/src/base/'
+ 'azure_auth.json'.format(args.keyfile, host_string, args.datalab_path))
+ conn.sudo('cp {0}sources/infrastructure-provisioning/src/base/azure_auth.json '
+ '/home/{1}/keys/azure_auth.json'.format(args.datalab_path, args.os_user))
+ if region == 'cn-north-1':
+ add_china_repository(datalab_path)
+ for image in image_list:
+ name = image['name']
+ tag = image['tag']
+ conn.sudo('cp {0}sources/infrastructure-provisioning/src/general/files/{1}/{2}_description.json '
+ '{0}sources/infrastructure-provisioning/src/{2}/description.json'.format(args.datalab_path, args.cloud_provider, name))
+ if name == 'base':
+ conn.sudo("bash -c 'cd {4}sources/infrastructure-provisioning/src/; docker build --build-arg OS={2} "
+ "--build-arg SRC_PATH=\"\" --file general/files/{3}/{0}_Dockerfile -t docker.datalab-{0}:{1} "
+ ".'".format(name, tag, args.os_family, args.cloud_provider, args.datalab_path))
+ else:
+ conn.sudo("bash -c 'cd {4}sources/infrastructure-provisioning/src/; docker build --build-arg OS={2} "
+ "--file general/files/{3}/{0}_Dockerfile -t docker.datalab-{0}:{1} .'".format(name, tag, args.os_family, args.cloud_provider, args.datalab_path))
+ conn.sudo('rm -f {}sources/infrastructure-provisioning/src/base/azure_auth.json'.format(args.datalab_path))
+ return True
except:
return False
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org