You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by dm...@apache.org on 2019/10/15 13:00:49 UTC
[incubator-dlab] branch DLAB-1107 updated: [DLAB-1107] Added
parameter for notebook image creation for test purpose[bugfix3]
This is an automated email from the ASF dual-hosted git repository.
dmysakovets pushed a commit to branch DLAB-1107
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/DLAB-1107 by this push:
new 039de65 [DLAB-1107] Added parameter for notebook image creation for test purpose[bugfix3]
039de65 is described below
commit 039de657b9382717d77fc3e93f00e8e7c35396a5
Author: Demyan Mysakovets <de...@gmail.com>
AuthorDate: Tue Oct 15 16:00:32 2019 +0300
[DLAB-1107] Added parameter for notebook image creation for test purpose[bugfix3]
---
infrastructure-provisioning/scripts/deploy_dlab.py | 2 +-
infrastructure-provisioning/src/general/conf/dlab.ini | 2 +-
.../src/general/lib/aws/actions_lib.py | 2 +-
.../src/general/scripts/aws/common_prepare_notebook.py | 2 +-
.../src/general/scripts/aws/deeplearning_configure.py | 6 +++---
.../src/general/scripts/aws/jupyter_configure.py | 4 ++--
.../src/general/scripts/aws/rstudio_configure.py | 4 ++--
.../general/scripts/aws/tensor-rstudio_configure.py | 6 +++---
.../src/general/scripts/aws/tensor_configure.py | 6 +++---
.../src/general/scripts/aws/zeppelin_configure.py | 4 ++--
.../general/scripts/azure/common_prepare_notebook.py | 9 ++++++++-
.../src/general/scripts/azure/jupyter_configure.py | 4 ++--
.../src/general/scripts/gcp/jupyter_configure.py | 18 +++++++++---------
.../src/ssn/templates/docker_build.py | 2 +-
14 files changed, 39 insertions(+), 32 deletions(-)
diff --git a/infrastructure-provisioning/scripts/deploy_dlab.py b/infrastructure-provisioning/scripts/deploy_dlab.py
index 653c0b4..a8dc13a 100644
--- a/infrastructure-provisioning/scripts/deploy_dlab.py
+++ b/infrastructure-provisioning/scripts/deploy_dlab.py
@@ -38,7 +38,7 @@ parser.add_argument('--conf_user_subnets_range', type=str, default='', help='Ran
'10.10.0.0/24 - 10.10.10.0/24')
parser.add_argument('--conf_additional_tags', type=str, default='', help='Additional tags in format '
'"Key1:Value1;Key2:Value2"')
-parser.add_argument('--conf_image_enabled', type=str, default='', help='Enable or Disable creating image at first time')#image_enabled
+parser.add_argument('--conf_image_enabled', type=str, default='', help='Enable or Disable creating image at first time')
parser.add_argument('--conf_shared_image_enabled', type=str, default='', help='Enable or Disable shared images')
parser.add_argument('--aws_user_predefined_s3_policies', type=str, default='', help='Predefined policies for users '
'instances')
diff --git a/infrastructure-provisioning/src/general/conf/dlab.ini b/infrastructure-provisioning/src/general/conf/dlab.ini
index 75fd1fe..4405373 100644
--- a/infrastructure-provisioning/src/general/conf/dlab.ini
+++ b/infrastructure-provisioning/src/general/conf/dlab.ini
@@ -45,7 +45,7 @@ pypi_mirror = pypi.doubanio.com
### Name of own GitLab SSL certificate
gitlab_certfile = dlab-gitlab.crt
###Enable or Disable creating image at first time
-# image_creation =
+# image_enabled =
###Enable or Disable shared images
# shared_image_enabled =
### CIDR of VPC
diff --git a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
index cb928b8..d6227f6 100644
--- a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
@@ -1350,7 +1350,7 @@ def create_image_from_instance(tag_name='', instance_name='', image_name='', tag
while image.state != 'available':
local("echo Waiting for image creation; sleep 20")
image.load()
- tag = {'Key': 'Name', 'Value': image_name},
+ tag = {'Key': 'Name', 'Value': image_name}
response = client.describe_images(ImageIds=[image.id]).get('Images')[0].get('BlockDeviceMappings')
for ebs in response:
if ebs.get('Ebs'):
diff --git a/infrastructure-provisioning/src/general/scripts/aws/common_prepare_notebook.py b/infrastructure-provisioning/src/general/scripts/aws/common_prepare_notebook.py
index f4cc7b1..654e161 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/common_prepare_notebook.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_prepare_notebook.py
@@ -75,7 +75,7 @@ if __name__ == "__main__":
os.environ['project_name'])
notebook_config['tag_name'] = '{}-Tag'.format(notebook_config['service_base_name'])
- if os.environ['conf_image_creation'] == 'true' and os.environ['conf_shared_image_enabled'] == 'false':
+ if os.environ['conf_image_enabled'] == 'true' and os.environ['conf_shared_image_enabled'] == 'false':
notebook_config['expected_image_name'] = '{0}-{1}-{2}-{3}-notebook-image'.format(notebook_config['service_base_name'],
os.environ['endpoint_name'],
os.environ['project_name'],
diff --git a/infrastructure-provisioning/src/general/scripts/aws/deeplearning_configure.py b/infrastructure-provisioning/src/general/scripts/aws/deeplearning_configure.py
index 76ed1ff..0be7104 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/deeplearning_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/deeplearning_configure.py
@@ -75,7 +75,7 @@ if __name__ == "__main__":
os.environ['project_name'])
notebook_config['tag_name'] = '{}-Tag'.format(notebook_config['service_base_name'])
notebook_config['dlab_ssh_user'] = os.environ['conf_os_user']
- notebook_config['image_creation'] = os.environ['conf_image_creation']
+ notebook_config['image_enabled'] = os.environ['conf_image_enabled']
notebook_config['shared_image_enabled'] = os.environ['conf_shared_image_enabled']
notebook_config['ip_address'] = get_instance_ip_address(notebook_config['tag_name'], notebook_config['instance_name']).get('Private')
@@ -258,10 +258,10 @@ if __name__ == "__main__":
remove_ec2(notebook_config['tag_name'], notebook_config['instance_name'])
sys.exit(1)
- if notebook_config['image_creation'] == 'true':
+ if notebook_config['image_enabled'] == 'true':
try:
print('[CREATING AMI]')
- ami_id = get_ami_id_by_name(`notebook_config['expected_image_name']`)
+ ami_id = get_ami_id_by_name(notebook_config['expected_image_name'])
if ami_id == '' and notebook_config['shared_image_enabled'] == 'false':
print("Looks like it's first time we configure notebook server. Creating image.")
try:
diff --git a/infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py b/infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py
index 4685de7..d77463c 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py
@@ -74,7 +74,7 @@ if __name__ == "__main__":
notebook_config['tag_name'] = '{}-Tag'.format(notebook_config['service_base_name'])
notebook_config['dlab_ssh_user'] = os.environ['conf_os_user']
notebook_config['shared_image_enabled'] = os.environ['conf_shared_image_enabled']
- notebook_config['image_creation'] = os.environ['conf_image_creation']
+ notebook_config['image_enabled'] = os.environ['conf_image_enabled']
notebook_config['ip_address'] = get_instance_ip_address(notebook_config['tag_name'], notebook_config['instance_name']).get('Private')
# generating variables regarding EDGE proxy on Notebook instance
@@ -271,7 +271,7 @@ if __name__ == "__main__":
remove_ec2(notebook_config['tag_name'], notebook_config['instance_name'])
sys.exit(1)
- if notebook_config['image_creation'] == 'true':
+ if notebook_config['image_enabled'] == 'true':
try:
print('[CREATING AMI]')
ami_id = get_ami_id_by_name(notebook_config['expected_image_name'])
diff --git a/infrastructure-provisioning/src/general/scripts/aws/rstudio_configure.py b/infrastructure-provisioning/src/general/scripts/aws/rstudio_configure.py
index 341c907..031bf41 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/rstudio_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/rstudio_configure.py
@@ -77,7 +77,7 @@ if __name__ == "__main__":
notebook_config['tag_name'] = '{}-Tag'.format(notebook_config['service_base_name'])
notebook_config['rstudio_pass'] = id_generator()
notebook_config['dlab_ssh_user'] = os.environ['conf_os_user']
- notebook_config['image_creation'] = os.environ['conf_image_creation']
+ notebook_config['image_enabled'] = os.environ['conf_image_enabled']
notebook_config['shared_image_enabled'] = os.environ['conf_shared_image_enabled']
notebook_config['ip_address'] = get_instance_ip_address(notebook_config['tag_name'], notebook_config['instance_name']).get('Private')
@@ -251,7 +251,7 @@ if __name__ == "__main__":
remove_ec2(notebook_config['tag_name'], notebook_config['instance_name'])
sys.exit(1)
- if notebook_config['image_creation'] == 'true':
+ if notebook_config['image_enabled'] == 'true':
try:
print('[CREATING AMI]')
ami_id = get_ami_id_by_name(`notebook_config['expected_image_name']`)
diff --git a/infrastructure-provisioning/src/general/scripts/aws/tensor-rstudio_configure.py b/infrastructure-provisioning/src/general/scripts/aws/tensor-rstudio_configure.py
index 1cd710b..087dac3 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/tensor-rstudio_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/tensor-rstudio_configure.py
@@ -77,7 +77,7 @@ if __name__ == "__main__":
os.environ['project_name'])
notebook_config['tag_name'] = '{}-Tag'.format(notebook_config['service_base_name'])
notebook_config['dlab_ssh_user'] = os.environ['conf_os_user']
- notebook_config['image_creation'] = os.environ['conf_image_creation']
+ notebook_config['image_enabled'] = os.environ['conf_image_enabled']
notebook_config['shared_image_enabled'] = os.environ['conf_shared_image_enabled']
notebook_config['ip_address'] = get_instance_ip_address(notebook_config['tag_name'],
notebook_config['instance_name']).get('Private')
@@ -256,10 +256,10 @@ if __name__ == "__main__":
remove_ec2(notebook_config['tag_name'], notebook_config['instance_name'])
sys.exit(1)
- if notebook_config['image_creation'] == 'true':
+ if notebook_config['image_enabled'] == 'true':
try:
print('[CREATING AMI]')
- ami_id = get_ami_id_by_name(`notebook_config['expected_image_name']`)
+ ami_id = get_ami_id_by_name(notebook_config['expected_image_name'])
if ami_id == '' and notebook_config['shared_image_enabled'] == 'false':
print("Looks like it's first time we configure notebook server. Creating image.")
try:
diff --git a/infrastructure-provisioning/src/general/scripts/aws/tensor_configure.py b/infrastructure-provisioning/src/general/scripts/aws/tensor_configure.py
index 2cce87f..44406e9 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/tensor_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/tensor_configure.py
@@ -77,7 +77,7 @@ if __name__ == "__main__":
os.environ['project_name'])
notebook_config['tag_name'] = '{}-Tag'.format(notebook_config['service_base_name'])
notebook_config['dlab_ssh_user'] = os.environ['conf_os_user']
- notebook_config['image_creation'] = os.environ['conf_image_creation']
+ notebook_config['image_enabled'] = os.environ['conf_image_enabled']
notebook_config['shared_image_enabled'] = os.environ['conf_shared_image_enabled']
tag = {"Key": notebook_config['tag_name'],
"Value": "{}-{}-subnet".format(notebook_config['service_base_name'], os.environ['project_name'])}
@@ -251,10 +251,10 @@ if __name__ == "__main__":
remove_ec2(notebook_config['tag_name'], notebook_config['instance_name'])
sys.exit(1)
- if notebook_config['image_creation'] == 'true':
+ if notebook_config['image_enabled'] == 'true':
try:
print('[CREATING AMI]')
- ami_id = get_ami_id_by_name(`notebook_config['expected_image_name']`)
+ ami_id = get_ami_id_by_name(notebook_config['expected_image_name'])
if ami_id == '' and notebook_config['shared_image_enabled'] == 'false':
print("Looks like it's first time we configure notebook server. Creating image.")
try:
diff --git a/infrastructure-provisioning/src/general/scripts/aws/zeppelin_configure.py b/infrastructure-provisioning/src/general/scripts/aws/zeppelin_configure.py
index 6a1ad93..846346a 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/zeppelin_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/zeppelin_configure.py
@@ -77,7 +77,7 @@ if __name__ == "__main__":
os.environ['project_name'])
notebook_config['tag_name'] = '{}-Tag'.format(notebook_config['service_base_name'])
notebook_config['dlab_ssh_user'] = os.environ['conf_os_user']
- notebook_config['image_creation'] = os.environ['conf_image_creation']
+ notebook_config['image_enabled'] = os.environ['conf_image_enabled']
notebook_config['shared_image_enabled'] = os.environ['conf_shared_image_enabled']
notebook_config['ip_address'] = get_instance_ip_address(notebook_config['tag_name'], notebook_config['instance_name']).get('Private')
@@ -270,7 +270,7 @@ if __name__ == "__main__":
remove_ec2(notebook_config['tag_name'], notebook_config['instance_name'])
sys.exit(1)
- if notebook_config['image_creation'] == 'true':
+ if notebook_config['image_enabled'] == 'true':
try:
print('[CREATING AMI]')
ami_id = get_ami_id_by_name(`notebook_config['expected_image_name']`)
diff --git a/infrastructure-provisioning/src/general/scripts/azure/common_prepare_notebook.py b/infrastructure-provisioning/src/general/scripts/azure/common_prepare_notebook.py
index 8d7270d..4a0f4dc 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/common_prepare_notebook.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_prepare_notebook.py
@@ -90,11 +90,18 @@ if __name__ == "__main__":
sudo_group = 'wheel'
notebook_config['image_type'] = 'default'
- notebook_config['expected_image_name'] = '{0}-{1}-{2}-{3}-notebook-image'.format(
+ notebook_config['shared_image_enabled'] = os.environ['conf_shared_image_enabled']
+ if notebook_config['shared_image_enabled'] == 'false':
+ notebook_config['expected_image_name'] = '{0}-{1}-{2}-{3}-notebook-image'.format(
notebook_config['service_base_name'],
notebook_config['endpoint_name'],
notebook_config['project_name'],
notebook_config['application'])
+ else:
+ notebook_config['expected_image_name'] = '{0}-{1}-{2}-notebook-image'.format(
+ notebook_config['service_base_name'],
+ notebook_config['endpoint_name'],
+ notebook_config['application'])
notebook_config['notebook_image_name'] = (lambda x: os.environ['notebook_image_name'].lower().replace('_', '-') if (x != 'None' and x != '')
else notebook_config['expected_image_name'])(str(os.environ.get('notebook_image_name')))
print('Searching pre-configured images')
diff --git a/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py b/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py
index e41346c..76b4be7 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py
@@ -91,7 +91,7 @@ if __name__ == "__main__":
"endpoint_tag": notebook_config['endpoint_tag'],
"Exploratory": notebook_config['exploratory_name'],
os.environ['conf_billing_tag_key']: os.environ['conf_billing_tag_value']}
- notebook_config['image_creation'] = os.environ['conf_image_creation']
+ notebook_config['image_enabled'] = os.environ['conf_image_enabled']
notebook_config['shared_image_enabled'] = os.environ['conf_shared_image_enabled']
notebook_config['ip_address'] = AzureMeta().get_private_ip_address(notebook_config['resource_group_name'],
notebook_config['instance_name'])
@@ -259,7 +259,7 @@ if __name__ == "__main__":
AzureActions().remove_instance(notebook_config['resource_group_name'], notebook_config['instance_name'])
sys.exit(1)
- if notebook_config['image_creation'] == 'true':
+ if notebook_config['image_enabled'] == 'true':
try:
print('[CREATING IMAGE]')
image = AzureMeta().get_image(notebook_config['resource_group_name'], notebook_config['expected_image_name'])
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py
index 5cad4f5..92fdce7 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py
@@ -54,19 +54,19 @@ if __name__ == "__main__":
notebook_config['project_name'],
notebook_config['exploratory_name'])
if notebook_config['shared_image_enabled'] == 'false':
- notebook_config['expected_primary_image_name'] = '{}-{}-{}-primary-image'.format(
- notebook_config['service_base_name'], notebook_config['endpoint_tag'], os.environ['application'])
+ notebook_config['expected_primary_image_name'] = '{}-{}-{}-{}-primary-image'.format(
+ notebook_config['service_base_name'], notebook_config['endpoint_tag'], notebook_config['project_name'], os.environ['application'])
notebook_config['expected_secondary_image_name'] = '{}-{}-{}-secondary-image'.format(
- notebook_config['service_base_name'], notebook_config['endpoint_tag'], os.environ['application'])
+ notebook_config['service_base_name'], notebook_config['endpoint_tag'], notebook_config['project_name'], os.environ['application'])
notebook_config['image_labels'] = {"sbn": notebook_config['service_base_name'],
"endpoint_tag": notebook_config['endpoint_tag'],
"project_tag": notebook_config['project_tag'],
"product": "dlab"}
else:
- notebook_config['expected_primary_image_name'] = '{}-{}-{}-{}-primary-image'.format(
- notebook_config['service_base_name'], notebook_config['endpoint_tag'], notebook_config['project_name'], os.environ['application'])
- notebook_config['expected_secondary_image_name'] = '{}-{}-{}-{}-secondary-image'.format(
- notebook_config['service_base_name'], notebook_config['endpoint_tag'], notebook_config['project_name'], os.environ['application'])
+ notebook_config['expected_primary_image_name'] = '{}-{}-{}-primary-image'.format(
+ notebook_config['service_base_name'], notebook_config['endpoint_tag'], os.environ['application'])
+ notebook_config['expected_secondary_image_name'] = '{}-{}-{}-secondary-image'.format(
+ notebook_config['service_base_name'], notebook_config['endpoint_tag'], os.environ['application'])
notebook_config['image_labels'] = {"sbn": notebook_config['service_base_name'],
"endpoint_tag": notebook_config['endpoint_tag'],
"product": "dlab"}
@@ -77,7 +77,7 @@ if __name__ == "__main__":
notebook_config['ssh_key_path'] = '{0}{1}.pem'.format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
notebook_config['dlab_ssh_user'] = os.environ['conf_os_user']
notebook_config['zone'] = os.environ['gcp_zone']
- notebook_config['image_creation'] = os.environ['conf_image_creation']
+ notebook_config['image_enabled'] = os.environ['conf_image_enabled']
notebook_config['shared_image_enabled'] = os.environ['conf_shared_image_enabled']
try:
@@ -203,7 +203,7 @@ if __name__ == "__main__":
GCPActions().remove_instance(notebook_config['instance_name'], notebook_config['zone'])
sys.exit(1)
- if notebook_config['image_creation'] == 'true':
+ if notebook_config['image_enabled'] == 'true':
try:
print('[CREATING IMAGE]')
primary_image_id = GCPMeta().get_image_by_name(notebook_config['expected_primary_image_name'])
diff --git a/infrastructure-provisioning/src/ssn/templates/docker_build.py b/infrastructure-provisioning/src/ssn/templates/docker_build.py
index fd44b42..07c27b0 100644
--- a/infrastructure-provisioning/src/ssn/templates/docker_build.py
+++ b/infrastructure-provisioning/src/ssn/templates/docker_build.py
@@ -21,8 +21,8 @@
#
# ******************************************************************************
-
from fabric.api import *
+from os.path import exists
import sys
import os
import traceback
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org