You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by dm...@apache.org on 2019/10/31 09:30:26 UTC
[incubator-dlab] 01/01: [DLAB-1195] Fixed dataengine creation from
notebook AMI[GCP]
This is an automated email from the ASF dual-hosted git repository.
dmysakovets pushed a commit to branch DLAB-1195
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 57b812a44d6970cb32bf4dbfcf7f18593c8b9712
Author: Demyan Mysakovets <de...@gmail.com>
AuthorDate: Thu Oct 31 11:30:10 2019 +0200
[DLAB-1195] Fixed dataengine creation from notebook AMI[GCP]
---
.../src/general/scripts/gcp/dataengine_prepare.py | 37 ++++++++++++++--------
1 file changed, 24 insertions(+), 13 deletions(-)
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py
index aea75c1..d0cf7ea 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py
@@ -99,9 +99,9 @@ if __name__ == "__main__":
data_engine['slave_node_name'] = data_engine['cluster_name'] + '-s'
data_engine['instance_count'] = int(os.environ['dataengine_instance_count'])
data_engine['notebook_name'] = os.environ['notebook_instance_name']
- data_engine['image_name'] = os.environ['gcp_{}_image_name'.format(os.environ['conf_os_family'])]
data_engine['primary_disk_size'] = '30'
+ data_engine['secondary_disk_size'] = os.environ['notebook_disk_size']
data_engine['shared_image_enabled'] = os.environ['conf_shared_image_enabled']
if data_engine['shared_image_enabled'] == 'false':
@@ -127,6 +127,13 @@ if __name__ == "__main__":
data_engine['primary_image_name'] = 'global/images/{}'.format(
data_engine['primary_image_name'].get('name'))
+ data_engine['secondary_image_name'] = GCPMeta().get_image_by_name(data_engine['expected_secondary_image_name'])
+ if data_engine['secondary_image_name'] == '':
+ data_engine['secondary_image_name'] = 'None'
+ else:
+ print('Pre-configured secondary image found. Using: {}'.format(data_engine['secondary_image_name'].get('name')))
+ data_engine['secondary_image_name'] = 'global/images/{}'.format(data_engine['secondary_image_name'].get('name'))
+
data_engine['gpu_accelerator_type'] = 'None'
if os.environ['application'] in ('tensor', 'tensor-rstudio', 'deeplearning'):
data_engine['gpu_accelerator_type'] = os.environ['gcp_gpu_accelerator_type']
@@ -152,15 +159,17 @@ if __name__ == "__main__":
try:
logging.info('[CREATE MASTER NODE]')
print('[CREATE MASTER NODE]')
- params = "--instance_name {} --region {} --zone {} --vpc_name {} --subnet_name {} --instance_size {} " \
- "--ssh_key_path {} --initial_user {} --service_account_name {} --image_name {} " \
- "--instance_class {} --primary_disk_size {} " \
- "--gpu_accelerator_type {} --network_tag {} --cluster_name {} --labels '{}'".\
+ params = "--instance_name {0} --region {1} --zone {2} --vpc_name {3} --subnet_name {4} --instance_size {5} " \
+ "--ssh_key_path {6} --initial_user {7} --service_account_name {8} --image_name {9} " \
+ "--secondary_image_name {10} --instance_class {11} --primary_disk_size {12} --secondary_disk_size {13} " \
+ "--gpu_accelerator_type {14} --network_tag {15} --cluster_name {16} --labels '{17}'". \
format(data_engine['master_node_name'], data_engine['region'], data_engine['zone'], data_engine['vpc_name'],
data_engine['subnet_name'], data_engine['master_size'], data_engine['ssh_key_path'], initial_user,
data_engine['dataengine_service_account_name'], data_engine['primary_image_name'],
- 'dataengine', '30', data_engine['gpu_accelerator_type'],
- data_engine['network_tag'], data_engine['cluster_name'], json.dumps(data_engine['master_labels']))
+ data_engine['secondary_image_name'], 'dataengine', data_engine['primary_disk_size'],
+ data_engine['secondary_disk_size'], data_engine['gpu_accelerator_type'],
+ data_engine['network_tag'], data_engine['cluster_name'],
+ json.dumps(data_engine['master_labels']))
try:
local("~/scripts/{}.py {}".format('common_create_instance', params))
except:
@@ -177,15 +186,17 @@ if __name__ == "__main__":
logging.info('[CREATE SLAVE NODE {}]'.format(i + 1))
print('[CREATE SLAVE NODE {}]'.format(i + 1))
slave_name = data_engine['slave_node_name'] + '{}'.format(i + 1)
- params = "--instance_name {} --region {} --zone {} --vpc_name {} --subnet_name {} --instance_size {} " \
- "--ssh_key_path {} --initial_user {} --service_account_name {} --image_name {} " \
- "--instance_class {} --primary_disk_size {} " \
- "--gpu_accelerator_type {} --network_tag {} --cluster_name {} --labels '{}'". \
+ params = "--instance_name {0} --region {1} --zone {2} --vpc_name {3} --subnet_name {4} --instance_size {5} " \
+ "--ssh_key_path {6} --initial_user {7} --service_account_name {8} --image_name {9} " \
+ "--secondary_image_name {10} --instance_class {11} --primary_disk_size {12} --secondary_disk_size {13} " \
+ "--gpu_accelerator_type {14} --network_tag {15} --cluster_name {16} --labels '{17}'". \
format(slave_name, data_engine['region'], data_engine['zone'],
data_engine['vpc_name'], data_engine['subnet_name'], data_engine['slave_size'],
data_engine['ssh_key_path'], initial_user, data_engine['dataengine_service_account_name'],
- data_engine['primary_image_name'], 'dataengine', '30',
- data_engine['gpu_accelerator_type'], data_engine['network_tag'], data_engine['cluster_name'],
+ data_engine['primary_image_name'], data_engine['secondary_image_name'], 'dataengine',
+ data_engine['primary_disk_size'],
+ data_engine['secondary_disk_size'], data_engine['gpu_accelerator_type'],
+ data_engine['network_tag'], data_engine['cluster_name'],
json.dumps(data_engine['slave_labels']))
try:
local("~/scripts/{}.py {}".format('common_create_instance', params))
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org