You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by om...@apache.org on 2020/02/27 09:55:52 UTC
[incubator-dlab] 01/01: Merge branch 'develop' of
github.com:apache/incubator-dlab into DLAB-1546
This is an automated email from the ASF dual-hosted git repository.
omartushevskyi pushed a commit to branch DLAB-1546
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit a4798b1a55abc56aab711efd2c6dac70878275bd
Merge: e8dc4ad 0e8c625
Author: Oleh Martushevskyi <Ol...@epam.com>
AuthorDate: Thu Feb 27 11:55:28 2020 +0200
Merge branch 'develop' of github.com:apache/incubator-dlab into DLAB-1546
Conflicts:
infrastructure-provisioning/src/general/scripts/aws/project_prepare.py
infrastructure-provisioning/src/general/scripts/azure/project_prepare.py
infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py
.../scripts/POST_DEPLOYMENT.md | 2 +
.../src/general/conf/dlab.ini | 2 +-
.../src/general/files/azure/jupyterlab_Dockerfile | 51 +++
.../files/azure/jupyterlab_description.json | 25 ++
.../general/files/gcp/jupyterlab_description.json | 2 +-
.../src/general/lib/aws/actions_lib.py | 19 +-
.../src/general/lib/gcp/actions_lib.py | 5 +-
.../src/general/lib/os/debian/common_lib.py | 36 +-
.../src/general/lib/os/debian/ssn_lib.py | 4 +
.../src/general/lib/os/redhat/common_lib.py | 34 +-
.../src/general/lib/os/redhat/edge_lib.py | 65 +++-
.../src/general/lib/os/redhat/notebook_lib.py | 2 +-
.../src/general/lib/os/redhat/ssn_lib.py | 13 +-
.../general/scripts/aws/common_create_bucket.py | 10 +-
.../src/general/scripts/aws/project_prepare.py | 23 +-
.../general/scripts/azure/jupyterlab_configure.py | 398 +++++++++++++++++++++
.../src/general/scripts/azure/project_prepare.py | 10 +-
.../src/general/scripts/azure/ssn_configure.py | 1 +
.../general/scripts/gcp/common_create_bucket.py | 3 +-
.../src/general/scripts/gcp/project_prepare.py | 18 +-
.../src/general/scripts/gcp/project_terminate.py | 7 +-
.../jenkins_jobs/create_notebook_image/config.xml | 1 +
.../jenkins_jobs/create_notebook_server/config.xml | 1 +
.../jenkins_jobs/manage_git_credentials/config.xml | 1 +
.../jenkins_jobs/start_notebook_server/config.xml | 1 +
.../jenkins_jobs/stop_notebook_server/config.xml | 1 +
.../terminate_notebook_image/config.xml | 1 +
.../terminate_notebook_server/config.xml | 1 +
.../src/jupyterlab/Dockerfile_jupyterlab | 8 +-
.../scripts/configure_jupyterlab_node.py | 7 -
.../reporting-grid/reporting-grid.component.html | 2 +-
.../src/app/reporting/reporting.component.ts | 30 +-
.../app/reporting/toolbar/toolbar.component.html | 1 -
.../app/reporting/toolbar/toolbar.component.scss | 1 +
.../src/app/reporting/toolbar/toolbar.component.ts | 14 +-
.../computational-resources-list.component.scss | 19 +
.../install-libraries.component.ts | 2 +
.../src/app/shared/navbar/navbar.component.ts | 4 +-
38 files changed, 709 insertions(+), 116 deletions(-)
diff --cc infrastructure-provisioning/src/general/scripts/aws/project_prepare.py
index 5b18969,009b81e..5b9a3f4
--- a/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py
@@@ -42,124 -38,111 +42,126 @@@ if __name__ == "__main__"
level=logging.DEBUG,
filename=local_log_filepath)
- create_aws_config_files()
- print('Generating infrastructure names and tags')
- project_conf = dict()
- project_conf['service_base_name'] = os.environ['conf_service_base_name'] = replace_multi_symbols(
- os.environ['conf_service_base_name'].lower()[:12], '-', True)
- project_conf['endpoint_name'] = os.environ['endpoint_name']
- project_conf['endpoint_tag'] = os.environ['endpoint_name']
- project_conf['project_name'] = os.environ['project_name']
- project_conf['project_tag'] = os.environ['project_name']
- project_conf['key_name'] = os.environ['conf_key_name']
- project_conf['public_subnet_id'] = os.environ['aws_subnet_id']
- project_conf['vpc_id'] = os.environ['aws_vpc_id']
- project_conf['region'] = os.environ['aws_region']
- project_conf['ami_id'] = get_ami_id(os.environ['aws_{}_image_name'.format(os.environ['conf_os_family'])])
- project_conf['instance_size'] = os.environ['aws_edge_instance_size']
- project_conf['sg_ids'] = os.environ['aws_security_groups_ids']
- project_conf['edge_instance_name'] = '{}-{}-{}-edge'.format(project_conf['service_base_name'],
- os.environ['project_name'], os.environ['endpoint_name'])
- project_conf['tag_name'] = '{}-Tag'.format(project_conf['service_base_name'])
- project_conf['bucket_name_tag'] = '{0}-{1}-{2}-bucket'.format(project_conf['service_base_name'],
- project_conf['project_name'],
- project_conf['endpoint_name'])
- project_conf['bucket_name'] = project_conf['bucket_name_tag'].lower().replace('_', '-')
- project_conf['shared_bucket_name_tag'] = '{0}-{1}-shared-bucket'.format(
- project_conf['service_base_name'], project_conf['endpoint_tag'])
- project_conf['shared_bucket_name'] = project_conf['shared_bucket_name_tag'].lower().replace('_', '-')
- project_conf['edge_role_name'] = '{}-{}-edge-Role'.format(
- project_conf['service_base_name'].lower().replace('-', '_'), os.environ['project_name'])
- project_conf['edge_role_profile_name'] = '{}-{}-edge-Profile'.format(
- project_conf['service_base_name'].lower().replace('-', '_'), os.environ['project_name'])
- project_conf['edge_policy_name'] = '{}-{}-edge-Policy'.format(
- project_conf['service_base_name'].lower().replace('-', '_'), os.environ['project_name'])
- project_conf['edge_security_group_name'] = '{}-sg'.format(project_conf['edge_instance_name'])
- project_conf['notebook_instance_name'] = '{}-{}-nb'.format(project_conf['service_base_name'],
- os.environ['project_name'])
- project_conf['dataengine_instances_name'] = '{}-{}-dataengine' \
- .format(project_conf['service_base_name'], os.environ['project_name'])
- project_conf['notebook_dataengine_role_name'] = '{}-{}-{}-nb-de-Role' \
- .format(project_conf['service_base_name'].lower().replace('-', '_'), os.environ['project_name'],os.environ['endpoint_name'])
- project_conf['notebook_dataengine_policy_name'] = '{}-{}-{}-nb-de-Policy' \
- .format(project_conf['service_base_name'].lower().replace('-', '_'), os.environ['project_name'],os.environ['endpoint_name'])
- project_conf['notebook_dataengine_role_profile_name'] = '{}-{}-{}-nb-de-Profile' \
- .format(project_conf['service_base_name'].lower().replace('-', '_'), os.environ['project_name'],os.environ['endpoint_name'])
- project_conf['notebook_security_group_name'] = '{}-{}-{}-nb-sg'.format(project_conf['service_base_name'],
- os.environ['project_name'],os.environ['endpoint_name'])
- project_conf['private_subnet_prefix'] = os.environ['conf_private_subnet_prefix']
- project_conf['private_subnet_name'] = '{0}-{1}-subnet'.format(project_conf['service_base_name'],
- os.environ['project_name'])
- project_conf['dataengine_master_security_group_name'] = '{}-{}-dataengine-master-sg' \
- .format(project_conf['service_base_name'], os.environ['project_name'])
- project_conf['dataengine_slave_security_group_name'] = '{}-{}-dataengine-slave-sg' \
- .format(project_conf['service_base_name'], os.environ['project_name'])
- project_conf['allowed_ip_cidr'] = list()
- for cidr in os.environ['conf_allowed_ip_cidr'].split(','):
- project_conf['allowed_ip_cidr'].append({"CidrIp": cidr.replace(' ','')})
- project_conf['network_type'] = os.environ['conf_network_type']
- project_conf['all_ip_cidr'] = '0.0.0.0/0'
- project_conf['zone'] = os.environ['aws_region'] + os.environ['aws_zone']
- project_conf['elastic_ip_name'] = '{0}-{1}-edge-EIP'.format(project_conf['service_base_name'],
- os.environ['project_name'])
- project_conf['provision_instance_ip'] = None
- project_conf['local_endpoint'] = False
try:
- project_conf['provision_instance_ip'] = get_instance_ip_address(
- project_conf['tag_name'], '{0}-{1}-endpoint'.format(project_conf['service_base_name'],
- os.environ['endpoint_name'])).get('Private') + "/32"
- except:
- project_conf['provision_instance_ip'] = get_instance_ip_address(project_conf['tag_name'], '{0}-ssn'.format(
- project_conf['service_base_name'])).get('Private') + "/32"
- project_conf['local_endpoint'] = True
- if 'aws_user_predefined_s3_policies' not in os.environ:
- os.environ['aws_user_predefined_s3_policies'] = 'None'
+ dlab.actions_lib.create_aws_config_files()
+ print('Generating infrastructure names and tags')
+ project_conf = dict()
+ project_conf['service_base_name'] = os.environ['conf_service_base_name'] = dlab.fab.replace_multi_symbols(
+ os.environ['conf_service_base_name'].lower()[:20], '-', True)
+ project_conf['endpoint_name'] = os.environ['endpoint_name'].lower()
+ project_conf['endpoint_tag'] = project_conf['endpoint_name']
+ project_conf['project_name'] = os.environ['project_name'].lower()
+ project_conf['project_tag'] = project_conf['project_name']
+ project_conf['key_name'] = os.environ['conf_key_name']
+ project_conf['public_subnet_id'] = os.environ['aws_subnet_id']
+ project_conf['vpc_id'] = os.environ['aws_vpc_id']
+ project_conf['region'] = os.environ['aws_region']
+ project_conf['ami_id'] = dlab.meta_lib.get_ami_id(os.environ['aws_{}_image_name'.format(
+ os.environ['conf_os_family'])])
+ project_conf['instance_size'] = os.environ['aws_edge_instance_size']
+ project_conf['sg_ids'] = os.environ['aws_security_groups_ids']
+ project_conf['edge_instance_name'] = '{}-{}-{}-edge'.format(project_conf['service_base_name'],
+ project_conf['project_name'],
+ project_conf['endpoint_name'])
+ project_conf['tag_name'] = '{}-tag'.format(project_conf['service_base_name'])
+ project_conf['bucket_name_tag'] = '{0}-{1}-{2}-bucket'.format(project_conf['service_base_name'],
+ project_conf['project_name'],
+ project_conf['endpoint_name'])
+ project_conf['bucket_name'] = project_conf['bucket_name_tag'].lower().replace('_', '-')
+ project_conf['shared_bucket_name_tag'] = '{0}-{1}-shared-bucket'.format(
+ project_conf['service_base_name'], project_conf['endpoint_tag'])
+ project_conf['shared_bucket_name'] = project_conf['shared_bucket_name_tag'].lower().replace('_', '-')
+ project_conf['edge_role_name'] = '{}-{}-{}-edge-role'.format(project_conf['service_base_name'],
+ project_conf['project_name'],
+ project_conf['endpoint_name'])
+ project_conf['edge_role_profile_name'] = '{}-{}-{}-edge-profile'.format(
+ project_conf['service_base_name'], project_conf['project_name'], project_conf['endpoint_name'])
+ project_conf['edge_policy_name'] = '{}-{}-{}-edge-policy'.format(
+ project_conf['service_base_name'], project_conf['project_name'], project_conf['endpoint_name'])
+ project_conf['edge_security_group_name'] = '{}-{}-{}-edge-sg'.format(project_conf['edge_instance_name'],
+ project_conf['project_name'],
+ project_conf['endpoint_name'])
+ project_conf['notebook_instance_name'] = '{}-{}-{}-nb'.format(project_conf['service_base_name'],
+ project_conf['project_name'],
+ project_conf['endpoint_name'])
+ project_conf['dataengine_instances_name'] = '{}-{}-{}-de'.format(project_conf['service_base_name'],
+ project_conf['project_name'],
+ project_conf['endpoint_name'])
+ project_conf['notebook_dataengine_role_name'] = '{}-{}-{}-nb-de-role'.format(project_conf['service_base_name'],
+ project_conf['project_name'],
+ project_conf['endpoint_name'])
+ project_conf['notebook_dataengine_policy_name'] = '{}-{}-{}-nb-de-policy'.format(
+ project_conf['service_base_name'], project_conf['project_name'], project_conf['endpoint_name'])
+ project_conf['notebook_dataengine_role_profile_name'] = '{}-{}-{}-nb-de-profile'.format(
+ project_conf['service_base_name'], project_conf['project_name'], project_conf['endpoint_name'])
+ project_conf['notebook_security_group_name'] = '{}-{}-{}-nb-sg'.format(project_conf['service_base_name'],
+ project_conf['project_name'],
+ project_conf['endpoint_name'])
+ project_conf['private_subnet_prefix'] = os.environ['conf_private_subnet_prefix']
+ project_conf['private_subnet_name'] = '{0}-{1}-{2}-subnet'.format(project_conf['service_base_name'],
+ project_conf['project_name'],
+ project_conf['endpoint_name'])
+ project_conf['dataengine_master_security_group_name'] = '{}-{}-{}-de-master-sg'.format(
+ project_conf['service_base_name'], project_conf['project_name'], project_conf['endpoint_name'])
+ project_conf['dataengine_slave_security_group_name'] = '{}-{}-{}-de-slave-sg'.format(
+ project_conf['service_base_name'], project_conf['project_name'], project_conf['endpoint_name'])
+ project_conf['allowed_ip_cidr'] = list()
+ for cidr in os.environ['conf_allowed_ip_cidr'].split(','):
+ project_conf['allowed_ip_cidr'].append({"CidrIp": cidr.replace(' ', '')})
+ project_conf['network_type'] = os.environ['conf_network_type']
+ project_conf['all_ip_cidr'] = '0.0.0.0/0'
+ project_conf['zone'] = os.environ['aws_region'] + os.environ['aws_zone']
+ project_conf['elastic_ip_name'] = '{0}-{1}-{2}-edge-static-ip'.format(project_conf['service_base_name'],
+ project_conf['project_name'],
+ project_conf['endpoint_name'])
+ project_conf['provision_instance_ip'] = None
+ project_conf['local_endpoint'] = False
+ try:
+ project_conf['provision_instance_ip'] = '{}/32'.format(dlab.meta_lib.get_instance_ip_address(
+ project_conf['tag_name'], '{0}-{1}-endpoint'.format(project_conf['service_base_name'],
+ project_conf['endpoint_name'])).get('Private'))
+ except:
+ project_conf['provision_instance_ip'] = '{}/32'.format(dlab.meta_lib.get_instance_ip_address(
+ project_conf['tag_name'], '{0}-ssn'.format(project_conf['service_base_name'])).get('Private'))
+ project_conf['local_endpoint'] = True
+ if 'aws_user_predefined_s3_policies' not in os.environ:
+ os.environ['aws_user_predefined_s3_policies'] = 'None'
- try:
- if os.environ['conf_user_subnets_range'] == '':
- raise KeyError
- except KeyError:
- os.environ['conf_user_subnets_range'] = ''
+ try:
+ if os.environ['conf_user_subnets_range'] == '':
+ raise KeyError
+ except KeyError:
+ os.environ['conf_user_subnets_range'] = ''
- # FUSE in case of absence of user's key
- try:
- project_conf['user_key'] = os.environ['key']
+ # FUSE in case of absence of user's key
try:
- local('echo "{0}" >> {1}{2}.pub'.format(project_conf['user_key'], os.environ['conf_key_dir'],
- project_conf['project_name']))
- except:
- print("ADMINSs PUBLIC KEY DOES NOT INSTALLED")
- except KeyError:
- print("ADMINSs PUBLIC KEY DOES NOT UPLOADED")
- sys.exit(1)
+ project_conf['user_key'] = os.environ['key']
+ try:
+ local('echo "{0}" >> {1}{2}.pub'.format(project_conf['user_key'], os.environ['conf_key_dir'],
+ project_conf['project_name']))
+ except:
+ print("ADMINSs PUBLIC KEY DOES NOT INSTALLED")
+ except KeyError:
+ print("ADMINSs PUBLIC KEY DOES NOT UPLOADED")
+ sys.exit(1)
- print("Will create exploratory environment with edge node as access point as following: {}".
- format(json.dumps(project_conf, sort_keys=True, indent=4, separators=(',', ': '))))
- logging.info(json.dumps(project_conf))
+ print("Will create exploratory environment with edge node as access point as following: {}".
+ format(json.dumps(project_conf, sort_keys=True, indent=4, separators=(',', ': '))))
+ logging.info(json.dumps(project_conf))
- if 'conf_additional_tags' in os.environ:
- project_conf['bucket_additional_tags'] = ';' + os.environ['conf_additional_tags']
- os.environ['conf_additional_tags'] = os.environ['conf_additional_tags'] + \
- ';project_tag:{0};endpoint_tag:{1};'.format(
- project_conf['project_tag'], project_conf['endpoint_tag'])
- else:
- project_conf['bucket_additional_tags'] = ''
- os.environ['conf_additional_tags'] = 'project_tag:{0};endpoint_tag:{1}'.format(project_conf['project_tag'],
- project_conf['endpoint_tag'])
- print('Additional tags will be added: {}'.format(os.environ['conf_additional_tags']))
+ if 'conf_additional_tags' in os.environ:
++ project_conf['bucket_additional_tags'] = ';' + os.environ['conf_additional_tags']
+ os.environ['conf_additional_tags'] = os.environ['conf_additional_tags'] + \
+ ';project_tag:{0};endpoint_tag:{1};'.format(
+ project_conf['project_tag'], project_conf['endpoint_tag'])
+ else:
++ project_conf['bucket_additional_tags'] = ''
+ os.environ['conf_additional_tags'] = 'project_tag:{0};endpoint_tag:{1}'.format(project_conf['project_tag'],
+ project_conf['endpoint_tag'])
+ print('Additional tags will be added: {}'.format(os.environ['conf_additional_tags']))
+ except Exception as err:
+ dlab.fab.append_result("Failed to generate variables dictionary.", str(err))
+ sys.exit(1)
if not project_conf['local_endpoint']:
# attach project_tag and endpoint_tag to endpoint
@@@ -549,19 -533,22 +551,24 @@@
try:
logging.info('[CREATE BUCKETS]')
print('[CREATE BUCKETS]')
- params = "--bucket_name {} --infra_tag_name {} --infra_tag_value {} --region {} " \
- "--bucket_name_tag {}".format(project_conf['shared_bucket_name'], project_conf['tag_name'],
- project_conf['shared_bucket_name'], project_conf['region'],
- project_conf['shared_bucket_name_tag'])
- project_conf['shared_bucket_tags'] = 'endpoint_tag:{0};{1}:{2};{3}:{4}{5}'.format(project_conf['endpoint_tag'],
- os.environ['conf_billing_tag_key'], os.environ['conf_billing_tag_value'],
- project_conf['tag_name'], project_conf['shared_bucket_name'], project_conf['bucket_additional_tags']).replace(';', ',')
++ project_conf['shared_bucket_tags'] = 'endpoint_tag:{0};{1}:{2};{3}:{4}{5}'.format(
++ project_conf['endpoint_tag'], os.environ['conf_billing_tag_key'], os.environ['conf_billing_tag_value'],
++ project_conf['tag_name'], project_conf['shared_bucket_name'],
++ project_conf['bucket_additional_tags']).replace(';', ',')
+ params = "--bucket_name {} --bucket_tags {} --region {} --bucket_name_tag {}". \
- format(project_conf['shared_bucket_name'], project_conf['shared_bucket_tags'], project_conf['region'], project_conf['shared_bucket_name_tag'])
++ format(project_conf['shared_bucket_name'], project_conf['shared_bucket_tags'], project_conf['region'],
++ project_conf['shared_bucket_name_tag'])
try:
local("~/scripts/{}.py {}".format('common_create_bucket', params))
except:
traceback.print_exc()
raise Exception
-
- params = "--bucket_name {} --infra_tag_name {} --infra_tag_value {} --region {} --bucket_name_tag {}" \
- .format(project_conf['bucket_name'], project_conf['tag_name'], project_conf['bucket_name'],
- project_conf['region'], project_conf['bucket_name_tag'])
- project_conf['bucket_tags'] = 'endpoint_tag:{0};{1}:{2};project_tag:{3};{4}:{5}{6}'.format(project_conf['endpoint_tag'],
- os.environ['conf_billing_tag_key'], os.environ['conf_billing_tag_value'],
- project_conf['project_tag'],
- project_conf['tag_name'], project_conf['bucket_name'], project_conf['bucket_additional_tags']).replace(';', ',')
++ project_conf['bucket_tags'] = 'endpoint_tag:{0};{1}:{2};project_tag:{3};{4}:{5}{6}'.format(
++ project_conf['endpoint_tag'], os.environ['conf_billing_tag_key'], os.environ['conf_billing_tag_value'],
++ project_conf['project_tag'], project_conf['tag_name'], project_conf['bucket_name'],
++ project_conf['bucket_additional_tags']).replace(';', ',')
+ params = "--bucket_name {} --bucket_tags {} --region {} --bucket_name_tag {}" \
+ .format(project_conf['bucket_name'], project_conf['bucket_tags'], project_conf['region'], project_conf['bucket_name_tag'])
try:
local("~/scripts/{}.py {}".format('common_create_bucket', params))
except:
diff --cc infrastructure-provisioning/src/general/scripts/azure/project_prepare.py
index 42a4fa3,f5c98ab..010e3df
--- a/infrastructure-provisioning/src/general/scripts/azure/project_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/project_prepare.py
@@@ -55,12 -49,10 +55,12 @@@ if __name__ == "__main__"
project_conf['azure_ad_user_name'] = os.environ['azure_iam_user']
project_conf['key_name'] = os.environ['conf_key_name']
-
+ project_conf['tag_name'] = project_conf['service_base_name'] + '-tag'
project_conf['vpc_name'] = os.environ['azure_vpc_name']
project_conf['subnet_name'] = os.environ['azure_subnet_name']
- project_conf['private_subnet_name'] = project_conf['service_base_name'] + '-' + project_conf['project_name'] + '-subnet'
+ project_conf['private_subnet_name'] = '{}-{}-{}-subnet'.format(project_conf['service_base_name'],
+ project_conf['project_name'],
+ project_conf['endpoint_name'])
if os.environ['conf_network_type'] == 'private':
project_conf['static_public_ip_name'] = 'None'
else:
@@@ -112,17 -95,17 +112,21 @@@
"SBN": project_conf['service_base_name'],
"project_tag": project_conf['project_tag'],
"endpoint_tag": project_conf['endpoint_tag'],
- os.environ['conf_billing_tag_key']: os.environ['conf_billing_tag_value'],
+ os.environ['conf_billing_tag_key']:
- os.environ['conf_billing_tag_value']}
++ os.environ['conf_billing_tag_value'],
+ project_conf['tag_name']: project_conf['edge_storage_account_name']}
project_conf['primary_disk_size'] = '32'
- project_conf['shared_storage_account_name'] = '{0}-{1}-shared-storage'.format(project_conf['service_base_name'],
- project_conf['endpoint_name'])
- project_conf['shared_container_name'] = '{}-{}-shared-container'.format(project_conf['service_base_name'], project_conf['endpoint_name']).lower()
+ project_conf['shared_storage_account_name'] = '{0}-{1}-shared-bucket'.format(project_conf['service_base_name'],
+ project_conf['endpoint_name'])
+ project_conf['shared_container_name'] = '{}-{}-shared-bucket'.format(project_conf['service_base_name'],
+ project_conf['endpoint_name'])
project_conf['shared_storage_account_tags'] = {"Name": project_conf['shared_storage_account_name'],
- "SBN": project_conf['service_base_name'],
- os.environ['conf_billing_tag_key']: os.environ[
- 'conf_billing_tag_value'], "endpoint_tag": project_conf['endpoint_tag'],
- project_conf['tag_name']: project_conf['shared_storage_account_name']}
+ "SBN": project_conf['service_base_name'],
+ os.environ['conf_billing_tag_key']: os.environ[
- 'conf_billing_tag_value']}
++ 'conf_billing_tag_value'], "endpoint_tag":
++ project_conf['endpoint_tag'],
++ project_conf['tag_name']:
++ project_conf['shared_storage_account_name']}
# FUSE in case of absence of user's key
try:
diff --cc infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py
index 1384e24,82edc16..7858d3f
--- a/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py
@@@ -39,108 -35,92 +39,109 @@@ if __name__ == "__main__"
logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
level=logging.DEBUG,
filename=local_log_filepath)
-
- print('Generating infrastructure names and tags')
- project_conf = dict()
- project_conf['edge_unique_index'] = str(uuid.uuid4())[:5]
- project_conf['ps_unique_index'] = str(uuid.uuid4())[:5]
- project_conf['service_base_name'] = (os.environ['conf_service_base_name']).lower().replace('_', '-')
- project_conf['key_name'] = os.environ['conf_key_name']
- project_conf['user_keyname'] = os.environ['project_name']
- project_conf['project_name'] = (os.environ['project_name']).lower().replace('_', '-')
- project_conf['project_tag'] = (os.environ['project_name']).lower().replace('_', '-')
- project_conf['endpoint_tag'] = (os.environ['endpoint_name']).lower().replace('_', '-')
- project_conf['endpoint_name'] = (os.environ['endpoint_name']).lower().replace('_', '-')
try:
- if os.environ['gcp_vpc_name'] == '':
- raise KeyError
- else:
- project_conf['vpc_name'] = os.environ['gcp_vpc_name']
- except KeyError:
- project_conf['vpc_name'] = project_conf['service_base_name'] + '-ssn-vpc'
- project_conf['vpc_cidr'] = os.environ['conf_vpc_cidr']
- project_conf['private_subnet_name'] = '{0}-{1}-subnet'.format(project_conf['service_base_name'],
- project_conf['project_name'])
- project_conf['subnet_name'] = os.environ['gcp_subnet_name']
- project_conf['region'] = os.environ['gcp_region']
- project_conf['zone'] = os.environ['gcp_zone']
- project_conf['vpc_selflink'] = GCPMeta().get_vpc(project_conf['vpc_name'])['selfLink']
- project_conf['private_subnet_prefix'] = os.environ['conf_private_subnet_prefix']
- project_conf['edge_service_account_name'] = '{}-{}-edge'.format(project_conf['service_base_name'],
- project_conf['project_name'])
- project_conf['edge_role_name'] = '{}-{}-{}-edge'.format(project_conf['service_base_name'],
- project_conf['project_name'], project_conf['edge_unique_index'])
- project_conf['ps_service_account_name'] = '{}-{}-ps'.format(project_conf['service_base_name'],
- project_conf['project_name'])
- project_conf['ps_role_name'] = '{}-{}-{}-ps'.format(project_conf['service_base_name'],
- project_conf['project_name'], project_conf['ps_unique_index'])
- project_conf['ps_policy_path'] = '/root/files/ps_policy.json'
- project_conf['ps_roles_path'] = '/root/files/ps_roles.json'
- project_conf['instance_name'] = '{0}-{1}-{2}-edge'.format(project_conf['service_base_name'],
- project_conf['project_name'], project_conf['endpoint_tag'])
- project_conf['ssn_instance_name'] = '{}-ssn'.format(project_conf['service_base_name'])
- project_conf['bucket_name'] = '{0}-{1}-{2}-bucket'.format(project_conf['service_base_name'],
+ print('Generating infrastructure names and tags')
+ project_conf = dict()
+ project_conf['edge_unique_index'] = str(uuid.uuid4())[:5]
+ project_conf['ps_unique_index'] = str(uuid.uuid4())[:5]
+ project_conf['service_base_name'] = (os.environ['conf_service_base_name']).lower()
+ project_conf['key_name'] = os.environ['conf_key_name']
+ project_conf['project_name'] = (os.environ['project_name']).lower()
+ project_conf['user_keyname'] = project_conf['project_name']
+ project_conf['project_tag'] = (project_conf['project_name'])
+ project_conf['endpoint_name'] = (os.environ['endpoint_name']).lower()
+ project_conf['endpoint_tag'] = project_conf['endpoint_name']
+ try:
+ if os.environ['gcp_vpc_name'] == '':
+ raise KeyError
+ else:
+ project_conf['vpc_name'] = os.environ['gcp_vpc_name']
+ except KeyError:
+ project_conf['vpc_name'] = project_conf['service_base_name'] + '-vpc'
+ project_conf['vpc_cidr'] = os.environ['conf_vpc_cidr']
+ project_conf['private_subnet_name'] = '{0}-{1}-{2}-subnet'.format(project_conf['service_base_name'],
+ project_conf['project_name'],
+ project_conf['endpoint_name'])
+ project_conf['subnet_name'] = os.environ['gcp_subnet_name']
+ project_conf['region'] = os.environ['gcp_region']
+ project_conf['zone'] = os.environ['gcp_zone']
+ project_conf['vpc_selflink'] = GCPMeta().get_vpc(project_conf['vpc_name'])['selfLink']
+ project_conf['private_subnet_prefix'] = os.environ['conf_private_subnet_prefix']
+ project_conf['edge_service_account_name'] = '{}-{}-{}-edge-sa'.format(project_conf['service_base_name'],
+ project_conf['project_name'],
+ project_conf['endpoint_name'])
+ project_conf['edge_role_name'] = '{}-{}-{}-{}-edge-role'.format(project_conf['service_base_name'],
+ project_conf['project_name'],
+ project_conf['endpoint_name'],
+ project_conf['edge_unique_index'])
+ project_conf['ps_service_account_name'] = '{}-{}-{}-ps-sa'.format(project_conf['service_base_name'],
+ project_conf['project_name'],
+ project_conf['endpoint_name'])
+ project_conf['ps_role_name'] = '{}-{}-{}-{}-ps-role'.format(project_conf['service_base_name'],
+ project_conf['project_name'],
+ project_conf['endpoint_name'],
+ project_conf['ps_unique_index'])
+ project_conf['ps_policy_path'] = '/root/files/ps_policy.json'
+ project_conf['ps_roles_path'] = '/root/files/ps_roles.json'
+ project_conf['instance_name'] = '{0}-{1}-{2}-edge'.format(project_conf['service_base_name'],
+ project_conf['project_name'],
+ project_conf['endpoint_name'])
+ project_conf['ssn_instance_name'] = '{}-ssn'.format(project_conf['service_base_name'])
+ project_conf['bucket_name'] = '{0}-{1}-{2}-bucket'.format(project_conf['service_base_name'],
+ project_conf['project_name'],
+ project_conf['endpoint_name'])
+ project_conf['shared_bucket_name'] = '{0}-{1}-shared-bucket'.format(project_conf['service_base_name'],
+ project_conf['endpoint_name'])
+ project_conf['instance_size'] = os.environ['gcp_edge_instance_size']
+ project_conf['ssh_key_path'] = '{0}{1}.pem'.format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
+ project_conf['image_name'] = os.environ['gcp_{}_image_name'.format(os.environ['conf_os_family'])]
+ project_conf['static_address_name'] = '{0}-{1}-{2}-static-ip'.format(project_conf['service_base_name'],
+ project_conf['project_name'],
+ project_conf['endpoint_name'])
+ project_conf['fw_edge_ingress_public'] = '{}-sg-ingress-public'.format(project_conf['instance_name'])
+ project_conf['fw_edge_ingress_internal'] = '{}-sg-ingress-internal'.format(project_conf['instance_name'])
+ project_conf['fw_edge_egress_public'] = '{}-sg-egress-public'.format(project_conf['instance_name'])
+ project_conf['fw_edge_egress_internal'] = '{}-sg-egress-internal'.format(project_conf['instance_name'])
+ project_conf['ps_firewall_target'] = '{0}-{1}-{2}-ps'.format(project_conf['service_base_name'],
+ project_conf['project_name'],
+ project_conf['endpoint_name'])
+ project_conf['fw_common_name'] = '{}-{}-{}-ps'.format(project_conf['service_base_name'],
project_conf['project_name'],
project_conf['endpoint_name'])
- project_conf['shared_bucket_name'] = '{0}-{1}-shared-bucket'.format(project_conf['service_base_name'],
- project_conf['endpoint_name'])
- project_conf['instance_size'] = os.environ['gcp_edge_instance_size']
- project_conf['ssh_key_path'] = '{0}{1}.pem'.format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
- project_conf['image_name'] = os.environ['gcp_{}_image_name'.format(os.environ['conf_os_family'])]
- project_conf['static_address_name'] = '{0}-{1}-ip'.format(project_conf['service_base_name'], project_conf['project_name'])
- project_conf['fw_edge_ingress_public'] = '{}-ingress-public'.format(project_conf['instance_name'])
- project_conf['fw_edge_ingress_internal'] = '{}-ingress-internal'.format(project_conf['instance_name'])
- project_conf['fw_edge_egress_public'] = '{}-egress-public'.format(project_conf['instance_name'])
- project_conf['fw_edge_egress_internal'] = '{}-egress-internal'.format(project_conf['instance_name'])
- project_conf['ps_firewall_target'] = '{0}-{1}-ps'.format(project_conf['service_base_name'],
- project_conf['project_name'])
- project_conf['fw_common_name'] = '{}-{}-ps'.format(project_conf['service_base_name'], project_conf['project_name'])
- project_conf['fw_ps_ingress'] = '{}-ingress'.format(project_conf['fw_common_name'])
- project_conf['fw_ps_egress_private'] = '{}-egress-private'.format(project_conf['fw_common_name'])
- project_conf['fw_ps_egress_public'] = '{}-egress-public'.format(project_conf['fw_common_name'])
- project_conf['network_tag'] = project_conf['instance_name']
- project_conf['instance_labels'] = {"name": project_conf['instance_name'],
- "sbn": project_conf['service_base_name'],
- "project_tag": project_conf['project_tag'],
- "endpoint_tag": project_conf['endpoint_tag'],
- "product": "dlab"}
- project_conf['tag_name'] = project_conf['service_base_name'] + '-tag'
- project_conf['allowed_ip_cidr'] = os.environ['conf_allowed_ip_cidr']
- if 'conf_user_subnets_range' in os.environ:
- project_conf['user_subnets_range'] = os.environ['conf_user_subnets_range']
- else:
- project_conf['user_subnets_range'] = ''
+ project_conf['fw_ps_ingress'] = '{}-sg-ingress'.format(project_conf['fw_common_name'])
+ project_conf['fw_ps_egress_private'] = '{}-sg-egress-private'.format(project_conf['fw_common_name'])
+ project_conf['fw_ps_egress_public'] = '{}-sg-egress-public'.format(project_conf['fw_common_name'])
+ project_conf['network_tag'] = project_conf['instance_name']
+ project_conf['instance_labels'] = {"name": project_conf['instance_name'],
+ "sbn": project_conf['service_base_name'],
+ "project_tag": project_conf['project_tag'],
+ "endpoint_tag": project_conf['endpoint_tag'],
+ "product": "dlab"}
++ project_conf['tag_name'] = project_conf['service_base_name'] + '-tag'
+ project_conf['allowed_ip_cidr'] = os.environ['conf_allowed_ip_cidr']
+ if 'conf_user_subnets_range' in os.environ:
+ project_conf['user_subnets_range'] = os.environ['conf_user_subnets_range']
+ else:
+ project_conf['user_subnets_range'] = ''
- # FUSE in case of absence of user's key
- try:
- project_conf['user_key'] = os.environ['key']
+ # FUSE in case of absence of user's key
try:
- local('echo "{0}" >> {1}{2}.pub'.format(project_conf['user_key'], os.environ['conf_key_dir'], os.environ['project_name']))
- except:
- print("ADMINSs PUBLIC KEY DOES NOT INSTALLED")
- except KeyError:
- print("ADMINSs PUBLIC KEY DOES NOT UPLOADED")
- sys.exit(1)
+ project_conf['user_key'] = os.environ['key']
+ try:
+ local('echo "{0}" >> {1}{2}.pub'.format(project_conf['user_key'], os.environ['conf_key_dir'],
+ project_conf['project_name']))
+ except:
+ print("ADMINSs PUBLIC KEY DOES NOT INSTALLED")
+ except KeyError:
+ print("ADMINSs PUBLIC KEY DOES NOT UPLOADED")
+ sys.exit(1)
- print("Will create exploratory environment with edge node as access point as following: ".format(json.dumps(
- project_conf, sort_keys=True, indent=4, separators=(',', ': '))))
- logging.info(json.dumps(project_conf))
+ print("Will create exploratory environment with edge node as access point as following: ".format(json.dumps(
+ project_conf, sort_keys=True, indent=4, separators=(',', ': '))))
+ logging.info(json.dumps(project_conf))
+ except Exception as err:
+ dlab.fab.append_result("Failed to generate infrastructure names", str(err))
+ sys.exit(1)
try:
logging.info('[CREATE SUBNET]')
@@@ -389,14 -367,25 +390,27 @@@
try:
logging.info('[CREATE BUCKETS]')
print('[CREATE BUCKETS]')
- params = "--bucket_name {}".format(project_conf['shared_bucket_name'])
+ project_conf['shared_bucket_tags'] = {
+ project_conf['tag_name']: project_conf['shared_bucket_name'],
+ "endpoint_tag": project_conf['endpoint_tag'],
+ os.environ['conf_billing_tag_key']: os.environ['conf_billing_tag_value'],
+ "sbn": project_conf['service_base_name']}
- params = "--bucket_name {} --tags '{}'".format(project_conf['shared_bucket_name'], json.dumps(project_conf['shared_bucket_tags']))
++ params = "--bucket_name {} --tags '{}'".format(project_conf['shared_bucket_name'],
++ json.dumps(project_conf['shared_bucket_tags']))
try:
local("~/scripts/{}.py {}".format('common_create_bucket', params))
except:
traceback.print_exc()
raise Exception
- params = "--bucket_name {}".format(project_conf['bucket_name'])
+ project_conf['bucket_tags'] = {
+ project_conf['tag_name']: project_conf['bucket_name'],
+ "endpoint_tag": project_conf['endpoint_tag'],
+ os.environ['conf_billing_tag_key']: os.environ['conf_billing_tag_value'],
+ "sbn": project_conf['service_base_name'],
+ "project_tag": project_conf['project_tag']}
- params = "--bucket_name {} --tags '{}'".format(project_conf['bucket_name'], json.dumps(project_conf['bucket_tags']))
++ params = "--bucket_name {} --tags '{}'".format(project_conf['bucket_name'],
++ json.dumps(project_conf['bucket_tags']))
try:
local("~/scripts/{}.py {}".format('common_create_bucket', params))
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org