You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by om...@apache.org on 2019/09/05 15:10:56 UTC
[incubator-dlab] branch DLAB-terraform updated: fixed issue with SG
groups;
This is an automated email from the ASF dual-hosted git repository.
omartushevskyi pushed a commit to branch DLAB-terraform
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/DLAB-terraform by this push:
new 1b16c7a fixed issue with SG groups;
1b16c7a is described below
commit 1b16c7ac23e9eb340a4ace05734356fdb16f6108
Author: Oleh Martushevskyi <Ol...@epam.com>
AuthorDate: Thu Sep 5 18:10:47 2019 +0300
fixed issue with SG groups;
---
.../src/general/scripts/aws/project_prepare.py | 82 +++++++++++++---------
1 file changed, 50 insertions(+), 32 deletions(-)
diff --git a/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py b/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py
index 9b9f2bd..1ac4aae 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py
@@ -43,7 +43,8 @@ if __name__ == "__main__":
project_conf = dict()
project_conf['service_base_name'] = os.environ['conf_service_base_name'] = replace_multi_symbols(
os.environ['conf_service_base_name'].lower()[:12], '-', True)
- project_conf['endpoint_name'] = '{0}-{1}-endpoint'.format(project_conf['service_base_name'], os.environ['endpoint_name'])
+ project_conf['endpoint_name'] = '{0}-{1}-endpoint'.format(project_conf['service_base_name'],
+ os.environ['endpoint_name'])
project_conf['endpoint_tag'] = os.environ['endpoint_name']
project_conf['project_name'] = os.environ['project_name']
project_conf['project_tag'] = os.environ['project_name']
@@ -54,21 +55,22 @@ if __name__ == "__main__":
project_conf['ami_id'] = get_ami_id(os.environ['aws_{}_image_name'.format(os.environ['conf_os_family'])])
project_conf['instance_size'] = os.environ['aws_edge_instance_size']
project_conf['sg_ids'] = os.environ['aws_security_groups_ids']
- project_conf['edge_instance_name'] = '{}-{}-edge'.format(project_conf['service_base_name'], os.environ['project_name'])
+ project_conf['edge_instance_name'] = '{}-{}-edge'.format(project_conf['service_base_name'],
+ os.environ['project_name'])
project_conf['tag_name'] = '{}-Tag'.format(project_conf['service_base_name'])
project_conf['bucket_name_tag'] = '{}-{}-bucket'.format(project_conf['service_base_name'],
os.environ['project_name'])
project_conf['bucket_name'] = project_conf['bucket_name_tag'].lower().replace('_', '-')
- project_conf['ssn_bucket_name'] = '{}-ssn-bucket'.format(project_conf['service_base_name']).lower().replace('_', '-')
- project_conf['shared_bucket_name'] = '{}-shared-bucket'.format(project_conf['service_base_name']).lower().replace('_',
- '-')
- project_conf['edge_role_name'] = '{}-{}-edge-Role'.format(project_conf['service_base_name'].lower().replace('-', '_'),
- os.environ['project_name'])
- project_conf['edge_role_profile_name'] = '{}-{}-edge-Profile'.format(project_conf['service_base_name'].lower().replace('-',
- '_'),
- os.environ['project_name'])
- project_conf['edge_policy_name'] = '{}-{}-edge-Policy'.format(project_conf['service_base_name'].lower().replace('-', '_'),
- os.environ['project_name'])
+ project_conf['ssn_bucket_name'] = '{}-ssn-bucket'.format(
+ project_conf['service_base_name']).lower().replace('_', '-')
+ project_conf['shared_bucket_name'] = '{}-shared-bucket'.format(
+ project_conf['service_base_name']).lower().replace('_', '-')
+ project_conf['edge_role_name'] = '{}-{}-edge-Role'.format(
+ project_conf['service_base_name'].lower().replace('-', '_'), os.environ['project_name'])
+ project_conf['edge_role_profile_name'] = '{}-{}-edge-Profile'.format(
+ project_conf['service_base_name'].lower().replace('-', '_'), os.environ['project_name'])
+ project_conf['edge_policy_name'] = '{}-{}-edge-Policy'.format(
+ project_conf['service_base_name'].lower().replace('-', '_'), os.environ['project_name'])
project_conf['edge_security_group_name'] = '{}-sg'.format(project_conf['edge_instance_name'])
project_conf['notebook_instance_name'] = '{}-{}-nb'.format(project_conf['service_base_name'],
os.environ['project_name'])
@@ -97,6 +99,14 @@ if __name__ == "__main__":
project_conf['zone'] = os.environ['aws_region'] + os.environ['aws_zone']
project_conf['elastic_ip_name'] = '{0}-{1}-edge-EIP'.format(project_conf['service_base_name'],
os.environ['project_name'])
+ project_conf['provision_instance_ip'] = None
+ try:
+ project_conf['provision_instance_ip'] = get_instance_ip_address(
+ project_conf['tag_name'], '{0}-{1}-endpoint'.format(project_conf['service_base_name'],
+ os.environ['endpoint_name'])).get('Private') + "/32"
+ except:
+ project_conf['provision_instance_ip'] = get_instance_ip_address(project_conf['tag_name'], '{0}-ssn'.format(
+ project_conf['service_base_name'])).get('Private') + "/32"
if 'aws_user_predefined_s3_policies' not in os.environ:
os.environ['aws_user_predefined_s3_policies'] = 'None'
@@ -110,7 +120,8 @@ if __name__ == "__main__":
try:
project_conf['user_key'] = os.environ['key']
try:
- local('echo "{0}" >> {1}{2}.pub'.format(project_conf['user_key'], os.environ['conf_key_dir'], project_conf['project_name']))
+ local('echo "{0}" >> {1}{2}.pub'.format(project_conf['user_key'], os.environ['conf_key_dir'],
+ project_conf['project_name']))
except:
print("ADMINSs PUBLIC KEY DOES NOT INSTALLED")
except KeyError:
@@ -122,9 +133,12 @@ if __name__ == "__main__":
logging.info(json.dumps(project_conf))
if 'conf_additional_tags' in os.environ:
- os.environ['conf_additional_tags'] = os.environ['conf_additional_tags'] + ';project_tag:{0};endpoint_tag:{1};'.format(project_conf['project_tag'], project_conf['endpoint_tag'])
+ os.environ['conf_additional_tags'] = os.environ['conf_additional_tags'] + \
+ ';project_tag:{0};endpoint_tag:{1};'.format(
+ project_conf['project_tag'], project_conf['endpoint_tag'])
else:
- os.environ['conf_additional_tags'] = 'project_tag:{0};endpoint_tag:{1}'.format(project_conf['project_tag'], project_conf['endpoint_tag'])
+ os.environ['conf_additional_tags'] = 'project_tag:{0};endpoint_tag:{1}'.format(project_conf['project_tag'],
+ project_conf['endpoint_tag'])
print('Additional tags will be added: {}'.format(os.environ['conf_additional_tags']))
# attach project_tag and endpoint_tag to endpoint
@@ -195,9 +209,10 @@ if __name__ == "__main__":
print('[CREATE BACKEND (NOTEBOOK) ROLES]')
params = "--role_name {} --role_profile_name {} --policy_name {} --region {} --infra_tag_name {} " \
"--infra_tag_value {}" \
- .format(project_conf['notebook_dataengine_role_name'], project_conf['notebook_dataengine_role_profile_name'],
- project_conf['notebook_dataengine_policy_name'], os.environ['aws_region'], project_conf['tag_name'],
- project_conf['service_base_name'])
+ .format(project_conf['notebook_dataengine_role_name'],
+ project_conf['notebook_dataengine_role_profile_name'],
+ project_conf['notebook_dataengine_policy_name'], os.environ['aws_region'],
+ project_conf['tag_name'], project_conf['service_base_name'])
try:
local("~/scripts/{}.py {}".format('common_create_role_policy', params))
except:
@@ -238,8 +253,7 @@ if __name__ == "__main__":
},
{
"IpProtocol": "-1",
- "IpRanges": [{"CidrIp": get_instance_ip_address(project_conf['tag_name'], '{0}-{1}-endpoint'.format(
- project_conf['service_base_name'], os.environ['endpoint_name'])).get('Private') + "/32"}],
+ "IpRanges": [{"CidrIp": project_conf['provision_instance_ip']}],
"UserIdGroupPairs": [],
"PrefixListIds": []
}
@@ -397,8 +411,7 @@ if __name__ == "__main__":
},
{
"IpProtocol": "-1",
- "IpRanges": [{"CidrIp": get_instance_ip_address(project_conf['tag_name'], '{0}-{1}-endpoint'.format(
- project_conf['service_base_name'], os.environ['endpoint_name'])).get('Private') + "/32"}],
+ "IpRanges": [{"CidrIp": project_conf['provision_instance_ip']}],
"UserIdGroupPairs": [],
"PrefixListIds": []
}
@@ -413,8 +426,7 @@ if __name__ == "__main__":
},
{
"IpProtocol": "-1",
- "IpRanges": [{"CidrIp": get_instance_ip_address(project_conf['tag_name'], '{0}-{1}-endpoint'.format(
- project_conf['service_base_name'], os.environ['endpoint_name'])).get('Private') + "/32"}],
+ "IpRanges": [{"CidrIp": project_conf['provision_instance_ip']}],
"UserIdGroupPairs": [],
"PrefixListIds": [],
},
@@ -437,7 +449,8 @@ if __name__ == "__main__":
params = "--name {} --vpc_id {} --security_group_rules '{}' --egress '{}' --infra_tag_name {} " \
"--infra_tag_value {} --force {}".format(project_conf['notebook_security_group_name'],
project_conf['vpc2_id'], json.dumps(private_sg_ingress),
- json.dumps(private_sg_egress), project_conf['service_base_name'],
+ json.dumps(private_sg_egress),
+ project_conf['service_base_name'],
project_conf['notebook_instance_name'], True)
try:
local("~/scripts/{}.py {}".format('common_create_security_group', params))
@@ -463,7 +476,8 @@ if __name__ == "__main__":
params = "--name {} --vpc_id {} --security_group_rules '{}' --egress '{}' --infra_tag_name {} " \
"--infra_tag_value {} --force {}".format(project_conf['dataengine_master_security_group_name'],
project_conf['vpc2_id'], json.dumps(private_sg_ingress),
- json.dumps(private_sg_egress), project_conf['service_base_name'],
+ json.dumps(private_sg_egress),
+ project_conf['service_base_name'],
project_conf['dataengine_instances_name'], True)
try:
local("~/scripts/{}.py {}".format('common_create_security_group', params))
@@ -485,7 +499,8 @@ if __name__ == "__main__":
params = "--name {} --vpc_id {} --security_group_rules '{}' --egress '{}' --infra_tag_name {} " \
"--infra_tag_value {} --force {}".format(project_conf['dataengine_slave_security_group_name'],
project_conf['vpc2_id'], json.dumps(private_sg_ingress),
- json.dumps(private_sg_egress), project_conf['service_base_name'],
+ json.dumps(private_sg_egress),
+ project_conf['service_base_name'],
project_conf['dataengine_instances_name'], True)
try:
local("~/scripts/{}.py {}".format('common_create_security_group', params))
@@ -528,7 +543,8 @@ if __name__ == "__main__":
print('[CREATING BUCKET POLICY FOR USER INSTANCES]')
params = '--bucket_name {} --ssn_bucket_name {} --shared_bucket_name {} --username {} --edge_role_name {} ' \
'--notebook_role_name {} --service_base_name {} --region {} ' \
- '--user_predefined_s3_policies "{}"'.format(project_conf['bucket_name'], project_conf['ssn_bucket_name'],
+ '--user_predefined_s3_policies "{}"'.format(project_conf['bucket_name'],
+ project_conf['ssn_bucket_name'],
project_conf['shared_bucket_name'],
os.environ['project_name'], project_conf['edge_role_name'],
project_conf['notebook_dataengine_role_name'],
@@ -555,9 +571,10 @@ if __name__ == "__main__":
print('[CREATE EDGE INSTANCE]')
params = "--node_name {} --ami_id {} --instance_type {} --key_name {} --security_group_ids {} " \
"--subnet_id {} --iam_profile {} --infra_tag_name {} --infra_tag_value {}" \
- .format(project_conf['edge_instance_name'], project_conf['ami_id'], project_conf['instance_size'], project_conf['key_name'],
- project_group_id, project_conf['public_subnet_id'], project_conf['edge_role_profile_name'],
- project_conf['tag_name'], project_conf['edge_instance_name'])
+ .format(project_conf['edge_instance_name'], project_conf['ami_id'], project_conf['instance_size'],
+ project_conf['key_name'], project_group_id, project_conf['public_subnet_id'],
+ project_conf['edge_role_profile_name'], project_conf['tag_name'],
+ project_conf['edge_instance_name'])
try:
local("~/scripts/{}.py {}".format('common_create_instance', params))
edge_instance = get_instance_by_name(project_conf['tag_name'], project_conf['edge_instance_name'])
@@ -586,7 +603,8 @@ if __name__ == "__main__":
except:
project_conf['elastic_ip'] = 'None'
params = "--elastic_ip {} --edge_id {} --infra_tag_name {} --infra_tag_value {}".format(
- project_conf['elastic_ip'], project_conf['edge_id'], project_conf['tag_name'], project_conf['elastic_ip_name'])
+ project_conf['elastic_ip'], project_conf['edge_id'], project_conf['tag_name'],
+ project_conf['elastic_ip_name'])
try:
local("~/scripts/{}.py {}".format('edge_associate_elastic_ip', params))
except:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org