You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by om...@apache.org on 2019/04/08 13:56:17 UTC
[incubator-dlab] branch DLAB-594 updated: [DLAB-594]: added tags
for VPCs, subnets, Route tables, SGs and IAM roles
This is an automated email from the ASF dual-hosted git repository.
omartushevskyi pushed a commit to branch DLAB-594
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/DLAB-594 by this push:
new 2cad14f [DLAB-594]: added tags for VPCs,subnets, Route tables, SGs and IAM roles
2cad14f is described below
commit 2cad14f67d7ced63ece795a3540640413e3458bd
Author: Oleh Martushevskyi <Ol...@epam.com>
AuthorDate: Mon Apr 8 16:56:08 2019 +0300
[DLAB-594]: added tags for VPCs,subnets, Route tables, SGs and IAM roles
---
.../src/general/lib/aws/actions_lib.py | 10 +++++++++-
.../scripts/aws/common_create_role_policy.py | 5 ++++-
.../src/general/scripts/aws/common_create_subnet.py | 8 ++++----
.../src/general/scripts/aws/edge_prepare.py | 21 +++++++++++++--------
.../src/general/scripts/aws/ssn_create_vpc.py | 5 +++++
.../src/general/scripts/aws/ssn_prepare.py | 21 ++++++++++-----------
6 files changed, 45 insertions(+), 25 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
index b6978c8..9d10c5a 100644
--- a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
@@ -244,9 +244,11 @@ def create_subnet(vpc_id, subnet, tag):
def create_security_group(security_group_name, vpc_id, security_group_rules, egress, tag):
try:
ec2 = boto3.resource('ec2')
+ tag_name = {"Key": "Name", "Value": security_group_name}
group = ec2.create_security_group(GroupName=security_group_name, Description='security_group_name', VpcId=vpc_id)
time.sleep(10)
create_tag(group.id, tag)
+ create_tag(group.id, tag_name)
try:
group.revoke_egress(IpPermissions=[{"IpProtocol": "-1", "IpRanges": [{"CidrIp": "0.0.0.0/0"}],
"UserIdGroupPairs": [], "PrefixListIds": []}])
@@ -446,6 +448,7 @@ def create_instance(definitions, instance_tag, primary_disk_size=12):
"error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)}))
traceback.print_exc(file=sys.stdout)
+
def tag_intance_volume(instance_id, node_name, instance_tag):
try:
print('volume tagging')
@@ -474,6 +477,7 @@ def tag_intance_volume(instance_id, node_name, instance_tag):
file=sys.stdout)}))
traceback.print_exc(file=sys.stdout)
+
def tag_emr_volume(cluster_id, node_name, billing_tag):
try:
client = boto3.client('emr')
@@ -492,7 +496,8 @@ def tag_emr_volume(cluster_id, node_name, billing_tag):
file=sys.stdout)}))
traceback.print_exc(file=sys.stdout)
-def create_iam_role(role_name, role_profile, region, service='ec2'):
+
+def create_iam_role(role_name, role_profile, region, service='ec2', tag=None):
conn = boto3.client('iam')
try:
if region == 'cn-north-1':
@@ -506,6 +511,9 @@ def create_iam_role(role_name, role_profile, region, service='ec2'):
RoleName=role_name, AssumeRolePolicyDocument=
'{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":["' + service +
'.amazonaws.com"]},"Action":["sts:AssumeRole"]}]}')
+ if tag:
+ conn.tag_role(RoleName=role_name, Tags=[tag])
+ conn.tag_role(RoleName=role_name, Tags=[{"Key": "Name", "Value": role_name}])
except botocore.exceptions.ClientError as e_role:
if e_role.response['Error']['Code'] == 'EntityAlreadyExists':
print("IAM role already exists. Reusing...")
diff --git a/infrastructure-provisioning/src/general/scripts/aws/common_create_role_policy.py b/infrastructure-provisioning/src/general/scripts/aws/common_create_role_policy.py
index 06739f3..d52c43f 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/common_create_role_policy.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_create_role_policy.py
@@ -33,6 +33,8 @@ parser.add_argument('--policy_name', type=str, default='')
parser.add_argument('--policy_arn', type=str, default='')
parser.add_argument('--policy_file_name', type=str, default='')
parser.add_argument('--region', type=str, default='')
+parser.add_argument('--infra_tag_name', type=str, default='')
+parser.add_argument('--infra_tag_value', type=str, default='')
args = parser.parse_args()
@@ -41,8 +43,9 @@ if __name__ == "__main__":
try:
role_name = get_role_by_name(args.role_name)
if role_name == '':
+ tag = {"Key": args.infra_tag_name, "Value": args.infra_tag_value}
print("Creating role {0}, profile name {1}".format(args.role_name, args.role_profile_name))
- create_iam_role(args.role_name, args.role_profile_name, args.region)
+ create_iam_role(args.role_name, args.role_profile_name, args.region, tag)
else:
print("ROLE AND ROLE PROFILE ARE ALREADY CREATED")
print("ROLE {} created. IAM group {} created".format(args.role_name, args.role_profile_name))
diff --git a/infrastructure-provisioning/src/general/scripts/aws/common_create_subnet.py b/infrastructure-provisioning/src/general/scripts/aws/common_create_subnet.py
index bd18cdc..ba1b051 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/common_create_subnet.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_create_subnet.py
@@ -39,14 +39,13 @@ parser.add_argument('--infra_tag_value', type=str, default='')
parser.add_argument('--prefix', type=str, default='')
parser.add_argument('--ssn', type=bool, default=False)
parser.add_argument('--user_subnets_range', type=str, default='')
+parser.add_argument('--subnet_name', type=str, default='')
args = parser.parse_args()
if __name__ == "__main__":
- if args.ssn:
- tag = {"Key": args.infra_tag_name, "Value": "{}-subnet".format(args.infra_tag_value)}
- else:
- tag = {"Key": args.infra_tag_name, "Value": "{}-{}-subnet".format(args.infra_tag_value, args.username)}
+ tag = {"Key": args.infra_tag_name, "Value": args.subnet_name}
+ tag_name = {"Key": "Name", "Value": args.subnet_name}
try:
if args.user_subnets_range == '' or args.ssn:
ec2 = boto3.resource('ec2')
@@ -123,6 +122,7 @@ if __name__ == "__main__":
print("Creating subnet {0} in vpc {1} with tag {2}".
format(dlab_subnet_cidr, args.vpc_id, json.dumps(tag)))
subnet_id = create_subnet(args.vpc_id, dlab_subnet_cidr, tag)
+ create_tag(subnet_id, tag_name)
else:
print("REQUESTED SUBNET ALREADY EXISTS. USING CIDR {}".format(subnet_check))
subnet_id = get_subnet_by_cidr(subnet_check)
diff --git a/infrastructure-provisioning/src/general/scripts/aws/edge_prepare.py b/infrastructure-provisioning/src/general/scripts/aws/edge_prepare.py
index 2e5380a..3c5c55f 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/edge_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/edge_prepare.py
@@ -79,6 +79,8 @@ if __name__ == "__main__":
edge_conf['notebook_security_group_name'] = '{}-{}-nb-SG'.format(edge_conf['service_base_name'],
os.environ['edge_user_name'])
edge_conf['private_subnet_prefix'] = os.environ['aws_private_subnet_prefix']
+ edge_conf['private_subnet_name'] = '{0}-{1}-subnet'.format(os.environ['aws_private_subnet_prefix'],
+ os.environ['edge_user_name'])
edge_conf['dataengine_master_security_group_name'] = '{}-{}-dataengine-master-sg' \
.format(edge_conf['service_base_name'], os.environ['edge_user_name'])
edge_conf['dataengine_slave_security_group_name'] = '{}-{}-dataengine-slave-sg' \
@@ -113,10 +115,9 @@ if __name__ == "__main__":
logging.info('[CREATE SUBNET]')
print('[CREATE SUBNET]')
params = "--vpc_id '{}' --infra_tag_name {} --infra_tag_value {} --username {} --prefix {} " \
- "--user_subnets_range '{}'".format(edge_conf['vpc2_id'], edge_conf['tag_name'],
- edge_conf['service_base_name'], os.environ['edge_user_name'],
- edge_conf['private_subnet_prefix'],
- os.environ['conf_user_subnets_range'])
+ "--user_subnets_range '{}' --subnet_name {}".format(
+ edge_conf['vpc2_id'], edge_conf['tag_name'], edge_conf['service_base_name'], os.environ['edge_user_name'],
+ edge_conf['private_subnet_prefix'], os.environ['conf_user_subnets_range'], edge_conf['private_subnet_name'])
try:
local("~/scripts/{}.py {}".format('common_create_subnet', params))
except:
@@ -135,9 +136,11 @@ if __name__ == "__main__":
try:
logging.info('[CREATE EDGE ROLES]')
print('[CREATE EDGE ROLES]')
- params = "--role_name {} --role_profile_name {} --policy_name {} --region {}" \
+ params = "--role_name {} --role_profile_name {} --policy_name {} --region {} --infra_tag_name {} " \
+ "--infra_tag_value {}" \
.format(edge_conf['role_name'], edge_conf['role_profile_name'],
- edge_conf['policy_name'], os.environ['aws_region'])
+ edge_conf['policy_name'], os.environ['aws_region'], edge_conf['tag_name'],
+ edge_conf['service_base_name'])
try:
local("~/scripts/{}.py {}".format('common_create_role_policy', params))
except:
@@ -151,9 +154,11 @@ if __name__ == "__main__":
try:
logging.info('[CREATE BACKEND (NOTEBOOK) ROLES]')
print('[CREATE BACKEND (NOTEBOOK) ROLES]')
- params = "--role_name {} --role_profile_name {} --policy_name {} --region {}" \
+ params = "--role_name {} --role_profile_name {} --policy_name {} --region {} --infra_tag_name {} " \
+ "--infra_tag_value {}" \
.format(edge_conf['notebook_dataengine_role_name'], edge_conf['notebook_dataengine_role_profile_name'],
- edge_conf['notebook_dataengine_policy_name'], os.environ['aws_region'])
+ edge_conf['notebook_dataengine_policy_name'], os.environ['aws_region'], edge_conf['tag_name'],
+ edge_conf['service_base_name'])
try:
local("~/scripts/{}.py {}".format('common_create_role_policy', params))
except:
diff --git a/infrastructure-provisioning/src/general/scripts/aws/ssn_create_vpc.py b/infrastructure-provisioning/src/general/scripts/aws/ssn_create_vpc.py
index 4e1676f..3ab99dd 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/ssn_create_vpc.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/ssn_create_vpc.py
@@ -32,6 +32,7 @@ parser.add_argument('--secondary', dest='secondary', action='store_true')
parser.add_argument('--region', type=str, default='')
parser.add_argument('--infra_tag_name', type=str, default='')
parser.add_argument('--infra_tag_value', type=str, default='')
+parser.add_argument('--vpc_name', type=str, default='')
parser.set_defaults(secondary=False)
args = parser.parse_args()
@@ -42,14 +43,18 @@ if args.secondary:
if __name__ == "__main__":
tag = {"Key": args.infra_tag_name, "Value": args.infra_tag_value}
+ vpc_tag_name = {"Key": "Name", "Value": args.vpc_name}
+ rt_tag_name = {"Key": "Name", "Value": "{0}-rt".format(args.vpc_name)}
if args.vpc != '':
try:
vpc_id = get_vpc_by_tag(args.infra_tag_name, args.infra_tag_value)
if vpc_id == '':
print("Creating {3}vpc {0} in region {1} with tag {2}".format(args.vpc, args.region, json.dumps(tag), sec_str))
vpc_id = create_vpc(args.vpc, tag)
+ create_tag(vpc_id, vpc_tag_name)
enable_vpc_dns(vpc_id)
rt_id = create_rt(vpc_id, args.infra_tag_name, args.infra_tag_value, args.secondary)
+ create_tag(rt_id, rt_tag_name)
else:
print("REQUESTED {}VPC ALREADY EXISTS".format(sec_str))
print("{0}VPC_ID: {1}".format(sec_str, vpc_id))
diff --git a/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py b/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py
index 04b47fb..10c81cc 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py
@@ -92,15 +92,14 @@ if __name__ == "__main__":
pre_defined_vpc = True
logging.info('[CREATE VPC AND ROUTE TABLE]')
print('[CREATE VPC AND ROUTE TABLE]')
- params = "--vpc {} --region {} --infra_tag_name {} --infra_tag_value {}".format(
- vpc_cidr, region, tag_name, service_base_name)
+ params = "--vpc {} --region {} --infra_tag_name {} --infra_tag_value {} --vpc_name {}".format(
+ vpc_cidr, region, tag_name, service_base_name, vpc_name)
try:
local("~/scripts/{}.py {}".format('ssn_create_vpc', params))
except:
traceback.print_exc()
raise Exception
os.environ['aws_vpc_id'] = get_vpc_by_tag(tag_name, service_base_name)
- create_tag(os.environ['aws_vpc_id'], {"Key": "Name", "Value": vpc_name})
except Exception as err:
print('Error: {0}'.format(err))
append_result("Failed to create VPC. Exception:" + str(err))
@@ -118,15 +117,14 @@ if __name__ == "__main__":
pre_defined_vpc2 = True
logging.info('[CREATE SECONDARY VPC AND ROUTE TABLE]')
print('[CREATE SECONDARY VPC AND ROUTE TABLE]')
- params = "--vpc {} --region {} --infra_tag_name {} --infra_tag_value {} --secondary".format(
- vpc2_cidr, region, tag2_name, service_base_name)
+ params = "--vpc {} --region {} --infra_tag_name {} --infra_tag_value {} --secondary " \
+ "--vpc_name {}".format(vpc2_cidr, region, tag2_name, service_base_name, vpc2_name)
try:
local("~/scripts/{}.py {}".format('ssn_create_vpc', params))
except:
traceback.print_exc()
raise Exception
os.environ['aws_vpc2_id'] = get_vpc_by_tag(tag2_name, service_base_name)
- create_tag(os.environ['aws_vpc2_id'], {"Key": "Name", "Value": vpc2_name})
except Exception as err:
print('Error: {0}'.format(err))
append_result("Failed to create secondary VPC. Exception:" + str(err))
@@ -145,7 +143,8 @@ if __name__ == "__main__":
logging.info('[CREATE SUBNET]')
print('[CREATE SUBNET]')
params = "--vpc_id {} --username {} --infra_tag_name {} --infra_tag_value {} --prefix {} " \
- "--ssn {}".format(os.environ['aws_vpc_id'], 'ssn', tag_name, service_base_name, '20', True)
+ "--ssn {} --subnet_name {}".format(os.environ['aws_vpc_id'], 'ssn', tag_name,
+ service_base_name, '20', True, subnet_name)
try:
local("~/scripts/{}.py {}".format('common_create_subnet', params))
except:
@@ -271,8 +270,10 @@ if __name__ == "__main__":
sys.exit(1)
logging.info('[CREATE ROLES]')
print('[CREATE ROLES]')
- params = "--role_name {} --role_profile_name {} --policy_name {} --policy_file_name {} --region {}".\
- format(role_name, role_profile_name, policy_name, policy_path, os.environ['aws_region'])
+ params = "--role_name {} --role_profile_name {} --policy_name {} --policy_file_name {} --region {} " \
+ "--infra_tag_name {} --infra_tag_value {}".\
+ format(role_name, role_profile_name, policy_name, policy_path, os.environ['aws_region'], tag_name,
+ service_base_name)
try:
local("~/scripts/{}.py {}".format('common_create_role_policy', params))
except:
@@ -321,8 +322,6 @@ if __name__ == "__main__":
remove_vpc(os.environ['aws_vpc2_id'])
sys.exit(1)
-
-
if os.environ['conf_duo_vpc_enable'] == 'true':
try:
logging.info('[CREATE ENDPOINT AND ROUTE-TABLE FOR NOTEBOOK VPC]')
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org