You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by my...@apache.org on 2020/01/29 13:01:06 UTC
[incubator-dlab] 01/02: [DLAB-1487]: [AWS] Shared endpoint bucket
should be created during project creation
This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a commit to branch DLAB-1487
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit f6ad61da70ea835d3b6cd2100f93b3ea2b8b2a4c
Author: Mykola_Bodnar1 <bo...@gmail.com>
AuthorDate: Wed Jan 29 12:25:03 2020 +0200
[DLAB-1487]: [AWS] Shared endpoint bucket should be created during project creation
---
.../general/scripts/aws/common_create_policy.py | 3 --
.../src/general/scripts/aws/project_prepare.py | 22 +++++++----
.../src/general/scripts/aws/ssn_prepare.py | 46 ----------------------
3 files changed, 14 insertions(+), 57 deletions(-)
diff --git a/infrastructure-provisioning/src/general/scripts/aws/common_create_policy.py b/infrastructure-provisioning/src/general/scripts/aws/common_create_policy.py
index 3cad721..d7f5ee0 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/common_create_policy.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_create_policy.py
@@ -29,7 +29,6 @@ import boto3, botocore
parser = argparse.ArgumentParser()
parser.add_argument('--bucket_name', type=str, default='')
-parser.add_argument('--ssn_bucket_name', type=str, default='')
parser.add_argument('--shared_bucket_name', type=str, default='')
parser.add_argument('--service_base_name', type=str, default='')
parser.add_argument('--username', type=str, default='')
@@ -46,8 +45,6 @@ if __name__ == "__main__":
handler = open('/root/templates/edge_s3_policy.json', 'r')
policy = handler.read()
policy = policy.replace('BUCKET_NAME', args.bucket_name)
- # Removed for multiple Endpoints per project
- # policy = policy.replace('SSN_BUCK', args.ssn_bucket_name)
policy = policy.replace('SHARED_BUCK', args.shared_bucket_name)
if args.region == 'cn-north-1':
policy = policy.replace('aws', 'aws-cn')
diff --git a/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py b/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py
index 8de0370..019eace 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py
@@ -61,10 +61,9 @@ if __name__ == "__main__":
project_conf['project_name'],
project_conf['endpoint_name'])
project_conf['bucket_name'] = project_conf['bucket_name_tag'].lower().replace('_', '-')
- project_conf['ssn_bucket_name'] = '{}-ssn-bucket'.format(
- project_conf['service_base_name']).lower().replace('_', '-')
- project_conf['shared_bucket_name'] = '{0}-{1}-shared-bucket'.format(
- project_conf['service_base_name'], project_conf['endpoint_tag']).lower().replace('_', '-')
+ project_conf['shared_bucket_name_tag'] = '{0}-{1}-shared-bucket'.format(
+ project_conf['service_base_name'], project_conf['endpoint_tag'])
+ project_conf['shared_bucket_name'] = project_conf['shared_bucket_name_tag'].lower().replace('_', '-')
project_conf['edge_role_name'] = '{}-{}-edge-Role'.format(
project_conf['service_base_name'].lower().replace('-', '_'), os.environ['project_name'])
project_conf['edge_role_profile_name'] = '{}-{}-edge-Profile'.format(
@@ -521,7 +520,7 @@ if __name__ == "__main__":
raise Exception
except Exception as err:
print('Error: {0}'.format(err))
- append_result("Failed to create bucket.", str(err))
+ append_result("Failed to create security group.", str(err))
remove_all_iam_resources('notebook', os.environ['project_name'])
remove_all_iam_resources('edge', os.environ['project_name'])
remove_sgroups(project_conf['dataengine_instances_name'])
@@ -532,6 +531,14 @@ if __name__ == "__main__":
try:
logging.info('[CREATE BUCKETS]')
print('[CREATE BUCKETS]')
+ params = "--bucket_name {} --infra_tag_name {} --infra_tag_value {} --region {} --bucket_name_tag {}". \
+ format(project_conf['shared_bucket_name'], project_conf['tag_name'], project_conf['shared_bucket_name'], project_conf['region'], project_conf['shared_bucket_name_tag'])
+ try:
+ local("~/scripts/{}.py {}".format('common_create_bucket', params))
+ except:
+ traceback.print_exc()
+ raise Exception
+
params = "--bucket_name {} --infra_tag_name {} --infra_tag_value {} --region {} --bucket_name_tag {}" \
.format(project_conf['bucket_name'], project_conf['tag_name'], project_conf['bucket_name'],
project_conf['region'], project_conf['bucket_name_tag'])
@@ -542,7 +549,7 @@ if __name__ == "__main__":
raise Exception
except Exception as err:
print('Error: {0}'.format(err))
- append_result("Failed to create bucket.", str(err))
+ append_result("Failed to create buckets.", str(err))
remove_all_iam_resources('notebook', os.environ['project_name'])
remove_all_iam_resources('edge', os.environ['project_name'])
remove_sgroups(project_conf['dataengine_instances_name'])
@@ -553,10 +560,9 @@ if __name__ == "__main__":
try:
logging.info('[CREATING BUCKET POLICY FOR PROJECT INSTANCES]')
print('[CREATING BUCKET POLICY FOR USER INSTANCES]')
- params = '--bucket_name {} --ssn_bucket_name {} --shared_bucket_name {} --username {} --edge_role_name {} ' \
+ params = '--bucket_name {} --shared_bucket_name {} --username {} --edge_role_name {} ' \
'--notebook_role_name {} --service_base_name {} --region {} ' \
'--user_predefined_s3_policies "{}"'.format(project_conf['bucket_name'],
- project_conf['ssn_bucket_name'],
project_conf['shared_bucket_name'],
os.environ['project_name'], project_conf['edge_role_name'],
project_conf['notebook_dataengine_role_name'],
diff --git a/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py b/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py
index 9acf264..8615a25 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py
@@ -61,11 +61,7 @@ if __name__ == "__main__":
role_name = service_base_name.lower().replace('-', '_') + '-ssn-Role'
role_profile_name = service_base_name.lower().replace('-', '_') + '-ssn-Profile'
policy_name = service_base_name.lower().replace('-', '_') + '-ssn-Policy'
- ssn_bucket_name_tag = service_base_name + '-ssn-bucket'
default_endpoint_name = os.environ['default_endpoint_name']
- shared_bucket_name_tag = '{0}-{1}-shared-bucket'.format(service_base_name, default_endpoint_name)
- ssn_bucket_name = ssn_bucket_name_tag.lower().replace('_', '-')
- shared_bucket_name = shared_bucket_name_tag.lower().replace('_', '-')
tag_name = service_base_name + '-Tag'
tag2_name = service_base_name + '-secondary-Tag'
user_tag = "{0}:{0}-ssn-Role".format(service_base_name)
@@ -386,48 +382,6 @@ if __name__ == "__main__":
remove_route_tables(tag2_name, True)
remove_vpc(os.environ['aws_vpc2_id'])
sys.exit(1)
- try:
- logging.info('[CREATE BUCKETS]')
- print('[CREATE BUCKETS]')
- params = "--bucket_name {} --infra_tag_name {} --infra_tag_value {} --region {} --bucket_name_tag {}". \
- format(ssn_bucket_name, tag_name, ssn_bucket_name, region, ssn_bucket_name_tag)
-
- try:
- local("~/scripts/{}.py {}".format('common_create_bucket', params))
- except:
- traceback.print_exc()
- raise Exception
-
- params = "--bucket_name {} --infra_tag_name {} --infra_tag_value {} --region {} --bucket_name_tag {}". \
- format(shared_bucket_name, tag_name, shared_bucket_name, region, shared_bucket_name_tag)
-
- try:
- local("~/scripts/{}.py {}".format('common_create_bucket', params))
- except:
- traceback.print_exc()
- raise Exception
- except Exception as err:
- print('Error: {0}'.format(err))
- append_result("Unable to create bucket.", str(err))
- remove_all_iam_resources(instance)
- if pre_defined_sg:
- remove_sgroups(tag_name)
- if pre_defined_subnet:
- remove_internet_gateways(os.environ['aws_vpc_id'], tag_name, service_base_name)
- remove_subnets(service_base_name + "-subnet")
- if pre_defined_vpc:
- remove_vpc_endpoints(os.environ['aws_vpc_id'])
- remove_route_tables(tag_name, True)
- remove_vpc(os.environ['aws_vpc_id'])
- if pre_defined_vpc2:
- remove_peering('*')
- try:
- remove_vpc_endpoints(os.environ['aws_vpc2_id'])
- except:
- print("There are no VPC Endpoints")
- remove_route_tables(tag2_name, True)
- remove_vpc(os.environ['aws_vpc2_id'])
- sys.exit(1)
try:
logging.info('[CREATE SSN INSTANCE]')
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org