You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2020/11/16 16:40:41 UTC
[incubator-datalab] 01/01: [DATALAB-2154]: changed how rules for
emr additional security group are created
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2154
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 1a2a3c2c36702c7b0c855ff66a89c28b844c9bba
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Mon Nov 16 18:28:07 2020 +0200
[DATALAB-2154]: changed how rules for emr additional security group are created
---
.../scripts/aws/dataengine-service_prepare.py | 39 ++++++++++++----------
1 file changed, 22 insertions(+), 17 deletions(-)
diff --git a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_prepare.py b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_prepare.py
index 1c14f14..94785ac 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_prepare.py
@@ -86,6 +86,8 @@ if __name__ == "__main__":
emr_conf['edge_instance_name'] = '{0}-{1}-{2}-edge'.format(emr_conf['service_base_name'],
emr_conf['project_name'], emr_conf['endpoint_name'])
emr_conf['edge_security_group_name'] = '{0}-sg'.format(emr_conf['edge_instance_name'])
+ emr_conf['nb_security_group_name'] = '{0}-{1}-{2}-nb-sg'.format(emr_conf['service_base_name'],
+ emr_conf['project_name'], emr_conf['endpoint_name'])
emr_conf['master_instance_type'] = os.environ['emr_master_instance_type']
emr_conf['slave_instance_type'] = os.environ['emr_slave_instance_type']
emr_conf['instance_count'] = os.environ['emr_instance_count']
@@ -171,8 +173,8 @@ if __name__ == "__main__":
logging.info('[CREATING ADDITIONAL SECURITY GROUPS FOR EMR]')
print("[CREATING ADDITIONAL SECURITY GROUPS FOR EMR]")
try:
- edge_group_id = datalab.meta_lib.check_security_group(emr_conf['edge_security_group_name'])
- cluster_sg_ingress = datalab.meta_lib.format_sg([
+ group_id = datalab.meta_lib.check_security_group(emr_conf['edge_security_group_name'])
+ cluster_sg_ingress = [
{
"IpProtocol": "-1",
"IpRanges": [{"CidrIp": emr_conf['subnet_cidr']}],
@@ -181,18 +183,12 @@ if __name__ == "__main__":
},
{
"IpProtocol": "-1",
- "IpRanges": [],
- "UserIdGroupPairs": [{"GroupId": edge_group_id}],
- "PrefixListIds": []
- },
- {
- "IpProtocol": "-1",
"IpRanges": [{"CidrIp": emr_conf['provision_instance_ip']}],
"UserIdGroupPairs": [],
"PrefixListIds": []
}
- ])
- cluster_sg_egress = datalab.meta_lib.format_sg([
+ ]
+ cluster_sg_egress = [
{
"IpProtocol": "-1",
"IpRanges": [{"CidrIp": emr_conf['subnet_cidr']}],
@@ -206,12 +202,6 @@ if __name__ == "__main__":
"PrefixListIds": [],
},
{
- "IpProtocol": "-1",
- "IpRanges": [],
- "UserIdGroupPairs": [{"GroupId": edge_group_id}],
- "PrefixListIds": []
- },
- {
"IpProtocol": "tcp",
"IpRanges": [{"CidrIp": emr_conf['all_ip_cidr']}],
"FromPort": 443,
@@ -219,7 +209,22 @@ if __name__ == "__main__":
"UserIdGroupPairs": [],
"PrefixListIds": [],
}
- ])
+ ]
+ if group_id:
+ cluster_sg_ingress.append({
+ "IpProtocol": "-1",
+ "IpRanges": [],
+ "UserIdGroupPairs": [{"GroupId": group_id}],
+ "PrefixListIds": []
+ })
+ cluster_sg_egress.append({
+ "IpProtocol": "-1",
+ "IpRanges": [],
+ "UserIdGroupPairs": [{"GroupId": group_id}],
+ "PrefixListIds": []
+ })
+ cluster_sg_ingress = datalab.meta_lib.format_sg(cluster_sg_ingress)
+ cluster_sg_egress = datalab.meta_lib.format_sg(cluster_sg_egress)
params = "--name {} " \
"--vpc_id {} " \
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org