You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2020/10/02 15:34:15 UTC
[incubator-datalab] 03/10: [DLAB-1911]: added posability to specify
predefined security group for edge node on azure,
addition of new rule with endpoint ip,
and it`s deletion during project termination
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DLAB-1911
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit f0261f6916ffb17d41bb53dee60f2c2c3f9f2775
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Wed Sep 30 14:52:04 2020 +0300
[DLAB-1911]: added posability to specify predefined security group for edge node on azure, addition of new rule with endpoint ip, and it`s deletion during project termination
---
.../scripts/deploy_datalab.py | 6 +-
.../src/general/conf/datalab.ini | 2 +
.../src/general/lib/azure/actions_lib.py | 15 +
.../src/general/scripts/azure/project_prepare.py | 570 +++++++++++----------
.../src/general/scripts/azure/project_terminate.py | 21 +-
5 files changed, 334 insertions(+), 280 deletions(-)
diff --git a/infrastructure-provisioning/scripts/deploy_datalab.py b/infrastructure-provisioning/scripts/deploy_datalab.py
index 096150c..530f19b 100644
--- a/infrastructure-provisioning/scripts/deploy_datalab.py
+++ b/infrastructure-provisioning/scripts/deploy_datalab.py
@@ -72,8 +72,10 @@ parser.add_argument('--azure_subnet_name', type=str, default='', help='Azure Sub
parser.add_argument('--gcp_subnet_name', type=str, default='', help='GCP Subnet Name')
parser.add_argument('--aws_security_groups_ids', type=str, default='', help='One of more comma-separated Security '
'groups IDs for SSN')
-parser.add_argument('--azure_security_group_name', type=str, default='', help='One of more comma-separated Security '
- 'groups names for SSN')
+parser.add_argument('--azure_security_group_name', type=str, default='', help='One Security'
+ 'group name for SSN')
+parser.add_argument('--azure_edge_security_group_name', type=str, default='', help='One Security '
+ 'group name for Edge node')
parser.add_argument('--gcp_firewall_name', type=str, default='', help='One of more comma-separated GCP Firewall rules '
'for SSN')
parser.add_argument('--key_path', type=str, default='', help='Path to admin key (WITHOUT KEY NAME)')
diff --git a/infrastructure-provisioning/src/general/conf/datalab.ini b/infrastructure-provisioning/src/general/conf/datalab.ini
index 3663402..cc50cd3 100644
--- a/infrastructure-provisioning/src/general/conf/datalab.ini
+++ b/infrastructure-provisioning/src/general/conf/datalab.ini
@@ -142,6 +142,8 @@ redhat_image_name = RHEL-7.4_HVM-20180103-x86_64-2-Hourly2-GP2
# vpc_name =
### Subnet name
# subnet_name =
+### Id of the security group for SSN instance
+# edge_security_group =
### EC2 instance type for SSN
ssn_instance_size = Standard_DS2_v2
### Instance type for EDGE
diff --git a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
index d11e223..597a5ca 100644
--- a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
@@ -225,6 +225,21 @@ class AzureActions:
file=sys.stdout)}))
traceback.print_exc(file=sys.stdout)
+ def remove_security_rules(self, resource_group_name, network_security_group_name, security_rule_name):
+ try:
+ result = self.network_client.security_rules.delete(
+ network_security_group_name,
+ resource_group_name,
+ security_rule_name).wait()
+ return result
+ except Exception as err:
+ logging.info(
+ "Unable to remove security rule: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout))
+ append_result(str({"error": "Unable to remove security rule",
+ "error_message": str(err) + "\n Traceback: " + traceback.print_exc(
+ file=sys.stdout)}))
+ traceback.print_exc(file=sys.stdout)
+
def remove_security_group(self, resource_group_name, network_security_group_name):
try:
result = self.network_client.network_security_groups.delete(
diff --git a/infrastructure-provisioning/src/general/scripts/azure/project_prepare.py b/infrastructure-provisioning/src/general/scripts/azure/project_prepare.py
index 92db0a2..25577a2 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/project_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/project_prepare.py
@@ -169,277 +169,305 @@ if __name__ == "__main__":
print('NEW SUBNET CIDR CREATED: {}'.format(project_conf['private_subnet_cidr']))
try:
- logging.info('[CREATE SECURITY GROUP FOR EDGE NODE]')
- print('[CREATE SECURITY GROUP FOR EDGE]')
- edge_list_rules = [
- {
- "name": "in-1",
- "protocol": "*",
- "source_port_range": "*",
- "destination_port_range": "*",
- "source_address_prefix": project_conf['private_subnet_cidr'],
- "destination_address_prefix": "*",
- "access": "Allow",
- "priority": 100,
- "direction": "Inbound"
- },
- {
- "name": "in-2",
- "protocol": "Tcp",
- "source_port_range": "*",
- "destination_port_range": "22",
- "source_address_prefix": "*",
- "destination_address_prefix": "*",
- "access": "Allow",
- "priority": 110,
- "direction": "Inbound"
- },
- {
- "name": "in-3",
- "protocol": "Tcp",
- "source_port_range": "*",
- "destination_port_range": "3128",
- "source_address_prefix": "*",
- "destination_address_prefix": "*",
- "access": "Allow",
- "priority": 120,
- "direction": "Inbound"
- },
- {
- "name": "in-4",
- "protocol": "Tcp",
- "source_port_range": "*",
- "destination_port_range": "80",
- "source_address_prefix": "*",
- "destination_address_prefix": "*",
- "access": "Allow",
- "priority": 130,
- "direction": "Inbound"
- },
- {
- "name": "in-5",
- "protocol": "Tcp",
- "source_port_range": "*",
- "destination_port_range": "443",
- "source_address_prefix": "*",
- "destination_address_prefix": "*",
- "access": "Allow",
- "priority": 140,
- "direction": "Inbound"
- },
- {
- "name": "out-1",
- "protocol": "Tcp",
- "source_port_range": "*",
- "destination_port_range": "22",
- "source_address_prefix": "*",
- "destination_address_prefix": project_conf['private_subnet_cidr'],
- "access": "Allow",
- "priority": 100,
- "direction": "Outbound"
- },
- {
- "name": "out-2",
- "protocol": "Tcp",
- "source_port_range": "*",
- "destination_port_range": "8888",
- "source_address_prefix": "*",
- "destination_address_prefix": project_conf['private_subnet_cidr'],
- "access": "Allow",
- "priority": 110,
- "direction": "Outbound"
- },
- {
- "name": "out-3",
- "protocol": "Tcp",
- "source_port_range": "*",
- "destination_port_range": "8080",
- "source_address_prefix": "*",
- "destination_address_prefix": project_conf['private_subnet_cidr'],
- "access": "Allow",
- "priority": 120,
- "direction": "Outbound"
- },
- {
- "name": "out-4",
- "protocol": "Tcp",
- "source_port_range": "*",
- "destination_port_range": "8787",
- "source_address_prefix": "*",
- "destination_address_prefix": project_conf['private_subnet_cidr'],
- "access": "Allow",
- "priority": 130,
- "direction": "Outbound"
- },
- {
- "name": "out-5",
- "protocol": "Tcp",
- "source_port_range": "*",
- "destination_port_range": "6006",
- "source_address_prefix": "*",
- "destination_address_prefix": project_conf['private_subnet_cidr'],
- "access": "Allow",
- "priority": 140,
- "direction": "Outbound"
- },
- {
- "name": "out-6",
- "protocol": "Tcp",
- "source_port_range": "*",
- "destination_port_range": "20888",
- "source_address_prefix": "*",
- "destination_address_prefix": project_conf['private_subnet_cidr'],
- "access": "Allow",
- "priority": 150,
- "direction": "Outbound"
- },
- {
- "name": "out-7",
- "protocol": "Tcp",
- "source_port_range": "*",
- "destination_port_range": "8088",
- "source_address_prefix": "*",
- "destination_address_prefix": project_conf['private_subnet_cidr'],
- "access": "Allow",
- "priority": 160,
- "direction": "Outbound"
- },
- {
- "name": "out-8",
- "protocol": "Tcp",
- "source_port_range": "*",
- "destination_port_range": "18080",
- "source_address_prefix": "*",
- "destination_address_prefix": project_conf['private_subnet_cidr'],
- "access": "Allow",
- "priority": 170,
- "direction": "Outbound"
- },
- {
- "name": "out-9",
- "protocol": "Tcp",
- "source_port_range": "*",
- "destination_port_range": "50070",
- "source_address_prefix": "*",
- "destination_address_prefix": project_conf['private_subnet_cidr'],
- "access": "Allow",
- "priority": 180,
- "direction": "Outbound"
- },
- {
- "name": "out-10",
- "protocol": "Tcp",
- "source_port_range": "*",
- "destination_port_range": "8085",
- "source_address_prefix": "*",
- "destination_address_prefix": project_conf['private_subnet_cidr'],
- "access": "Allow",
- "priority": 190,
- "direction": "Outbound"
- },
- {
- "name": "out-11",
- "protocol": "Tcp",
- "source_port_range": "*",
- "destination_port_range": "8081",
- "source_address_prefix": "*",
- "destination_address_prefix": project_conf['private_subnet_cidr'],
- "access": "Allow",
- "priority": 200,
- "direction": "Outbound"
- },
- {
- "name": "out-12",
- "protocol": "Tcp",
- "source_port_range": "*",
- "destination_port_range": "4040-4140",
- "source_address_prefix": "*",
- "destination_address_prefix": project_conf['private_subnet_cidr'],
- "access": "Allow",
- "priority": 210,
- "direction": "Outbound"
- },
- {
- "name": "out-13",
- "protocol": "Udp",
- "source_port_range": "*",
- "destination_port_range": "53",
- "source_address_prefix": '*',
- "destination_address_prefix": "*",
- "access": "Allow",
- "priority": 220,
- "direction": "Outbound"
- },
- {
- "name": "out-14",
- "protocol": "Tcp",
- "source_port_range": "*",
- "destination_port_range": "80",
- "source_address_prefix": '*',
- "destination_address_prefix": "*",
- "access": "Allow",
- "priority": 230,
- "direction": "Outbound"
- },
- {
- "name": "out-15",
- "protocol": "Tcp",
- "source_port_range": "*",
- "destination_port_range": "443",
- "source_address_prefix": '*',
- "destination_address_prefix": "*",
- "access": "Allow",
- "priority": 240,
- "direction": "Outbound"
- },
- {
- "name": "out-16",
- "protocol": "Tcp",
- "source_port_range": "*",
- "destination_port_range": "389",
- "source_address_prefix": '*',
- "destination_address_prefix": "*",
- "access": "Allow",
- "priority": 250,
- "direction": "Outbound"
- },
- {
- "name": "out-17",
- "protocol": "Tcp",
- "source_port_range": "*",
- "destination_port_range": "8042",
- "source_address_prefix": "*",
- "destination_address_prefix": project_conf['private_subnet_cidr'],
- "access": "Allow",
- "priority": 260,
- "direction": "Outbound"
- },
- {
- "name": "out-18",
- "protocol": "Udp",
- "source_port_range": "*",
- "destination_port_range": "123",
- "source_address_prefix": "*",
- "destination_address_prefix": "*",
- "access": "Allow",
- "priority": 270,
- "direction": "Outbound"
- },
- {
- "name": "out-19",
- "protocol": "*",
- "source_port_range": "*",
- "destination_port_range": "*",
- "source_address_prefix": "*",
- "destination_address_prefix": "*",
- "access": "Deny",
- "priority": 300,
- "direction": "Outbound"
- }
- ]
- params = "--resource_group_name {} --security_group_name {} --region {} --tags '{}' --list_rules '{}'". \
- format(project_conf['resource_group_name'], project_conf['edge_security_group_name'],
- project_conf['region'], json.dumps(project_conf['instance_tags']), json.dumps(edge_list_rules))
+ if 'azure_edge_security_group_name' in os.environ:
+ logging.info('Security group predefined, adding new rule with endpoint IP')
+ print('Security group predefined, adding new rule with endpoint IP')
+ if project_conf['endpoint_name'] == 'local':
+ endpoint_ip = AzureMeta.get_instance_public_ip_address(edge_conf['resource_group_name'],
+ '{}-ssn'.format(project_conf['service_base_name']))
+ else:
+ endpoint_ip = AzureMeta.get_instance_public_ip_address(edge_conf['resource_group_name'],
+ '{}-{}-endpoint'.format(project_conf['service_base_name'], project_conf['endpoint_name']))
+ edge_list_rules = [
+ {
+ "name": '{}-{}-{}-rule'.format(project_conf['service_base_name'],
+ project_conf['project_name'],
+ project_conf['endpoint_tag']),
+ "protocol": "*",
+ "source_port_range": "*",
+ "destination_port_range": "*",
+ "source_address_prefix": endpoint_ip,
+ "destination_address_prefix": "*",
+ "access": "Allow",
+ "priority": 100,
+ "direction": "Inbound"
+ }
+ ]
+ params = "--resource_group_name {} --security_group_name {} --region {} --list_rules '{}'". \
+ format(project_conf['resource_group_name'], os.environ['azure_edge_security_group_name'],
+ project_conf['region'], json.dumps(edge_list_rules))
+ else:
+ logging.info('[CREATE SECURITY GROUP FOR EDGE NODE]')
+ print('[CREATE SECURITY GROUP FOR EDGE]')
+ edge_list_rules = [
+ {
+ "name": "in-1",
+ "protocol": "*",
+ "source_port_range": "*",
+ "destination_port_range": "*",
+ "source_address_prefix": project_conf['private_subnet_cidr'],
+ "destination_address_prefix": "*",
+ "access": "Allow",
+ "priority": 100,
+ "direction": "Inbound"
+ },
+ {
+ "name": "in-2",
+ "protocol": "Tcp",
+ "source_port_range": "*",
+ "destination_port_range": "22",
+ "source_address_prefix": "*",
+ "destination_address_prefix": "*",
+ "access": "Allow",
+ "priority": 110,
+ "direction": "Inbound"
+ },
+ {
+ "name": "in-3",
+ "protocol": "Tcp",
+ "source_port_range": "*",
+ "destination_port_range": "3128",
+ "source_address_prefix": "*",
+ "destination_address_prefix": "*",
+ "access": "Allow",
+ "priority": 120,
+ "direction": "Inbound"
+ },
+ {
+ "name": "in-4",
+ "protocol": "Tcp",
+ "source_port_range": "*",
+ "destination_port_range": "80",
+ "source_address_prefix": "*",
+ "destination_address_prefix": "*",
+ "access": "Allow",
+ "priority": 130,
+ "direction": "Inbound"
+ },
+ {
+ "name": "in-5",
+ "protocol": "Tcp",
+ "source_port_range": "*",
+ "destination_port_range": "443",
+ "source_address_prefix": "*",
+ "destination_address_prefix": "*",
+ "access": "Allow",
+ "priority": 140,
+ "direction": "Inbound"
+ },
+ {
+ "name": "out-1",
+ "protocol": "Tcp",
+ "source_port_range": "*",
+ "destination_port_range": "22",
+ "source_address_prefix": "*",
+ "destination_address_prefix": project_conf['private_subnet_cidr'],
+ "access": "Allow",
+ "priority": 100,
+ "direction": "Outbound"
+ },
+ {
+ "name": "out-2",
+ "protocol": "Tcp",
+ "source_port_range": "*",
+ "destination_port_range": "8888",
+ "source_address_prefix": "*",
+ "destination_address_prefix": project_conf['private_subnet_cidr'],
+ "access": "Allow",
+ "priority": 110,
+ "direction": "Outbound"
+ },
+ {
+ "name": "out-3",
+ "protocol": "Tcp",
+ "source_port_range": "*",
+ "destination_port_range": "8080",
+ "source_address_prefix": "*",
+ "destination_address_prefix": project_conf['private_subnet_cidr'],
+ "access": "Allow",
+ "priority": 120,
+ "direction": "Outbound"
+ },
+ {
+ "name": "out-4",
+ "protocol": "Tcp",
+ "source_port_range": "*",
+ "destination_port_range": "8787",
+ "source_address_prefix": "*",
+ "destination_address_prefix": project_conf['private_subnet_cidr'],
+ "access": "Allow",
+ "priority": 130,
+ "direction": "Outbound"
+ },
+ {
+ "name": "out-5",
+ "protocol": "Tcp",
+ "source_port_range": "*",
+ "destination_port_range": "6006",
+ "source_address_prefix": "*",
+ "destination_address_prefix": project_conf['private_subnet_cidr'],
+ "access": "Allow",
+ "priority": 140,
+ "direction": "Outbound"
+ },
+ {
+ "name": "out-6",
+ "protocol": "Tcp",
+ "source_port_range": "*",
+ "destination_port_range": "20888",
+ "source_address_prefix": "*",
+ "destination_address_prefix": project_conf['private_subnet_cidr'],
+ "access": "Allow",
+ "priority": 150,
+ "direction": "Outbound"
+ },
+ {
+ "name": "out-7",
+ "protocol": "Tcp",
+ "source_port_range": "*",
+ "destination_port_range": "8088",
+ "source_address_prefix": "*",
+ "destination_address_prefix": project_conf['private_subnet_cidr'],
+ "access": "Allow",
+ "priority": 160,
+ "direction": "Outbound"
+ },
+ {
+ "name": "out-8",
+ "protocol": "Tcp",
+ "source_port_range": "*",
+ "destination_port_range": "18080",
+ "source_address_prefix": "*",
+ "destination_address_prefix": project_conf['private_subnet_cidr'],
+ "access": "Allow",
+ "priority": 170,
+ "direction": "Outbound"
+ },
+ {
+ "name": "out-9",
+ "protocol": "Tcp",
+ "source_port_range": "*",
+ "destination_port_range": "50070",
+ "source_address_prefix": "*",
+ "destination_address_prefix": project_conf['private_subnet_cidr'],
+ "access": "Allow",
+ "priority": 180,
+ "direction": "Outbound"
+ },
+ {
+ "name": "out-10",
+ "protocol": "Tcp",
+ "source_port_range": "*",
+ "destination_port_range": "8085",
+ "source_address_prefix": "*",
+ "destination_address_prefix": project_conf['private_subnet_cidr'],
+ "access": "Allow",
+ "priority": 190,
+ "direction": "Outbound"
+ },
+ {
+ "name": "out-11",
+ "protocol": "Tcp",
+ "source_port_range": "*",
+ "destination_port_range": "8081",
+ "source_address_prefix": "*",
+ "destination_address_prefix": project_conf['private_subnet_cidr'],
+ "access": "Allow",
+ "priority": 200,
+ "direction": "Outbound"
+ },
+ {
+ "name": "out-12",
+ "protocol": "Tcp",
+ "source_port_range": "*",
+ "destination_port_range": "4040-4140",
+ "source_address_prefix": "*",
+ "destination_address_prefix": project_conf['private_subnet_cidr'],
+ "access": "Allow",
+ "priority": 210,
+ "direction": "Outbound"
+ },
+ {
+ "name": "out-13",
+ "protocol": "Udp",
+ "source_port_range": "*",
+ "destination_port_range": "53",
+ "source_address_prefix": '*',
+ "destination_address_prefix": "*",
+ "access": "Allow",
+ "priority": 220,
+ "direction": "Outbound"
+ },
+ {
+ "name": "out-14",
+ "protocol": "Tcp",
+ "source_port_range": "*",
+ "destination_port_range": "80",
+ "source_address_prefix": '*',
+ "destination_address_prefix": "*",
+ "access": "Allow",
+ "priority": 230,
+ "direction": "Outbound"
+ },
+ {
+ "name": "out-15",
+ "protocol": "Tcp",
+ "source_port_range": "*",
+ "destination_port_range": "443",
+ "source_address_prefix": '*',
+ "destination_address_prefix": "*",
+ "access": "Allow",
+ "priority": 240,
+ "direction": "Outbound"
+ },
+ {
+ "name": "out-16",
+ "protocol": "Tcp",
+ "source_port_range": "*",
+ "destination_port_range": "389",
+ "source_address_prefix": '*',
+ "destination_address_prefix": "*",
+ "access": "Allow",
+ "priority": 250,
+ "direction": "Outbound"
+ },
+ {
+ "name": "out-17",
+ "protocol": "Tcp",
+ "source_port_range": "*",
+ "destination_port_range": "8042",
+ "source_address_prefix": "*",
+ "destination_address_prefix": project_conf['private_subnet_cidr'],
+ "access": "Allow",
+ "priority": 260,
+ "direction": "Outbound"
+ },
+ {
+ "name": "out-18",
+ "protocol": "Udp",
+ "source_port_range": "*",
+ "destination_port_range": "123",
+ "source_address_prefix": "*",
+ "destination_address_prefix": "*",
+ "access": "Allow",
+ "priority": 270,
+ "direction": "Outbound"
+ },
+ {
+ "name": "out-19",
+ "protocol": "*",
+ "source_port_range": "*",
+ "destination_port_range": "*",
+ "source_address_prefix": "*",
+ "destination_address_prefix": "*",
+ "access": "Deny",
+ "priority": 300,
+ "direction": "Outbound"
+ }
+ ]
+ params = "--resource_group_name {} --security_group_name {} --region {} --tags '{}' --list_rules '{}'". \
+ format(project_conf['resource_group_name'], project_conf['edge_security_group_name'],
+ project_conf['region'], json.dumps(project_conf['instance_tags']), json.dumps(edge_list_rules))
try:
local("~/scripts/{}.py {}".format('common_create_security_group', params))
except Exception as err:
diff --git a/infrastructure-provisioning/src/general/scripts/azure/project_terminate.py b/infrastructure-provisioning/src/general/scripts/azure/project_terminate.py
index 7549eac..691d7b2 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/project_terminate.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/project_terminate.py
@@ -124,13 +124,20 @@ def terminate_edge_node(resource_group_name, service_base_name, project_tag, sub
print("Removing security groups")
try:
- for sg in AzureMeta.network_client.network_security_groups.list(resource_group_name):
- try:
- if project_tag == sg.tags["project_tag"]:
- AzureActions.remove_security_group(resource_group_name, sg.name)
- print("Security group {} has been terminated".format(sg.name))
- except:
- pass
+ if 'azure_edge_security_group_name' in os.environ:
+ AzureActions.remove_security_rules(resource_group_name,
+ os.environ['azure_edge_security_group_name'],
+ '{}-{}-{}-rule'.format(project_conf['service_base_name'],
+ project_conf['project_name'],
+ project_conf['endpoint_name']))
+ else:
+ for sg in AzureMeta.network_client.network_security_groups.list(resource_group_name):
+ try:
+ if project_tag == sg.tags["project_tag"]:
+ AzureActions.remove_security_group(resource_group_name, sg.name)
+ print("Security group {} has been terminated".format(sg.name))
+ except:
+ pass
except Exception as err:
datalab.fab.append_result("Failed to remove security groups.", str(err))
sys.exit(1)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org