You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2020/10/01 14:32:59 UTC
[incubator-datalab] branch DLAB-1911 updated: [DLAB-1911]: added
creation of separate security rules in predefined edge sg in azure
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DLAB-1911
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
The following commit(s) were added to refs/heads/DLAB-1911 by this push:
new 1b306f6 [DLAB-1911]: added creation of separate security rules in predefined edge sg in azure
1b306f6 is described below
commit 1b306f68e18deac57fe777e6edb617bb762547bc
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Thu Oct 1 17:32:45 2020 +0300
[DLAB-1911]: added creation of separate security rules in predefined edge sg in azure
---
.../src/general/lib/azure/actions_lib.py | 25 ++++----
.../scripts/azure/common_create_security_group.py | 5 +-
.../src/general/scripts/azure/project_prepare.py | 67 ++++++++++++++--------
.../src/general/scripts/azure/project_terminate.py | 19 +++---
4 files changed, 71 insertions(+), 45 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
index 2a1aa03..70d8915 100644
--- a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
@@ -200,16 +200,18 @@ class AzureActions:
file=sys.stdout)}))
traceback.print_exc(file=sys.stdout)
- def create_security_group(self, resource_group_name, network_security_group_name, region, tags, list_rules):
+ def create_security_group(self, resource_group_name, network_security_group_name, region, tags, list_rules, preexisting_sg = False):
try:
- result = self.network_client.network_security_groups.create_or_update(
- resource_group_name,
- network_security_group_name,
- {
- 'location': region,
- 'tags': tags,
- }
- ).wait()
+ result = ''
+ if not preexisting_sg:
+ result = self.network_client.network_security_groups.create_or_update(
+ resource_group_name,
+ network_security_group_name,
+ {
+ 'location': region,
+ 'tags': tags,
+ }
+ ).wait()
for rule in list_rules:
self.network_client.security_rules.create_or_update(
resource_group_name,
@@ -217,7 +219,8 @@ class AzureActions:
security_rule_name=rule['name'],
security_rule_parameters=rule
).wait()
- return result
+ if result:
+ return result
except Exception as err:
logging.info(
"Unable to create security group: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout))
@@ -226,7 +229,7 @@ class AzureActions:
file=sys.stdout)}))
traceback.print_exc(file=sys.stdout)
- def remove_security_rules(self, resource_group_name, network_security_group_name, security_rule_name):
+ def remove_security_rules(self, network_security_group_name, resource_group_name, security_rule_name):
try:
result = self.network_client.security_rules.delete(
network_security_group_name,
diff --git a/infrastructure-provisioning/src/general/scripts/azure/common_create_security_group.py b/infrastructure-provisioning/src/general/scripts/azure/common_create_security_group.py
index 62cd0d7..0a13256 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/common_create_security_group.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_create_security_group.py
@@ -39,7 +39,10 @@ args = parser.parse_args()
if __name__ == "__main__":
try:
if AzureMeta().get_security_group(args.resource_group_name, args.security_group_name):
- print("REQUESTED SECURITY GROUP {} ALREADY EXISTS".format(args.security_group_name))
+ print("REQUESTED SECURITY GROUP {} ALREADY EXISTS. Updating rules".format(args.security_group_name))
+ security_group = AzureActions().create_security_group(args.resource_group_name, args.security_group_name,
+ args.region, json.loads(args.tags),
+ json.loads(args.list_rules), True)
else:
print("Creating security group {}.".format(args.security_group_name))
security_group = AzureActions().create_security_group(args.resource_group_name, args.security_group_name,
diff --git a/infrastructure-provisioning/src/general/scripts/azure/project_prepare.py b/infrastructure-provisioning/src/general/scripts/azure/project_prepare.py
index cc756d3..beb1851 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/project_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/project_prepare.py
@@ -178,11 +178,17 @@ if __name__ == "__main__":
logging.info('Security group predefined, adding new rule with endpoint IP')
print('Security group predefined, adding new rule with endpoint IP')
if project_conf['endpoint_name'] == 'local':
- endpoint_ip = AzureMeta.get_instance_public_ip_address(edge_conf['resource_group_name'],
+ endpoint_ip = AzureMeta.get_instance_public_ip_address(project_conf['resource_group_name'],
'{}-ssn'.format(project_conf['service_base_name']))
else:
- endpoint_ip = AzureMeta.get_instance_public_ip_address(edge_conf['resource_group_name'],
+ endpoint_ip = AzureMeta.get_instance_public_ip_address(project_conf['resource_group_name'],
'{}-{}-endpoint'.format(project_conf['service_base_name'], project_conf['endpoint_name']))
+ priority = 110
+ rules_list = AzureMeta.get_security_group(project_conf['resource_group_name'], os.environ['azure_edge_security_group_name'])
+ for rule in rules_list.as_dict()['security_rules']:
+ priorities.append(rule['priority'])
+ while priority in priorities:
+ priority += 10
edge_list_rules = [
{
"name": '{}-{}-{}-rule'.format(project_conf['service_base_name'],
@@ -194,12 +200,18 @@ if __name__ == "__main__":
"source_address_prefix": endpoint_ip,
"destination_address_prefix": "*",
"access": "Allow",
+ "priority": priority,
"direction": "Inbound"
}
]
- params = "--resource_group_name {} --security_group_name {} --region {} --list_rules '{}'". \
+ params = "--resource_group_name {} --security_group_name {} --region {} --tags '{}' --list_rules '{}'". \
format(project_conf['resource_group_name'], os.environ['azure_edge_security_group_name'],
- project_conf['region'], json.dumps(edge_list_rules))
+ project_conf['region'], json.dumps({"product": "datalab"}), json.dumps(edge_list_rules))
+ try:
+ local("~/scripts/{}.py {}".format('common_create_security_group', params))
+ except Exception as err:
+ AzureActions.remove_subnet(project_conf['resource_group_name'], project_conf['vpc_name'],
+ project_conf['private_subnet_name'])
else:
logging.info('[CREATE SECURITY GROUP FOR EDGE NODE]')
print('[CREATE SECURITY GROUP FOR EDGE]')
@@ -472,19 +484,19 @@ if __name__ == "__main__":
params = "--resource_group_name {} --security_group_name {} --region {} --tags '{}' --list_rules '{}'". \
format(project_conf['resource_group_name'], project_conf['edge_security_group_name'],
project_conf['region'], json.dumps(project_conf['instance_tags']), json.dumps(edge_list_rules))
- try:
- local("~/scripts/{}.py {}".format('common_create_security_group', params))
- except Exception as err:
- AzureActions.remove_subnet(project_conf['resource_group_name'], project_conf['vpc_name'],
- project_conf['private_subnet_name'])
try:
- AzureActions.remove_security_group(project_conf['resource_group_name'],
- project_conf['edge_security_group_name'])
- except:
- print("Edge Security group hasn't been created.")
- traceback.print_exc()
- dlab.fab.append_result("Failed creating security group for edge node.", str(err))
- raise Exception
+ local("~/scripts/{}.py {}".format('common_create_security_group', params))
+ except Exception as err:
+ AzureActions.remove_subnet(project_conf['resource_group_name'], project_conf['vpc_name'],
+ project_conf['private_subnet_name'])
+ try:
+ AzureActions.remove_security_group(project_conf['resource_group_name'],
+ project_conf['edge_security_group_name'])
+ except:
+ print("Edge Security group hasn't been created.")
+ traceback.print_exc()
+ dlab.fab.append_result("Failed creating security group for edge node.", str(err))
+ raise Exception
except:
traceback.print_exc()
sys.exit(1)
@@ -587,7 +599,8 @@ if __name__ == "__main__":
dlab.fab.append_result("Failed creating security group for private subnet.", str(err))
AzureActions.remove_subnet(project_conf['resource_group_name'], project_conf['vpc_name'],
project_conf['private_subnet_name'])
- AzureActions.remove_security_group(project_conf['resource_group_name'],
+ if 'azure_edge_security_group_name' not in os.environ:
+ AzureActions.remove_security_group(project_conf['resource_group_name'],
project_conf['edge_security_group_name'])
try:
AzureActions.remove_security_group(project_conf['resource_group_name'],
@@ -693,7 +706,8 @@ if __name__ == "__main__":
except Exception as err:
AzureActions.remove_subnet(project_conf['resource_group_name'], project_conf['vpc_name'],
project_conf['private_subnet_name'])
- AzureActions.remove_security_group(project_conf['resource_group_name'],
+ if 'azure_edge_security_group_name' not in os.environ:
+ AzureActions.remove_security_group(project_conf['resource_group_name'],
project_conf['edge_security_group_name'])
AzureActions.remove_security_group(project_conf['resource_group_name'],
project_conf['notebook_security_group_name'])
@@ -719,7 +733,8 @@ if __name__ == "__main__":
except Exception as err:
AzureActions.remove_subnet(project_conf['resource_group_name'], project_conf['vpc_name'],
project_conf['private_subnet_name'])
- AzureActions.remove_security_group(project_conf['resource_group_name'],
+ if 'azure_edge_security_group_name' not in os.environ:
+ AzureActions.remove_security_group(project_conf['resource_group_name'],
project_conf['edge_security_group_name'])
AzureActions.remove_security_group(project_conf['resource_group_name'],
project_conf['notebook_security_group_name'])
@@ -744,7 +759,8 @@ if __name__ == "__main__":
dlab.fab.append_result("Failed to create storage account.", str(err))
AzureActions.remove_subnet(project_conf['resource_group_name'], project_conf['vpc_name'],
project_conf['private_subnet_name'])
- AzureActions.remove_security_group(project_conf['resource_group_name'],
+ if 'azure_edge_security_group_name' not in os.environ:
+ AzureActions.remove_security_group(project_conf['resource_group_name'],
project_conf['edge_security_group_name'])
AzureActions.remove_security_group(project_conf['resource_group_name'],
project_conf['notebook_security_group_name'])
@@ -773,7 +789,8 @@ if __name__ == "__main__":
dlab.fab.append_result("Failed to create storage account.", str(err))
AzureActions.remove_subnet(project_conf['resource_group_name'], project_conf['vpc_name'],
project_conf['private_subnet_name'])
- AzureActions.remove_security_group(project_conf['resource_group_name'],
+ if 'azure_edge_security_group_name' not in os.environ:
+ AzureActions.remove_security_group(project_conf['resource_group_name'],
project_conf['edge_security_group_name'])
AzureActions.remove_security_group(project_conf['resource_group_name'],
project_conf['notebook_security_group_name'])
@@ -807,7 +824,8 @@ if __name__ == "__main__":
dlab.fab.append_result("Failed to create Data Lake Store directory.", str(err))
AzureActions.remove_subnet(project_conf['resource_group_name'], project_conf['vpc_name'],
project_conf['private_subnet_name'])
- AzureActions.remove_security_group(project_conf['resource_group_name'],
+ if 'azure_edge_security_group_name' not in os.environ:
+ AzureActions.remove_security_group(project_conf['resource_group_name'],
project_conf['edge_security_group_name'])
AzureActions.remove_security_group(project_conf['resource_group_name'],
project_conf['notebook_security_group_name'])
@@ -839,6 +857,8 @@ if __name__ == "__main__":
try:
logging.info('[CREATE EDGE INSTANCE]')
print('[CREATE EDGE INSTANCE]')
+ if 'azure_edge_security_group_name' in os.environ:
+ project_conf['edge_security_group_name'] = os.environ['azure_edge_security_group_name']
params = "--instance_name {} --instance_size {} --region {} --vpc_name {} --network_interface_name {} \
--security_group_name {} --subnet_name {} --service_base_name {} --resource_group_name {} \
--dlab_ssh_user_name {} --public_ip_name {} --public_key '''{}''' --primary_disk_size {} \
@@ -863,7 +883,8 @@ if __name__ == "__main__":
print("The instance hasn't been created.")
AzureActions.remove_subnet(project_conf['resource_group_name'], project_conf['vpc_name'],
project_conf['private_subnet_name'])
- AzureActions.remove_security_group(project_conf['resource_group_name'],
+ if 'azure_edge_security_group_name' not in os.environ:
+ AzureActions.remove_security_group(project_conf['resource_group_name'],
project_conf['edge_security_group_name'])
AzureActions.remove_security_group(project_conf['resource_group_name'],
project_conf['notebook_security_group_name'])
diff --git a/infrastructure-provisioning/src/general/scripts/azure/project_terminate.py b/infrastructure-provisioning/src/general/scripts/azure/project_terminate.py
index e93cd35..11ce413 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/project_terminate.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/project_terminate.py
@@ -126,19 +126,18 @@ def terminate_edge_node(resource_group_name, service_base_name, project_tag, sub
print("Removing security groups")
try:
if 'azure_edge_security_group_name' in os.environ:
- AzureActions.remove_security_rules(resource_group_name,
- os.environ['azure_edge_security_group_name'],
+ AzureActions.remove_security_rules(os.environ['azure_edge_security_group_name'],
+ resource_group_name,
'{}-{}-{}-rule'.format(project_conf['service_base_name'],
project_conf['project_name'],
project_conf['endpoint_name']))
- else:
- for sg in AzureMeta.network_client.network_security_groups.list(resource_group_name):
- try:
- if project_tag == sg.tags["project_tag"]:
- AzureActions.remove_security_group(resource_group_name, sg.name)
- print("Security group {} has been terminated".format(sg.name))
- except:
- pass
+ for sg in AzureMeta.network_client.network_security_groups.list(resource_group_name):
+ try:
+ if project_tag == sg.tags["project_tag"]:
+ AzureActions.remove_security_group(resource_group_name, sg.name)
+ print("Security group {} has been terminated".format(sg.name))
+ except:
+ pass
except Exception as err:
dlab.fab.append_result("Failed to remove security groups.", str(err))
sys.exit(1)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org