You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by my...@apache.org on 2020/01/29 13:01:07 UTC
[incubator-dlab] 02/02: [DLAB-1487]: [Azure] Shared endpoint bucket
should be created during project creation
This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a commit to branch DLAB-1487
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit f8de1c2bfce89ddcd95cfe4179b3d14165d4c71c
Author: Mykola_Bodnar1 <bo...@gmail.com>
AuthorDate: Wed Jan 29 15:00:34 2020 +0200
[DLAB-1487]: [Azure] Shared endpoint bucket should be created during project creation
---
.../src/general/scripts/azure/project_prepare.py | 39 ++++++++++++
.../src/general/scripts/azure/ssn_prepare.py | 73 +---------------------
2 files changed, 40 insertions(+), 72 deletions(-)
diff --git a/infrastructure-provisioning/src/general/scripts/azure/project_prepare.py b/infrastructure-provisioning/src/general/scripts/azure/project_prepare.py
index df4cfae..c485ff9 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/project_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/project_prepare.py
@@ -97,6 +97,14 @@ if __name__ == "__main__":
"endpoint_tag": project_conf['endpoint_tag'],
os.environ['conf_billing_tag_key']: os.environ['conf_billing_tag_value']}
project_conf['primary_disk_size'] = '32'
+ project_conf['default_endpoint_name'] = os.environ['default_endpoint_name']
+ project_conf['shared_storage_account_name'] = '{0}-{1}-shared-storage'.format(project_conf['service_base_name'],
+ project_conf['default_endpoint_name'])
+ project_conf['shared_container_name'] = '{}-shared-container'.format(project_conf['service_base_name']).lower()
+ project_conf['shared_storage_account_tags'] = {"Name": project_conf['shared_storage_account_name'],
+ "SBN": project_conf['service_base_name'],
+ os.environ['conf_billing_tag_key']: os.environ[
+ 'conf_billing_tag_value']}
# FUSE in case of absence of user's key
try:
@@ -672,6 +680,31 @@ if __name__ == "__main__":
sys.exit(1)
try:
+ logging.info('[CREATE SHARED STORAGE ACCOUNT AND CONTAINER]')
+ print('[CREATE SHARED STORAGE ACCOUNT AND CONTAINER]')
+ params = "--container_name {} --account_tags '{}' --resource_group_name {} --region {}". \
+ format(project_conf['shared_container_name'], json.dumps(project_conf['shared_storage_account_tags']),
+ project_conf['resource_group_name'], project_conf['region'])
+ local("~/scripts/{}.py {}".format('common_create_storage_account', params))
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ append_result("Failed to create storage account.", str(err))
+ AzureActions().remove_subnet(project_conf['resource_group_name'], project_conf['vpc_name'],
+ project_conf['private_subnet_name'])
+ AzureActions().remove_security_group(project_conf['resource_group_name'],
+ project_conf['edge_security_group_name'])
+ AzureActions().remove_security_group(project_conf['resource_group_name'],
+ project_conf['notebook_security_group_name'])
+ AzureActions().remove_security_group(project_conf['resource_group_name'],
+ project_conf['master_security_group_name'])
+ AzureActions().remove_security_group(project_conf['resource_group_name'],
+ project_conf['slave_security_group_name'])
+ for storage_account in AzureMeta().list_storage_accounts(project_conf['resource_group_name']):
+ if project_conf['shared_storage_account_name'] == storage_account.tags["Name"]:
+ AzureActions().remove_storage_account(project_conf['resource_group_name'], storage_account.name)
+ sys.exit(1)
+
+ try:
logging.info('[CREATE STORAGE ACCOUNT AND CONTAINERS]')
print('[CREATE STORAGE ACCOUNT AND CONTAINERS]')
@@ -697,6 +730,8 @@ if __name__ == "__main__":
for storage_account in AzureMeta().list_storage_accounts(project_conf['resource_group_name']):
if project_conf['edge_storage_account_name'] == storage_account.tags["Name"]:
AzureActions().remove_storage_account(project_conf['resource_group_name'], storage_account.name)
+ if project_conf['shared_storage_account_name'] == storage_account.tags["Name"]:
+ AzureActions().remove_storage_account(project_conf['resource_group_name'], storage_account.name)
sys.exit(1)
if os.environ['azure_datalake_enable'] == 'true':
@@ -726,6 +761,8 @@ if __name__ == "__main__":
for storage_account in AzureMeta().list_storage_accounts(project_conf['resource_group_name']):
if project_conf['edge_storage_account_name'] == storage_account.tags["Name"]:
AzureActions().remove_storage_account(project_conf['resource_group_name'], storage_account.name)
+ if project_conf['shared_storage_account_name'] == storage_account.tags["Name"]:
+ AzureActions().remove_storage_account(project_conf['resource_group_name'], storage_account.name)
try:
for datalake in AzureMeta().list_datalakes(project_conf['resource_group_name']):
if project_conf['datalake_store_name'] == datalake.tags["Name"]:
@@ -777,6 +814,8 @@ if __name__ == "__main__":
for storage_account in AzureMeta().list_storage_accounts(project_conf['resource_group_name']):
if project_conf['edge_storage_account_name'] == storage_account.tags["Name"]:
AzureActions().remove_storage_account(project_conf['resource_group_name'], storage_account.name)
+ if project_conf['shared_storage_account_name'] == storage_account.tags["Name"]:
+ AzureActions().remove_storage_account(project_conf['resource_group_name'], storage_account.name)
if os.environ['azure_datalake_enable'] == 'true':
for datalake in AzureMeta().list_datalakes(project_conf['resource_group_name']):
if project_conf['datalake_store_name'] == datalake.tags["Name"]:
diff --git a/infrastructure-provisioning/src/general/scripts/azure/ssn_prepare.py b/infrastructure-provisioning/src/general/scripts/azure/ssn_prepare.py
index e9dff4d..bb6c793 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/ssn_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/ssn_prepare.py
@@ -62,12 +62,6 @@ if __name__ == "__main__":
ssn_conf['vpc_cidr'] = os.environ['conf_vpc_cidr']
ssn_conf['subnet_prefix'] = '20'
ssn_conf['ssn_image_name'] = os.environ['azure_{}_image_name'.format(os.environ['conf_os_family'])]
- ssn_conf['ssn_storage_account_name'] = '{}-ssn-storage'.format(ssn_conf['service_base_name'])
- ssn_conf['ssn_container_name'] = '{}-ssn-container'.format(ssn_conf['service_base_name']).lower()
- ssn_conf['default_endpoint_name'] = os.environ['default_endpoint_name']
- ssn_conf['shared_storage_account_name'] = '{0}-{1}-shared-storage'.format(ssn_conf['service_base_name'],
- ssn_conf['default_endpoint_name'])
- ssn_conf['shared_container_name'] = '{}-shared-container'.format(ssn_conf['service_base_name']).lower()
ssn_conf['datalake_store_name'] = '{}-ssn-datalake'.format(ssn_conf['service_base_name'])
ssn_conf['datalake_shared_directory_name'] = '{}-shared-folder'.format(ssn_conf['service_base_name'])
ssn_conf['instance_name'] = '{}-ssn'.format(ssn_conf['service_base_name'])
@@ -82,12 +76,7 @@ if __name__ == "__main__":
ssn_conf['instance_tags'] = {"Name": ssn_conf['instance_name'],
"SBN": ssn_conf['service_base_name'],
os.environ['conf_billing_tag_key']: os.environ['conf_billing_tag_value']}
- ssn_conf['ssn_storage_account_tags'] = {"Name": ssn_conf['ssn_storage_account_name'],
- "SBN": ssn_conf['service_base_name'],
- os.environ['conf_billing_tag_key']: os.environ['conf_billing_tag_value']}
- ssn_conf['shared_storage_account_tags'] = {"Name": ssn_conf['shared_storage_account_name'],
- "SBN": ssn_conf['service_base_name'],
- os.environ['conf_billing_tag_key']: os.environ['conf_billing_tag_value']}
+
ssn_conf['datalake_store_tags'] = {"Name": ssn_conf['datalake_store_name'],
"SBN": ssn_conf['service_base_name'],
os.environ['conf_billing_tag_key']: os.environ['conf_billing_tag_value']}
@@ -254,58 +243,6 @@ if __name__ == "__main__":
append_result("Failed to create Security group. Exception: " + str(err))
sys.exit(1)
- try:
- logging.info('[CREATE SSN STORAGE ACCOUNT AND CONTAINER]')
- print('[CREATE SSN STORAGE ACCOUNT AND CONTAINER]')
- params = "--container_name {} --account_tags '{}' --resource_group_name {} --region {}". \
- format(ssn_conf['ssn_container_name'], json.dumps(ssn_conf['ssn_storage_account_tags']),
- ssn_conf['resource_group_name'], ssn_conf['region'])
- local("~/scripts/{}.py {}".format('common_create_storage_account', params))
- except Exception as err:
- traceback.print_exc()
- print('Error: {0}'.format(err))
- if 'azure_resource_group_name' not in os.environ:
- AzureActions().remove_resource_group(ssn_conf['service_base_name'], ssn_conf['region'])
- if 'azure_vpc_name' not in os.environ:
- AzureActions().remove_vpc(ssn_conf['resource_group_name'], ssn_conf['vpc_name'])
- if 'azure_subnet_name' not in os.environ:
- AzureActions().remove_subnet(ssn_conf['resource_group_name'], ssn_conf['vpc_name'],
- ssn_conf['subnet_name'])
- if 'azure_security_group_name' not in os.environ:
- AzureActions().remove_security_group(ssn_conf['resource_group_name'], ssn_conf['security_group_name'])
- for storage_account in AzureMeta().list_storage_accounts(ssn_conf['resource_group_name']):
- if ssn_conf['ssn_storage_account_name'] == storage_account.tags["Name"]:
- AzureActions().remove_storage_account(ssn_conf['resource_group_name'], storage_account.name)
- append_result("Failed to create SSN storage account and container. Exception:" + str(err))
- sys.exit(1)
-
- try:
- logging.info('[CREATE SHARED STORAGE ACCOUNT AND CONTAINER]')
- print('[CREATE SHARED STORAGE ACCOUNT AND CONTAINER]')
- params = "--container_name {} --account_tags '{}' --resource_group_name {} --region {}". \
- format(ssn_conf['shared_container_name'], json.dumps(ssn_conf['shared_storage_account_tags']),
- ssn_conf['resource_group_name'], ssn_conf['region'])
- local("~/scripts/{}.py {}".format('common_create_storage_account', params))
- except Exception as err:
- traceback.print_exc()
- print('Error: {0}'.format(err))
- if 'azure_resource_group_name' not in os.environ:
- AzureActions().remove_resource_group(ssn_conf['service_base_name'], ssn_conf['region'])
- if 'azure_vpc_name' not in os.environ:
- AzureActions().remove_vpc(ssn_conf['resource_group_name'], ssn_conf['vpc_name'])
- if 'azure_subnet_name' not in os.environ:
- AzureActions().remove_subnet(ssn_conf['resource_group_name'], ssn_conf['vpc_name'],
- ssn_conf['subnet_name'])
- if 'azure_security_group_name' not in os.environ:
- AzureActions().remove_security_group(ssn_conf['resource_group_name'], ssn_conf['security_group_name'])
- for storage_account in AzureMeta().list_storage_accounts(ssn_conf['resource_group_name']):
- if ssn_conf['ssn_storage_account_name'] == storage_account.tags["Name"]:
- AzureActions().remove_storage_account(ssn_conf['resource_group_name'], storage_account.name)
- if ssn_conf['shared_storage_account_name'] == storage_account.tags["Name"]:
- AzureActions().remove_storage_account(ssn_conf['resource_group_name'], storage_account.name)
- append_result("Failed to create SSN storage account and container. Exception:" + str(err))
- sys.exit(1)
-
if os.environ['azure_datalake_enable'] == 'true':
try:
logging.info('[CREATE DATA LAKE STORE]')
@@ -342,9 +279,6 @@ if __name__ == "__main__":
ssn_conf['subnet_name'])
if 'azure_security_group_name' not in os.environ:
AzureActions().remove_security_group(ssn_conf['resource_group_name'], ssn_conf['security_group_name'])
- for storage_account in AzureMeta().list_storage_accounts(ssn_conf['resource_group_name']):
- if ssn_conf['ssn_storage_account_name'] == storage_account.tags["Name"]:
- AzureActions().remove_storage_account(ssn_conf['resource_group_name'], storage_account.name)
for datalake in AzureMeta().list_datalakes(ssn_conf['resource_group_name']):
if ssn_conf['datalake_store_name'] == datalake.tags["Name"]:
AzureActions().delete_datalake_store(ssn_conf['resource_group_name'], datalake.name)
@@ -384,11 +318,6 @@ if __name__ == "__main__":
ssn_conf['subnet_name'])
if 'azure_security_group_name' not in os.environ:
AzureActions().remove_security_group(ssn_conf['resource_group_name'], ssn_conf['security_group_name'])
- for storage_account in AzureMeta().list_storage_accounts(ssn_conf['resource_group_name']):
- if ssn_conf['ssn_storage_account_name'] == storage_account.tags["Name"]:
- AzureActions().remove_storage_account(ssn_conf['resource_group_name'], storage_account.name)
- if ssn_conf['shared_storage_account_name'] == storage_account.tags["Name"]:
- AzureActions().remove_storage_account(ssn_conf['resource_group_name'], storage_account.name)
for datalake in AzureMeta().list_datalakes(ssn_conf['resource_group_name']):
if ssn_conf['datalake_store_name'] == datalake.tags["Name"]:
AzureActions().delete_datalake_store(ssn_conf['resource_group_name'], datalake.name)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org