You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by lf...@apache.org on 2020/02/21 15:50:33 UTC
[incubator-dlab] branch DLAB-1531 updated: DLAB-1531 changed in
code "Name" tag to SBN-tag
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DLAB-1531
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/DLAB-1531 by this push:
new cbf4bf2 DLAB-1531 changed in code "Name" tag to SBN-tag
cbf4bf2 is described below
commit cbf4bf2a37855e1454c115b0bb0686629245f37e
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Fri Feb 21 17:50:16 2020 +0200
DLAB-1531 changed in code "Name" tag to SBN-tag
---
.../src/general/lib/azure/actions_lib.py | 5 +++--
.../scripts/azure/common_create_storage_account.py | 3 ++-
.../scripts/azure/common_download_git_certfile.py | 3 ++-
.../src/general/scripts/azure/edge_configure.py | 17 +++++++++--------
.../src/general/scripts/azure/edge_prepare.py | 11 ++++++-----
.../src/general/scripts/azure/project_prepare.py | 18 +++++++++---------
6 files changed, 31 insertions(+), 26 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
index 9c4c636..22d43fe 100644
--- a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
@@ -1084,14 +1084,15 @@ def configure_local_spark(jars_dir, templates_dir, memory_type='driver'):
spark_jars_paths = None
user_storage_account_tag = os.environ['conf_service_base_name'] + '-' + (os.environ['project_name'].lower().replace('_', '-')).\
replace('_', '-') + '-' + os.environ['endpoint_name'].lower().replace('_', '-') + '-storage'
+ tag_name = os.environ['conf_service_base_name'] + '-tag'
shared_storage_account_tag = '{0}-{1}-shared-storage'.format(os.environ['conf_service_base_name'],
os.environ['endpoint_name'])
for storage_account in meta_lib.AzureMeta().list_storage_accounts(os.environ['azure_resource_group_name']):
- if user_storage_account_tag == storage_account.tags["Name"]:
+ if user_storage_account_tag == storage_account.tags[tag_name]:
user_storage_account_name = storage_account.name
user_storage_account_key = meta_lib.AzureMeta().list_storage_keys(os.environ['azure_resource_group_name'],
user_storage_account_name)[0]
- if shared_storage_account_tag == storage_account.tags["Name"]:
+ if shared_storage_account_tag == storage_account.tags[tag_name]:
shared_storage_account_name = storage_account.name
shared_storage_account_key = meta_lib.AzureMeta().list_storage_keys(os.environ['azure_resource_group_name'],
shared_storage_account_name)[0]
diff --git a/infrastructure-provisioning/src/general/scripts/azure/common_create_storage_account.py b/infrastructure-provisioning/src/general/scripts/azure/common_create_storage_account.py
index d2e3dcc..3ca019e 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/common_create_storage_account.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_create_storage_account.py
@@ -34,6 +34,7 @@ parser.add_argument('--container_name', type=str, default='')
parser.add_argument('--account_tags', type=str, default='{"empty":"string"}')
parser.add_argument('--resource_group_name', type=str, default='')
parser.add_argument('--region', type=str, default='')
+parser.add_argument('--tag_name', type=str, default='')
args = parser.parse_args()
@@ -42,7 +43,7 @@ if __name__ == "__main__":
check_account = False
account_tags = json.loads(args.account_tags)
for storage_account in AzureMeta().list_storage_accounts(args.resource_group_name):
- if account_tags["Name"] == storage_account.tags["Name"]:
+ if account_tags["Name"] == storage_account.tags[tag_name]:
check_account = True
print("REQUESTED STORAGE ACCOUNT {} ALREADY EXISTS".format(storage_account.name))
if not check_account:
diff --git a/infrastructure-provisioning/src/general/scripts/azure/common_download_git_certfile.py b/infrastructure-provisioning/src/general/scripts/azure/common_download_git_certfile.py
index cfe37fc..2cd914e 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/common_download_git_certfile.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_download_git_certfile.py
@@ -35,6 +35,7 @@ parser.add_argument('--os_user', type=str, default='')
args = parser.parse_args()
resource_group_name = os.environ['azure_resource_group_name']
+tag_name = os.environ['conf_service_base_name'] + '-tag'
ssn_storage_account_tag = ('{0}-{1}-{2}-storage'.format(os.environ['conf_service_base_name'], os.environ['project_name'],
os.environ['endpoint_name']))
container_name = ('{}-ssn-container'.format(os.environ['conf_service_base_name'])).lower().replace('_', '-')
@@ -48,7 +49,7 @@ if __name__ == "__main__":
env.host_string = env.user + "@" + env.hosts
for storage_account in AzureMeta().list_storage_accounts(resource_group_name):
- if ssn_storage_account_tag == storage_account.tags["Name"]:
+ if ssn_storage_account_tag == storage_account.tags[tag_name]:
ssn_storage_account_name = storage_account.name
if AzureActions().download_from_container(resource_group_name, ssn_storage_account_name, container_name, gitlab_certfile):
put(gitlab_certfile, gitlab_certfile)
diff --git a/infrastructure-provisioning/src/general/scripts/azure/edge_configure.py b/infrastructure-provisioning/src/general/scripts/azure/edge_configure.py
index fd2f940..f2251b9 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/edge_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/edge_configure.py
@@ -44,6 +44,7 @@ if __name__ == "__main__":
edge_conf['vpc_name'] = os.environ['azure_vpc_name']
edge_conf['region'] = os.environ['azure_region']
edge_conf['subnet_name'] = os.environ['azure_subnet_name']
+ edge_conf['tag_name'] = edge_conf['service_base_name'] + '-tag'
edge_conf['project_name'] = os.environ['project_name'].lower().replace('_', '-')
edge_conf['endpoint_name'] = os.environ['endpoint_name'].lower().replace('_', '-')
edge_conf['user_keyname'] = os.environ['project_name']
@@ -119,7 +120,7 @@ if __name__ == "__main__":
AzureActions().remove_security_group(edge_conf['resource_group_name'],
edge_conf['slave_security_group_name'])
for storage_account in AzureMeta().list_storage_accounts(edge_conf['resource_group_name']):
- if edge_conf['user_storage_account_name'] == storage_account.tags["Name"]:
+ if edge_conf['user_storage_account_name'] == storage_account.tags[edge_conf['tag_name']]:
AzureActions().remove_storage_account(edge_conf['resource_group_name'], storage_account.name)
if os.environ['azure_datalake_enable'] == 'true':
for datalake in AzureMeta().list_datalakes(edge_conf['resource_group_name']):
@@ -160,7 +161,7 @@ if __name__ == "__main__":
AzureActions().remove_security_group(edge_conf['resource_group_name'],
edge_conf['slave_security_group_name'])
for storage_account in AzureMeta().list_storage_accounts(edge_conf['resource_group_name']):
- if edge_conf['user_storage_account_name'] == storage_account.tags["Name"]:
+ if edge_conf['user_storage_account_name'] == storage_account.tags[edge_conf['tag_name']]:
AzureActions().remove_storage_account(edge_conf['resource_group_name'], storage_account.name)
if os.environ['azure_datalake_enable'] == 'true':
for datalake in AzureMeta().list_datalakes(edge_conf['resource_group_name']):
@@ -192,7 +193,7 @@ if __name__ == "__main__":
AzureActions().remove_security_group(edge_conf['resource_group_name'],
edge_conf['slave_security_group_name'])
for storage_account in AzureMeta().list_storage_accounts(edge_conf['resource_group_name']):
- if edge_conf['user_storage_account_name'] == storage_account.tags["Name"]:
+ if edge_conf['user_storage_account_name'] == storage_account.tags[edge_conf['tag_name']]:
AzureActions().remove_storage_account(edge_conf['resource_group_name'], storage_account.name)
if os.environ['azure_datalake_enable'] == 'true':
for datalake in AzureMeta().list_datalakes(edge_conf['resource_group_name']):
@@ -233,7 +234,7 @@ if __name__ == "__main__":
AzureActions().remove_security_group(edge_conf['resource_group_name'],
edge_conf['slave_security_group_name'])
for storage_account in AzureMeta().list_storage_accounts(edge_conf['resource_group_name']):
- if edge_conf['user_storage_account_name'] == storage_account.tags["Name"]:
+ if edge_conf['user_storage_account_name'] == storage_account.tags[edge_conf['tag_name']]:
AzureActions().remove_storage_account(edge_conf['resource_group_name'], storage_account.name)
if os.environ['azure_datalake_enable'] == 'true':
for datalake in AzureMeta().list_datalakes(edge_conf['resource_group_name']):
@@ -268,7 +269,7 @@ if __name__ == "__main__":
AzureActions().remove_security_group(edge_conf['resource_group_name'],
edge_conf['slave_security_group_name'])
for storage_account in AzureMeta().list_storage_accounts(edge_conf['resource_group_name']):
- if edge_conf['user_storage_account_name'] == storage_account.tags["Name"]:
+ if edge_conf['user_storage_account_name'] == storage_account.tags[edge_conf['tag_name']]:
AzureActions().remove_storage_account(edge_conf['resource_group_name'], storage_account.name)
if os.environ['azure_datalake_enable'] == 'true':
for datalake in AzureMeta().list_datalakes(edge_conf['resource_group_name']):
@@ -317,7 +318,7 @@ if __name__ == "__main__":
AzureActions().remove_security_group(edge_conf['resource_group_name'],
edge_conf['slave_security_group_name'])
for storage_account in AzureMeta().list_storage_accounts(edge_conf['resource_group_name']):
- if edge_conf['user_storage_account_name'] == storage_account.tags["Name"]:
+ if edge_conf['user_storage_account_name'] == storage_account.tags[edge_conf['tag_name']]:
AzureActions().remove_storage_account(edge_conf['resource_group_name'], storage_account.name)
if os.environ['azure_datalake_enable'] == 'true':
for datalake in AzureMeta().list_datalakes(edge_conf['resource_group_name']):
@@ -327,9 +328,9 @@ if __name__ == "__main__":
try:
for storage_account in AzureMeta().list_storage_accounts(edge_conf['resource_group_name']):
- if edge_conf['shared_storage_account_name'] == storage_account.tags["Name"]:
+ if edge_conf['shared_storage_account_name'] == storage_account.tags[edge_conf['tag_name']]:
shared_storage_account_name = storage_account.name
- if edge_conf['user_storage_account_name'] == storage_account.tags["Name"]:
+ if edge_conf['user_storage_account_name'] == storage_account.tags[edge_conf['tag_name']]:
user_storage_account_name = storage_account.name
print('[SUMMARY]')
diff --git a/infrastructure-provisioning/src/general/scripts/azure/edge_prepare.py b/infrastructure-provisioning/src/general/scripts/azure/edge_prepare.py
index d4bda85..1d78726 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/edge_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/edge_prepare.py
@@ -49,6 +49,7 @@ if __name__ == "__main__":
edge_conf['user_keyname'] = os.environ['edge_user_name']
edge_conf['vpc_name'] = os.environ['azure_vpc_name']
edge_conf['subnet_name'] = os.environ['azure_subnet_name']
+ edge_conf['tag_name'] = edge_conf['service_base_name'] + '-tag'
edge_conf['private_subnet_name'] = edge_conf['service_base_name'] + '-' + edge_conf['user_name'] + '-subnet'
edge_conf['network_interface_name'] = edge_conf['service_base_name'] + "-" + edge_conf['user_name'] + '-edge-nif'
if os.environ['conf_network_type'] == 'private':
@@ -650,9 +651,9 @@ if __name__ == "__main__":
logging.info('[CREATE STORAGE ACCOUNT AND CONTAINERS]')
print('[CREATE STORAGE ACCOUNT AND CONTAINERS]')
- params = "--container_name {} --account_tags '{}' --resource_group_name {} --region {}". \
+ params = "--container_name {} --account_tags '{}' --resource_group_name {} --region {} --tag_name {}". \
format(edge_conf['edge_container_name'], json.dumps(edge_conf['storage_account_tags']),
- edge_conf['resource_group_name'], edge_conf['region'])
+ edge_conf['resource_group_name'], edge_conf['region'], edge_conf['tag_name'])
try:
local("~/scripts/{}.py {}".format('common_create_storage_account', params))
except:
@@ -670,7 +671,7 @@ if __name__ == "__main__":
AzureActions().remove_security_group(edge_conf['resource_group_name'],
edge_conf['slave_security_group_name'])
for storage_account in AzureMeta().list_storage_accounts(edge_conf['resource_group_name']):
- if edge_conf['edge_storage_account_name'] == storage_account.tags["Name"]:
+ if edge_conf['edge_storage_account_name'] == storage_account.tags[edge_conf['tag_name']]:
AzureActions().remove_storage_account(edge_conf['resource_group_name'], storage_account.name)
sys.exit(1)
@@ -699,7 +700,7 @@ if __name__ == "__main__":
AzureActions().remove_security_group(edge_conf['resource_group_name'],
edge_conf['slave_security_group_name'])
for storage_account in AzureMeta().list_storage_accounts(edge_conf['resource_group_name']):
- if edge_conf['edge_storage_account_name'] == storage_account.tags["Name"]:
+ if edge_conf['edge_storage_account_name'] == storage_account.tags[edge_conf['tag_name']]:
AzureActions().remove_storage_account(edge_conf['resource_group_name'], storage_account.name)
try:
for datalake in AzureMeta().list_datalakes(edge_conf['resource_group_name']):
@@ -750,7 +751,7 @@ if __name__ == "__main__":
AzureActions().remove_security_group(edge_conf['resource_group_name'],
edge_conf['slave_security_group_name'])
for storage_account in AzureMeta().list_storage_accounts(edge_conf['resource_group_name']):
- if edge_conf['edge_storage_account_name'] == storage_account.tags["Name"]:
+ if edge_conf['edge_storage_account_name'] == storage_account.tags[edge_conf['tag_name']]:
AzureActions().remove_storage_account(edge_conf['resource_group_name'], storage_account.name)
if os.environ['azure_datalake_enable'] == 'true':
for datalake in AzureMeta().list_datalakes(edge_conf['resource_group_name']):
diff --git a/infrastructure-provisioning/src/general/scripts/azure/project_prepare.py b/infrastructure-provisioning/src/general/scripts/azure/project_prepare.py
index db1ad75..4ddba9a 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/project_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/project_prepare.py
@@ -681,9 +681,9 @@ if __name__ == "__main__":
try:
logging.info('[CREATE SHARED STORAGE ACCOUNT AND CONTAINER]')
print('[CREATE SHARED STORAGE ACCOUNT AND CONTAINER]')
- params = "--container_name {} --account_tags '{}' --resource_group_name {} --region {}". \
+ params = "--container_name {} --account_tags '{}' --resource_group_name {} --region {} --tag_name {}". \
format(project_conf['shared_container_name'], json.dumps(project_conf['shared_storage_account_tags']),
- project_conf['resource_group_name'], project_conf['region'])
+ project_conf['resource_group_name'], project_conf['region'], project_conf['tag_name'])
local("~/scripts/{}.py {}".format('common_create_storage_account', params))
except Exception as err:
print('Error: {0}'.format(err))
@@ -699,7 +699,7 @@ if __name__ == "__main__":
AzureActions().remove_security_group(project_conf['resource_group_name'],
project_conf['slave_security_group_name'])
for storage_account in AzureMeta().list_storage_accounts(project_conf['resource_group_name']):
- if project_conf['shared_storage_account_name'] == storage_account.tags["Name"]:
+ if project_conf['shared_storage_account_name'] == storage_account.tags[project_conf['tag_name']]:
AzureActions().remove_storage_account(project_conf['resource_group_name'], storage_account.name)
sys.exit(1)
@@ -727,9 +727,9 @@ if __name__ == "__main__":
AzureActions().remove_security_group(project_conf['resource_group_name'],
project_conf['slave_security_group_name'])
for storage_account in AzureMeta().list_storage_accounts(project_conf['resource_group_name']):
- if project_conf['edge_storage_account_name'] == storage_account.tags["Name"]:
+ if project_conf['edge_storage_account_name'] == storage_account.tags[project_conf['tag_name']]:
AzureActions().remove_storage_account(project_conf['resource_group_name'], storage_account.name)
- if project_conf['shared_storage_account_name'] == storage_account.tags["Name"]:
+ if project_conf['shared_storage_account_name'] == storage_account.tags[project_conf['tag_name']]:
AzureActions().remove_storage_account(project_conf['resource_group_name'], storage_account.name)
sys.exit(1)
@@ -758,9 +758,9 @@ if __name__ == "__main__":
AzureActions().remove_security_group(project_conf['resource_group_name'],
project_conf['slave_security_group_name'])
for storage_account in AzureMeta().list_storage_accounts(project_conf['resource_group_name']):
- if project_conf['edge_storage_account_name'] == storage_account.tags["Name"]:
+ if project_conf['edge_storage_account_name'] == storage_account.tags[project_conf['tag_name']]:
AzureActions().remove_storage_account(project_conf['resource_group_name'], storage_account.name)
- if project_conf['shared_storage_account_name'] == storage_account.tags["Name"]:
+ if project_conf['shared_storage_account_name'] == storage_account.tags[project_conf['tag_name']]:
AzureActions().remove_storage_account(project_conf['resource_group_name'], storage_account.name)
try:
for datalake in AzureMeta().list_datalakes(project_conf['resource_group_name']):
@@ -811,9 +811,9 @@ if __name__ == "__main__":
AzureActions().remove_security_group(project_conf['resource_group_name'],
project_conf['slave_security_group_name'])
for storage_account in AzureMeta().list_storage_accounts(project_conf['resource_group_name']):
- if project_conf['edge_storage_account_name'] == storage_account.tags["Name"]:
+ if project_conf['edge_storage_account_name'] == storage_account.tags[project_conf['tag_name']]:
AzureActions().remove_storage_account(project_conf['resource_group_name'], storage_account.name)
- if project_conf['shared_storage_account_name'] == storage_account.tags["Name"]:
+ if project_conf['shared_storage_account_name'] == storage_account.tags[project_conf['tag_name']]:
AzureActions().remove_storage_account(project_conf['resource_group_name'], storage_account.name)
if os.environ['azure_datalake_enable'] == 'true':
for datalake in AzureMeta().list_datalakes(project_conf['resource_group_name']):
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org