You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by lf...@apache.org on 2020/02/21 15:50:33 UTC

[incubator-dlab] branch DLAB-1531 updated: DLAB-1531 changed in code "Name" tag to SBN-tag

This is an automated email from the ASF dual-hosted git repository.

lfrolov pushed a commit to branch DLAB-1531
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git


The following commit(s) were added to refs/heads/DLAB-1531 by this push:
     new cbf4bf2  DLAB-1531 changed in code "Name" tag to SBN-tag
cbf4bf2 is described below

commit cbf4bf2a37855e1454c115b0bb0686629245f37e
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Fri Feb 21 17:50:16 2020 +0200

    DLAB-1531 changed in code "Name" tag to SBN-tag
---
 .../src/general/lib/azure/actions_lib.py               |  5 +++--
 .../scripts/azure/common_create_storage_account.py     |  3 ++-
 .../scripts/azure/common_download_git_certfile.py      |  3 ++-
 .../src/general/scripts/azure/edge_configure.py        | 17 +++++++++--------
 .../src/general/scripts/azure/edge_prepare.py          | 11 ++++++-----
 .../src/general/scripts/azure/project_prepare.py       | 18 +++++++++---------
 6 files changed, 31 insertions(+), 26 deletions(-)

diff --git a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
index 9c4c636..22d43fe 100644
--- a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
@@ -1084,14 +1084,15 @@ def configure_local_spark(jars_dir, templates_dir, memory_type='driver'):
                 spark_jars_paths = None
         user_storage_account_tag = os.environ['conf_service_base_name'] + '-' + (os.environ['project_name'].lower().replace('_', '-')).\
             replace('_', '-') + '-' + os.environ['endpoint_name'].lower().replace('_', '-') + '-storage'
+        tag_name = os.environ['conf_service_base_name'] + '-tag'
         shared_storage_account_tag = '{0}-{1}-shared-storage'.format(os.environ['conf_service_base_name'],
                                                                      os.environ['endpoint_name'])
         for storage_account in meta_lib.AzureMeta().list_storage_accounts(os.environ['azure_resource_group_name']):
-            if user_storage_account_tag == storage_account.tags["Name"]:
+            if user_storage_account_tag == storage_account.tags[tag_name]:
                 user_storage_account_name = storage_account.name
                 user_storage_account_key = meta_lib.AzureMeta().list_storage_keys(os.environ['azure_resource_group_name'],
                                                                                   user_storage_account_name)[0]
-            if shared_storage_account_tag == storage_account.tags["Name"]:
+            if shared_storage_account_tag == storage_account.tags[tag_name]:
                 shared_storage_account_name = storage_account.name
                 shared_storage_account_key = meta_lib.AzureMeta().list_storage_keys(os.environ['azure_resource_group_name'],
                                                                                     shared_storage_account_name)[0]
diff --git a/infrastructure-provisioning/src/general/scripts/azure/common_create_storage_account.py b/infrastructure-provisioning/src/general/scripts/azure/common_create_storage_account.py
index d2e3dcc..3ca019e 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/common_create_storage_account.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_create_storage_account.py
@@ -34,6 +34,7 @@ parser.add_argument('--container_name', type=str, default='')
 parser.add_argument('--account_tags', type=str, default='{"empty":"string"}')
 parser.add_argument('--resource_group_name', type=str, default='')
 parser.add_argument('--region', type=str, default='')
+parser.add_argument('--tag_name', type=str, default='')
 args = parser.parse_args()
 
 
@@ -42,7 +43,7 @@ if __name__ == "__main__":
         check_account = False
         account_tags = json.loads(args.account_tags)
         for storage_account in AzureMeta().list_storage_accounts(args.resource_group_name):
-            if account_tags["Name"] == storage_account.tags["Name"]:
+            if account_tags["Name"] == storage_account.tags[tag_name]:
                 check_account = True
                 print("REQUESTED STORAGE ACCOUNT {} ALREADY EXISTS".format(storage_account.name))
         if not check_account:
diff --git a/infrastructure-provisioning/src/general/scripts/azure/common_download_git_certfile.py b/infrastructure-provisioning/src/general/scripts/azure/common_download_git_certfile.py
index cfe37fc..2cd914e 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/common_download_git_certfile.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_download_git_certfile.py
@@ -35,6 +35,7 @@ parser.add_argument('--os_user', type=str, default='')
 args = parser.parse_args()
 
 resource_group_name = os.environ['azure_resource_group_name']
+tag_name = os.environ['conf_service_base_name'] + '-tag'
 ssn_storage_account_tag = ('{0}-{1}-{2}-storage'.format(os.environ['conf_service_base_name'], os.environ['project_name'],
                                                         os.environ['endpoint_name']))
 container_name = ('{}-ssn-container'.format(os.environ['conf_service_base_name'])).lower().replace('_', '-')
@@ -48,7 +49,7 @@ if __name__ == "__main__":
     env.host_string = env.user + "@" + env.hosts
 
     for storage_account in AzureMeta().list_storage_accounts(resource_group_name):
-        if ssn_storage_account_tag == storage_account.tags["Name"]:
+        if ssn_storage_account_tag == storage_account.tags[tag_name]:
             ssn_storage_account_name = storage_account.name
     if AzureActions().download_from_container(resource_group_name, ssn_storage_account_name, container_name, gitlab_certfile):
         put(gitlab_certfile, gitlab_certfile)
diff --git a/infrastructure-provisioning/src/general/scripts/azure/edge_configure.py b/infrastructure-provisioning/src/general/scripts/azure/edge_configure.py
index fd2f940..f2251b9 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/edge_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/edge_configure.py
@@ -44,6 +44,7 @@ if __name__ == "__main__":
         edge_conf['vpc_name'] = os.environ['azure_vpc_name']
         edge_conf['region'] = os.environ['azure_region']
         edge_conf['subnet_name'] = os.environ['azure_subnet_name']
+        edge_conf['tag_name'] = edge_conf['service_base_name'] + '-tag'
         edge_conf['project_name'] = os.environ['project_name'].lower().replace('_', '-')
         edge_conf['endpoint_name'] = os.environ['endpoint_name'].lower().replace('_', '-')
         edge_conf['user_keyname'] = os.environ['project_name']
@@ -119,7 +120,7 @@ if __name__ == "__main__":
         AzureActions().remove_security_group(edge_conf['resource_group_name'],
                                                  edge_conf['slave_security_group_name'])
         for storage_account in AzureMeta().list_storage_accounts(edge_conf['resource_group_name']):
-            if edge_conf['user_storage_account_name'] == storage_account.tags["Name"]:
+            if edge_conf['user_storage_account_name'] == storage_account.tags[edge_conf['tag_name']]:
                 AzureActions().remove_storage_account(edge_conf['resource_group_name'], storage_account.name)
         if os.environ['azure_datalake_enable'] == 'true':
             for datalake in AzureMeta().list_datalakes(edge_conf['resource_group_name']):
@@ -160,7 +161,7 @@ if __name__ == "__main__":
         AzureActions().remove_security_group(edge_conf['resource_group_name'],
                                                  edge_conf['slave_security_group_name'])
         for storage_account in AzureMeta().list_storage_accounts(edge_conf['resource_group_name']):
-            if edge_conf['user_storage_account_name'] == storage_account.tags["Name"]:
+            if edge_conf['user_storage_account_name'] == storage_account.tags[edge_conf['tag_name']]:
                 AzureActions().remove_storage_account(edge_conf['resource_group_name'], storage_account.name)
         if os.environ['azure_datalake_enable'] == 'true':
             for datalake in AzureMeta().list_datalakes(edge_conf['resource_group_name']):
@@ -192,7 +193,7 @@ if __name__ == "__main__":
         AzureActions().remove_security_group(edge_conf['resource_group_name'],
                                                  edge_conf['slave_security_group_name'])
         for storage_account in AzureMeta().list_storage_accounts(edge_conf['resource_group_name']):
-            if edge_conf['user_storage_account_name'] == storage_account.tags["Name"]:
+            if edge_conf['user_storage_account_name'] == storage_account.tags[edge_conf['tag_name']]:
                 AzureActions().remove_storage_account(edge_conf['resource_group_name'], storage_account.name)
         if os.environ['azure_datalake_enable'] == 'true':
             for datalake in AzureMeta().list_datalakes(edge_conf['resource_group_name']):
@@ -233,7 +234,7 @@ if __name__ == "__main__":
         AzureActions().remove_security_group(edge_conf['resource_group_name'],
                                                  edge_conf['slave_security_group_name'])
         for storage_account in AzureMeta().list_storage_accounts(edge_conf['resource_group_name']):
-            if edge_conf['user_storage_account_name'] == storage_account.tags["Name"]:
+            if edge_conf['user_storage_account_name'] == storage_account.tags[edge_conf['tag_name']]:
                 AzureActions().remove_storage_account(edge_conf['resource_group_name'], storage_account.name)
         if os.environ['azure_datalake_enable'] == 'true':
             for datalake in AzureMeta().list_datalakes(edge_conf['resource_group_name']):
@@ -268,7 +269,7 @@ if __name__ == "__main__":
         AzureActions().remove_security_group(edge_conf['resource_group_name'],
                                                  edge_conf['slave_security_group_name'])
         for storage_account in AzureMeta().list_storage_accounts(edge_conf['resource_group_name']):
-            if edge_conf['user_storage_account_name'] == storage_account.tags["Name"]:
+            if edge_conf['user_storage_account_name'] == storage_account.tags[edge_conf['tag_name']]:
                 AzureActions().remove_storage_account(edge_conf['resource_group_name'], storage_account.name)
         if os.environ['azure_datalake_enable'] == 'true':
             for datalake in AzureMeta().list_datalakes(edge_conf['resource_group_name']):
@@ -317,7 +318,7 @@ if __name__ == "__main__":
         AzureActions().remove_security_group(edge_conf['resource_group_name'],
                                                  edge_conf['slave_security_group_name'])
         for storage_account in AzureMeta().list_storage_accounts(edge_conf['resource_group_name']):
-            if edge_conf['user_storage_account_name'] == storage_account.tags["Name"]:
+            if edge_conf['user_storage_account_name'] == storage_account.tags[edge_conf['tag_name']]:
                 AzureActions().remove_storage_account(edge_conf['resource_group_name'], storage_account.name)
         if os.environ['azure_datalake_enable'] == 'true':
             for datalake in AzureMeta().list_datalakes(edge_conf['resource_group_name']):
@@ -327,9 +328,9 @@ if __name__ == "__main__":
 
     try:
         for storage_account in AzureMeta().list_storage_accounts(edge_conf['resource_group_name']):
-            if edge_conf['shared_storage_account_name'] == storage_account.tags["Name"]:
+            if edge_conf['shared_storage_account_name'] == storage_account.tags[edge_conf['tag_name']]:
                 shared_storage_account_name = storage_account.name
-            if edge_conf['user_storage_account_name'] == storage_account.tags["Name"]:
+            if edge_conf['user_storage_account_name'] == storage_account.tags[edge_conf['tag_name']]:
                 user_storage_account_name = storage_account.name
 
         print('[SUMMARY]')
diff --git a/infrastructure-provisioning/src/general/scripts/azure/edge_prepare.py b/infrastructure-provisioning/src/general/scripts/azure/edge_prepare.py
index d4bda85..1d78726 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/edge_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/edge_prepare.py
@@ -49,6 +49,7 @@ if __name__ == "__main__":
         edge_conf['user_keyname'] = os.environ['edge_user_name']
         edge_conf['vpc_name'] = os.environ['azure_vpc_name']
         edge_conf['subnet_name'] = os.environ['azure_subnet_name']
+        edge_conf['tag_name'] = edge_conf['service_base_name'] + '-tag'
         edge_conf['private_subnet_name'] = edge_conf['service_base_name'] + '-' + edge_conf['user_name'] + '-subnet'
         edge_conf['network_interface_name'] = edge_conf['service_base_name'] + "-" + edge_conf['user_name'] + '-edge-nif'
         if os.environ['conf_network_type'] == 'private':
@@ -650,9 +651,9 @@ if __name__ == "__main__":
         logging.info('[CREATE STORAGE ACCOUNT AND CONTAINERS]')
         print('[CREATE STORAGE ACCOUNT AND CONTAINERS]')
 
-        params = "--container_name {} --account_tags '{}' --resource_group_name {} --region {}". \
+        params = "--container_name {} --account_tags '{}' --resource_group_name {} --region {} --tag_name {}". \
             format(edge_conf['edge_container_name'], json.dumps(edge_conf['storage_account_tags']),
-                   edge_conf['resource_group_name'], edge_conf['region'])
+                   edge_conf['resource_group_name'], edge_conf['region'], edge_conf['tag_name'])
         try:
             local("~/scripts/{}.py {}".format('common_create_storage_account', params))
         except:
@@ -670,7 +671,7 @@ if __name__ == "__main__":
         AzureActions().remove_security_group(edge_conf['resource_group_name'],
                                                  edge_conf['slave_security_group_name'])
         for storage_account in AzureMeta().list_storage_accounts(edge_conf['resource_group_name']):
-            if edge_conf['edge_storage_account_name'] == storage_account.tags["Name"]:
+            if edge_conf['edge_storage_account_name'] == storage_account.tags[edge_conf['tag_name']]:
                 AzureActions().remove_storage_account(edge_conf['resource_group_name'], storage_account.name)
         sys.exit(1)
 
@@ -699,7 +700,7 @@ if __name__ == "__main__":
             AzureActions().remove_security_group(edge_conf['resource_group_name'],
                                                      edge_conf['slave_security_group_name'])
             for storage_account in AzureMeta().list_storage_accounts(edge_conf['resource_group_name']):
-                if edge_conf['edge_storage_account_name'] == storage_account.tags["Name"]:
+                if edge_conf['edge_storage_account_name'] == storage_account.tags[edge_conf['tag_name']]:
                     AzureActions().remove_storage_account(edge_conf['resource_group_name'], storage_account.name)
             try:
                 for datalake in AzureMeta().list_datalakes(edge_conf['resource_group_name']):
@@ -750,7 +751,7 @@ if __name__ == "__main__":
         AzureActions().remove_security_group(edge_conf['resource_group_name'],
                                                  edge_conf['slave_security_group_name'])
         for storage_account in AzureMeta().list_storage_accounts(edge_conf['resource_group_name']):
-            if edge_conf['edge_storage_account_name'] == storage_account.tags["Name"]:
+            if edge_conf['edge_storage_account_name'] == storage_account.tags[edge_conf['tag_name']]:
                 AzureActions().remove_storage_account(edge_conf['resource_group_name'], storage_account.name)
         if os.environ['azure_datalake_enable'] == 'true':
             for datalake in AzureMeta().list_datalakes(edge_conf['resource_group_name']):
diff --git a/infrastructure-provisioning/src/general/scripts/azure/project_prepare.py b/infrastructure-provisioning/src/general/scripts/azure/project_prepare.py
index db1ad75..4ddba9a 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/project_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/project_prepare.py
@@ -681,9 +681,9 @@ if __name__ == "__main__":
     try:
         logging.info('[CREATE SHARED STORAGE ACCOUNT AND CONTAINER]')
         print('[CREATE SHARED STORAGE ACCOUNT AND CONTAINER]')
-        params = "--container_name {} --account_tags '{}' --resource_group_name {} --region {}". \
+        params = "--container_name {} --account_tags '{}' --resource_group_name {} --region {} --tag_name {}". \
             format(project_conf['shared_container_name'], json.dumps(project_conf['shared_storage_account_tags']),
-                   project_conf['resource_group_name'], project_conf['region'])
+                   project_conf['resource_group_name'], project_conf['region'], project_conf['tag_name'])
         local("~/scripts/{}.py {}".format('common_create_storage_account', params))
     except Exception as err:
         print('Error: {0}'.format(err))
@@ -699,7 +699,7 @@ if __name__ == "__main__":
         AzureActions().remove_security_group(project_conf['resource_group_name'],
                                              project_conf['slave_security_group_name'])
         for storage_account in AzureMeta().list_storage_accounts(project_conf['resource_group_name']):
-            if project_conf['shared_storage_account_name'] == storage_account.tags["Name"]:
+            if project_conf['shared_storage_account_name'] == storage_account.tags[project_conf['tag_name']]:
                 AzureActions().remove_storage_account(project_conf['resource_group_name'], storage_account.name)
         sys.exit(1)
 
@@ -727,9 +727,9 @@ if __name__ == "__main__":
         AzureActions().remove_security_group(project_conf['resource_group_name'],
                                                  project_conf['slave_security_group_name'])
         for storage_account in AzureMeta().list_storage_accounts(project_conf['resource_group_name']):
-            if project_conf['edge_storage_account_name'] == storage_account.tags["Name"]:
+            if project_conf['edge_storage_account_name'] == storage_account.tags[project_conf['tag_name']]:
                 AzureActions().remove_storage_account(project_conf['resource_group_name'], storage_account.name)
-            if project_conf['shared_storage_account_name'] == storage_account.tags["Name"]:
+            if project_conf['shared_storage_account_name'] == storage_account.tags[project_conf['tag_name']]:
                 AzureActions().remove_storage_account(project_conf['resource_group_name'], storage_account.name)
         sys.exit(1)
 
@@ -758,9 +758,9 @@ if __name__ == "__main__":
             AzureActions().remove_security_group(project_conf['resource_group_name'],
                                                      project_conf['slave_security_group_name'])
             for storage_account in AzureMeta().list_storage_accounts(project_conf['resource_group_name']):
-                if project_conf['edge_storage_account_name'] == storage_account.tags["Name"]:
+                if project_conf['edge_storage_account_name'] == storage_account.tags[project_conf['tag_name']]:
                     AzureActions().remove_storage_account(project_conf['resource_group_name'], storage_account.name)
-                if project_conf['shared_storage_account_name'] == storage_account.tags["Name"]:
+                if project_conf['shared_storage_account_name'] == storage_account.tags[project_conf['tag_name']]:
                     AzureActions().remove_storage_account(project_conf['resource_group_name'], storage_account.name)
             try:
                 for datalake in AzureMeta().list_datalakes(project_conf['resource_group_name']):
@@ -811,9 +811,9 @@ if __name__ == "__main__":
         AzureActions().remove_security_group(project_conf['resource_group_name'],
                                                  project_conf['slave_security_group_name'])
         for storage_account in AzureMeta().list_storage_accounts(project_conf['resource_group_name']):
-            if project_conf['edge_storage_account_name'] == storage_account.tags["Name"]:
+            if project_conf['edge_storage_account_name'] == storage_account.tags[project_conf['tag_name']]:
                 AzureActions().remove_storage_account(project_conf['resource_group_name'], storage_account.name)
-            if project_conf['shared_storage_account_name'] == storage_account.tags["Name"]:
+            if project_conf['shared_storage_account_name'] == storage_account.tags[project_conf['tag_name']]:
                 AzureActions().remove_storage_account(project_conf['resource_group_name'], storage_account.name)
         if os.environ['azure_datalake_enable'] == 'true':
             for datalake in AzureMeta().list_datalakes(project_conf['resource_group_name']):


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org