You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2022/09/06 13:09:21 UTC
[incubator-datalab] 03/06: [DATALAB-1408]: changed container type
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-1408
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit f153bbeb365ae70bead702f44b0aa153b41c85c6
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Tue Sep 6 14:30:14 2022 +0300
[DATALAB-1408]: changed container type
---
.../src/general/lib/azure/actions_lib.py | 25 +++++++++++-----------
.../scripts/azure/common_create_storage_account.py | 7 +++---
.../scripts/azure/dataengine-service_prepare.py | 16 ++++++--------
3 files changed, 23 insertions(+), 25 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
index 563095bf4..db9088aa6 100644
--- a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
@@ -450,25 +450,23 @@ class AzureActions:
file=sys.stdout)}))
traceback.print_exc(file=sys.stdout)
- def create_storage_account(self, resource_group_name, account_name, region, tags):
+ def create_storage_account(self, resource_group_name, account_name, region, tags, kind='BlobStorage'):
try:
ssn_network_id = datalab.meta_lib.AzureMeta().get_subnet(resource_group_name,
vpc_name=os.environ['azure_vpc_name'],
- subnet_name=os.environ['azure_subnet_name']
- ).id
+ subnet_name=os.environ['azure_subnet_name']).id
edge_network_id = datalab.meta_lib.AzureMeta().get_subnet(resource_group_name,
- vpc_name=os.environ['azure_vpc_name'],
- subnet_name='{}-{}-{}-subnet'.format(
- os.environ['conf_service_base_name'],
- (os.environ['project_name']),
- (os.environ['endpoint_name']))
- ).id
+ vpc_name=os.environ['azure_vpc_name'],
+ subnet_name='{}-{}-{}-subnet'.format(
+ os.environ['conf_service_base_name'],
+ (os.environ['project_name']),
+ (os.environ['endpoint_name']))).id
result = self.storage_client.storage_accounts.begin_create(
resource_group_name,
account_name,
{
"sku": {"name": "Standard_LRS"},
- "kind": "BlobStorage",
+ "kind": kind,
"location": region,
"tags": tags,
"access_tier": "Hot",
@@ -520,13 +518,14 @@ class AzureActions:
file=sys.stdout)}))
traceback.print_exc(file=sys.stdout)
- def create_blob_container(self, resource_group_name, account_name, container_name):
+ def create_blob_container(self, account_name, container_name):
try:
- block_blob_service = BlobServiceClient(account_url="https://" + account_name + ".blob.core.windows.net/", credential=self.credential)
+ block_blob_service = BlobServiceClient(account_url="https://" + account_name + ".blob.core.windows.net/",
+ credential=self.credential)
result = block_blob_service.create_container(
container_name,
{
- "public_access": "Off"
+ "public_access": "Off"
}
)
return result
diff --git a/infrastructure-provisioning/src/general/scripts/azure/common_create_storage_account.py b/infrastructure-provisioning/src/general/scripts/azure/common_create_storage_account.py
index 17676b084..8c94592da 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/common_create_storage_account.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_create_storage_account.py
@@ -34,6 +34,7 @@ parser.add_argument('--container_name', type=str, default='')
parser.add_argument('--account_tags', type=str, default='{"empty":"string"}')
parser.add_argument('--resource_group_name', type=str, default='')
parser.add_argument('--region', type=str, default='')
+parser.add_argument('--storage_account_kind', type=str, default='BlobStorage')
args = parser.parse_args()
if __name__ == "__main__":
@@ -50,9 +51,9 @@ if __name__ == "__main__":
if check.name_available:
logging.info("Creating storage account {}.".format(account_name))
storage_account = AzureActions().create_storage_account(args.resource_group_name, account_name,
- args.region, account_tags)
- blob_container = AzureActions().create_blob_container(args.resource_group_name, account_name,
- args.container_name)
+ args.region, account_tags,
+ args.storage_account_kind)
+ blob_container = AzureActions().create_blob_container(account_name, args.container_name)
logging.info("STORAGE ACCOUNT {} has been created".format(account_name))
logging.info("CONTAINER {} has been created".format(args.container_name))
else:
diff --git a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py
index 5725c2c0c..49f636566 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py
@@ -77,14 +77,8 @@ if __name__ == "__main__":
hdinsight_conf['release_label'] = os.environ['hdinsight_version']
key = RSA.importKey(open(hdinsight_conf['key_path'], 'rb').read())
ssh_admin_pubkey = key.publickey().exportKey("OpenSSH").decode('UTF-8')
- hdinsight_conf['container_name'] = ('{0}-bucket'.format(hdinsight_conf['service_base_name'],
- hdinsight_conf['project_name'],
- hdinsight_conf['endpoint_name'],
- hdinsight_conf['cluster_name'])).lower()
- hdinsight_conf['storage_account_name_tag'] = ('{0}-bucket'.format(hdinsight_conf['service_base_name'],
- hdinsight_conf['project_name'],
- hdinsight_conf['endpoint_name'],
- hdinsight_conf['cluster_name'])).lower()
+ hdinsight_conf['container_name'] = ('{}-bucket'.format(hdinsight_conf['cluster_name'])).lower()
+ hdinsight_conf['storage_account_name_tag'] = ('{}-bucket'.format(hdinsight_conf['cluster_name'])).lower()
hdinsight_conf['storage_account_tags'] = {"Name": hdinsight_conf['storage_account_name_tag'],
"SBN": hdinsight_conf['service_base_name'],
"project_tag": hdinsight_conf['project_name'],
@@ -112,7 +106,8 @@ if __name__ == "__main__":
try:
logging.info('[CREATE STORAGE ACCOUNT AND CONTAINERS]')
- params = "--container_name {} --account_tags '{}' --resource_group_name {} --region {}". \
+ params = "--container_name {} --account_tags '{}' --resource_group_name {} --region {} " \
+ "--storage_account_kind StorageV2". \
format(hdinsight_conf['container_name'], json.dumps(hdinsight_conf['storage_account_tags']),
hdinsight_conf['resource_group_name'], hdinsight_conf['region'])
try:
@@ -156,5 +151,8 @@ if __name__ == "__main__":
except Exception as err:
datalab.fab.append_result("Failed to create hdinsight Cluster.", str(err))
+ for storage_account in AzureMeta.list_storage_accounts(hdinsight_conf['resource_group_name']):
+ if hdinsight_conf['storage_account_name_tag'] == storage_account.tags["Name"]:
+ AzureActions.remove_storage_account(hdinsight_conf['resource_group_name'], storage_account.name)
#subprocess.run('rm /response/.hdinsight_creating_{}'.format(os.environ['exploratory_name']), shell=True, check=True)
sys.exit(1)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org