You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2022/09/01 13:15:24 UTC
[incubator-datalab] 01/02: [DATALAB-1408]: added hdinsight storage creation
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-1408
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit d1f90ba76778f427756b8363e5d8ea4775eeb0cc
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Thu Sep 1 16:13:25 2022 +0300
[DATALAB-1408]: added hdinsight storage creation
---
.../scripts/azure/dataengine-service_prepare.py | 39 ++++++++++++++++++++++
1 file changed, 39 insertions(+)
diff --git a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py
index e3724532d..50049fc70 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py
@@ -61,6 +61,7 @@ if __name__ == "__main__":
hdinsight_conf['key_path'] = '{0}{1}.pem'.format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
hdinsight_conf['resource_group_name'] = os.environ['azure_resource_group_name']
hdinsight_conf['region'] = os.environ['azure_region']
+ hdinsight_conf['tag_name'] = hdinsight_conf['service_base_name'] + '-tag'
hdinsight_conf['cluster_name'] = '{}-{}-{}-des-{}'.format(hdinsight_conf['service_base_name'],
hdinsight_conf['project_name'],
hdinsight_conf['endpoint_name'],
@@ -79,12 +80,50 @@ if __name__ == "__main__":
hdinsight_conf['release_label'] = os.environ['hdinsight_version']
key = RSA.importKey(open(hdinsight_conf['key_path'], 'rb').read())
ssh_admin_pubkey = key.publickey().exportKey("OpenSSH").decode('UTF-8')
+ hdinsight_conf['container_name'] = ('{0}-{1}-{2}-{3}-bucket'.format(hdinsight_conf['service_base_name'],
+ hdinsight_conf['project_name'],
+ hdinsight_conf['endpoint_name'],
+ hdinsight_conf['cluster_name'])).lower()
+ hdinsight_conf['storage_account_name_tag'] = ('{0}-{1}-{2}-{3}-bucket'.format(hdinsight_conf['service_base_name'],
+ hdinsight_conf['project_name'],
+ hdinsight_conf['endpoint_name'],
+ hdinsight_conf['cluster_name']
+ )).lower()
+ hdinsight_conf['storage_account_tags'] = {"Name": hdinsight_conf['storage_account_name_tag'],
+ "SBN": hdinsight_conf['service_base_name'],
+ "project_tag": hdinsight_conf['project_name'],
+ "endpoint_tag": hdinsight_conf['endpoint_name'],
+ os.environ['conf_billing_tag_key']: os.environ['conf_billing_tag_value'],
+ hdinsight_conf['tag_name']: hdinsight_conf['storage_account_name_tag']}
except Exception as err:
datalab.fab.append_result("Failed to generate variables dictionary. Exception:" + str(err))
sys.exit(1)
+ try:
+ logging.info('[CREATE STORAGE ACCOUNT AND CONTAINERS]')
+
+ params = "--container_name {} --account_tags '{}' --resource_group_name {} --region {}". \
+ format(hdinsight_conf['container_name'], json.dumps(hdinsight_conf['storage_account_tags']),
+ hdinsight_conf['resource_group_name'], hdinsight_conf['region'])
+ try:
+ subprocess.run("~/scripts/{}.py {}".format('common_create_storage_account', params), shell=True, check=True)
+ except:
+ traceback.print_exc()
+ raise Exception
+ except Exception as err:
+ datalab.fab.append_result("Failed to create storage account.", str(err))
+ for storage_account in AzureMeta.list_storage_accounts(hdinsight_conf['resource_group_name']):
+ if hdinsight_conf['storage_account_name_tag'] == storage_account.tags["Name"]:
+ AzureActions.remove_storage_account(hdinsight_conf['resource_group_name'], storage_account.name)
+ sys.exit(1)
+
try:
logging.info('[Creating HDInsight Cluster]')
+ for storage_account in AzureMeta.list_storage_accounts(hdinsight_conf['resource_group_name']):
+ if hdinsight_conf['storage_account_name_tag'] == storage_account.tags["Name"]:
+ hdinsight_conf['storage_account_name'] = storage_account.name
+ hdinsight_conf['storage_account_key'] = AzureMeta.list_storage_keys(
+ hdinsight_conf['resource_group_name'], hdinsight_conf['storage_account_name'])[0]
params = "--resource_group_name {} --cluster_name {} " \
"--cluster_version {} --location {} " \
"--master_instance_type {} --worker_instance_type {} " \
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org