You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2022/09/01 07:12:04 UTC
[incubator-datalab] 35/36: [DATALAB-1408]: added some creation changes
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-1408
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 14f36c05fd0ce5f2c92589329649f382f41f399c
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Tue Aug 23 16:01:19 2022 +0300
[DATALAB-1408]: added some creation changes
---
.../scripts/azure/dataengine-service_prepare.py | 107 ++++++++++++++++++++-
.../scripts/gcp/dataengine-service_create.py | 2 +-
2 files changed, 105 insertions(+), 4 deletions(-)
diff --git a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py
index 177275f78..7bd369910 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py
@@ -1,5 +1,26 @@
#!/usr/bin/python3
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ******************************************************************************
+
import datalab.actions_lib
import datalab.fab
import datalab.meta_lib
@@ -42,7 +63,7 @@ CONTAINER_NAME = 'hdinsight'
# Blob Storage endpoint suffix.
BLOB_ENDPOINT_SUFFIX = '.blob.core.windows.net'
-def create_cluster_parameters():
+def create_cluster_parameters(LOCATION, CLUSTER_LOGIN_USER_NAME, PASSWORD, SSH_USER_NAME):
# Returns cluster parameters
@@ -92,7 +113,7 @@ def create_cluster_parameters():
storage_profile=StorageProfile(
storageaccounts=[
StorageAccount(
- name=STORAGE_ACCOUNT_NAME + BLOB_ENDPOINT_SUFFIX,
+ name=STORAGE_ACCOUNT_NAME + ".blob.core.windows.net",
key=STORAGE_ACCOUNT_KEY,
container=CONTAINER_NAME.lower(),
is_default=True
@@ -104,4 +125,84 @@ def create_cluster_parameters():
if __name__ == "__main__":
#params = create_cluster_parameters()
- #create_hdinsight_cluster(RESOURCE_GROUP_NAME,CLUSTER_NAME, params)
\ No newline at end of file
+ #create_hdinsight_cluster(RESOURCE_GROUP_NAME,CLUSTER_NAME, params)
+
+ try:
+ AzureMeta = datalab.meta_lib.AzureMeta()
+ AzureActions = datalab.actions_lib.AzureActions()
+ logging.info('Generating infrastructure names and tags')
+ hdinsight_conf = dict()
+ if 'exploratory_name' in os.environ:
+ hdinsight_conf['exploratory_name'] = os.environ['exploratory_name'].replace('_', '-').lower()
+ else:
+ hdinsight_conf['exploratory_name'] = ''
+ if 'computational_name' in os.environ:
+ hdinsight_conf['computational_name'] = os.environ['computational_name'].replace('_', '-').lower()
+ else:
+ hdinsight_conf['computational_name'] = ''
+
+ hdinsight_conf['service_base_name'] = (os.environ['conf_service_base_name'])
+ hdinsight_conf['edge_user_name'] = (os.environ['edge_user_name'])
+ hdinsight_conf['project_name'] = (os.environ['project_name']).replace('_', '-').lower()
+ hdinsight_conf['project_tag'] = hdinsight_conf['project_name']
+ hdinsight_conf['endpoint_name'] = (os.environ['endpoint_name']).replace('_', '-').lower()
+ hdinsight_conf['endpoint_tag'] = hdinsight_conf['endpoint_name']
+ hdinsight_conf['key_name'] = os.environ['conf_key_name']
+ hdinsight_conf['key_path'] = '{0}{1}.pem'.format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
+ hdinsight_conf['zone'] = os.environ['gcp_zone']
+ hdinsight_conf['resource_group_name'] = os.environ['azure_resource_group_name']
+ hdinsight_conf['region'] = os.environ['azure_region']
+ data_engine['vpc_name'] = os.environ['azure_vpc_name']
+ data_engine['private_subnet_name'] = '{}-{}-{}-subnet'.format(data_engine['service_base_name'],
+ data_engine['project_name'],
+ data_engine['endpoint_name'])
+ data_engine['private_subnet_cidr'] = AzureMeta.get_subnet(data_engine['resource_group_name'],
+ data_engine['vpc_name'],
+ data_engine['private_subnet_name']).address_prefix
+ data_engine['cluster_name'] = '{}-{}-{}-des-{}'.format(data_engine['service_base_name'],
+ data_engine['project_name'],
+ data_engine['endpoint_name'],
+ data_engine['computational_name'])
+
+
+
+ hdinsight_conf['subnet'] = '{0}-{1}-{2}-subnet'.format(hdinsight_conf['service_base_name'],
+ hdinsight_conf['project_name'],
+ hdinsight_conf['endpoint_name'])
+ hdinsight_conf['cluster_name'] = '{0}-{1}-{2}-des-{3}'.format(hdinsight_conf['service_base_name'],
+ hdinsight_conf['project_name'],
+ hdinsight_conf['endpoint_name'],
+ hdinsight_conf['computational_name'])
+ hdinsight_conf['cluster_tag'] = '{0}-{1}-{2}-ps'.format(hdinsight_conf['service_base_name'],
+ hdinsight_conf['project_name'],
+ hdinsight_conf['endpoint_name'])
+ hdinsight_conf['bucket_name'] = '{0}-{1}-{2}-bucket'.format(hdinsight_conf['service_base_name'],
+ hdinsight_conf['project_name'],
+ hdinsight_conf['endpoint_name'])
+
+ hdinsight_conf['edge_instance_hostname'] = '{0}-{1}-{2}-edge'.format(hdinsight_conf['service_base_name'],
+ hdinsight_conf['project_name'],
+ hdinsight_conf['endpoint_name'])
+ hdinsight_conf['datalab_ssh_user'] = os.environ['conf_os_user']
+ except Exception as err:
+ datalab.fab.append_result("Failed to generate variables dictionary. Exception:" + str(err))
+ sys.exit(1)
+
+ try:
+ logging.info('[Creating HDInsight Cluster]')
+ params = "--region {0} --bucket {1} --params '{2}'".format(hdinsight_conf['region'],
+ hdinsight_conf['bucket_name'],
+ json.dumps(hdinsight_cluster))
+
+ try:
+ subprocess.run("~/scripts/{}.py {}".format('dataengine-service_create', params), shell=True, check=True)
+ except:
+ traceback.print_exc()
+ raise Exception
+
+ keyfile_name = "/root/keys/{}.pem".format(hdinsight_conf['key_name'])
+ subprocess.run('rm /response/.hdinsight_creating_{}'.format(os.environ['exploratory_name']), shell=True, check=True)
+ except Exception as err:
+ datalab.fab.append_result("Failed to create hdinsight Cluster.", str(err))
+ subprocess.run('rm /response/.hdinsight_creating_{}'.format(os.environ['exploratory_name']), shell=True, check=True)
+ sys.exit(1)
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_create.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_create.py
index 978d2d4fe..2ea506d7b 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_create.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_create.py
@@ -81,4 +81,4 @@ if __name__ == "__main__":
build_dataproc_cluster(args, cluster_name)
send_parser_job(args, cluster_name, cluster_version)
- sys.exit(0)
\ No newline at end of file
+ sys.exit(0)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org