You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2022/09/06 15:20:07 UTC
[incubator-datalab] branch DATALAB-1408 updated (345e3399f -> 0f583e86b)
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a change to branch DATALAB-1408
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
from 345e3399f [DATALAB-1408]: added cluster termination
new b72051fd2 [DATALAB-1408]: added password generation
new 0f583e86b [DATALAB-1408]: added cluster status checks
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../src/general/lib/azure/actions_lib.py | 16 ++++++++--
.../src/general/lib/azure/meta_lib.py | 36 ++++++++++++++++++++++
.../scripts/azure/dataengine-service_create.py | 3 +-
3 files changed, 52 insertions(+), 3 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 02/02: [DATALAB-1408]: added cluster status checks
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-1408
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 0f583e86bf9201d503f0654a4cb2374c0f331d94
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Tue Sep 6 18:19:56 2022 +0300
[DATALAB-1408]: added cluster status checks
---
.../src/general/lib/azure/actions_lib.py | 16 ++++++++--
.../src/general/lib/azure/meta_lib.py | 36 ++++++++++++++++++++++
2 files changed, 50 insertions(+), 2 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
index 0c38da8e5..4ea7037dc 100644
--- a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
@@ -1175,7 +1175,13 @@ class AzureActions:
def create_hdinsight_cluster(self, resource_group_name, cluster_name, cluster_parameters):
try:
print('Starting to create HDInsight Spark cluster {}'.format(cluster_name))
- return self.hdinsight_client.clusters.begin_create(resource_group_name, cluster_name, cluster_parameters)
+ result = self.hdinsight_client.clusters.begin_create(resource_group_name, cluster_name, cluster_parameters)
+ cluster = datalab.meta_lib.AzureMeta().get_hdinsight_cluster(resource_group_name, cluster_name)
+ while cluster.properties.cluster_state != 'Running':
+ time.sleep(15)
+ print('The cluster is being provisioned... Please wait')
+ cluster = datalab.meta_lib.AzureMeta().get_hdinsight_cluster(resource_group_name, cluster_name)
+ return result
except Exception as err:
logging.info(
"Unable to create HDInsight Spark cluster: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout))
@@ -1188,7 +1194,13 @@ class AzureActions:
def terminate_hdinsight_cluster(self, resource_group_name, cluster_name):
try:
print('Starting to terminate HDInsight Spark cluster {}'.format(cluster_name))
- return self.hdinsight_client.clusters.begin_delete(resource_group_name, cluster_name)
+ result = self.hdinsight_client.clusters.begin_delete(resource_group_name, cluster_name)
+ cluster_status = datalab.meta_lib.AzureMeta().get_hdinsight_cluster(resource_group_name, cluster_name)
+ while cluster_status:
+ time.sleep(15)
+ print('The cluster is being terminated... Please wait')
+ cluster_status = datalab.meta_lib.AzureMeta().get_hdinsight_cluster(resource_group_name, cluster_name)
+ return result
except Exception as err:
logging.info(
"Unable to terminate HDInsight Spark cluster: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout))
diff --git a/infrastructure-provisioning/src/general/lib/azure/meta_lib.py b/infrastructure-provisioning/src/general/lib/azure/meta_lib.py
index e795c7789..3c557a664 100644
--- a/infrastructure-provisioning/src/general/lib/azure/meta_lib.py
+++ b/infrastructure-provisioning/src/general/lib/azure/meta_lib.py
@@ -29,6 +29,7 @@ from azure.mgmt.datalake.store import DataLakeStoreAccountManagementClient
from azure.datalake.store import core, lib
from azure.identity import ClientSecretCredential
from azure.core.exceptions import ResourceNotFoundError
+from azure.mgmt.hdinsight import HDInsightManagementClient
import logging
import traceback
import sys
@@ -78,6 +79,11 @@ class AzureMeta:
json_dict["subscriptionId"],
base_url=json_dict["resourceManagerEndpointUrl"]
)
+ self.hdinsight_client = HDInsightManagementClient(
+ credential,
+ json_dict["subscriptionId"],
+ base_url=json_dict["resourceManagerEndpointUrl"]
+ )
self.sp_creds = json.loads(open(os.environ['AZURE_AUTH_LOCATION']).read())
self.dl_filesystem_creds = lib.auth(tenant_id=json.dumps(self.sp_creds['tenantId']).replace('"', ''),
client_secret=json.dumps(self.sp_creds['clientSecret']).replace('"', ''),
@@ -637,6 +643,36 @@ class AzureMeta:
file=sys.stdout)}))
traceback.print_exc(file=sys.stdout)
+ def get_hdinsight_cluster(self, resource_group_name, cluster_name):
+ try:
+ result = self.hdinsight_client.clusters.get(resource_group_name, cluster_name)
+ return result
+ except ResourceNotFoundError as err:
+ if err.status_code == 404:
+ return ''
+ except Exception as err:
+ logging.info(
+ "Unable to get hdinsight cluster: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout))
+ append_result(str({"error": "Unable to get hdinsight cluster",
+ "error_message": str(err) + "\n Traceback: " + traceback.print_exc(
+ file=sys.stdout)}))
+ traceback.print_exc(file=sys.stdout)
+
+ def list_hdinsight_clusters(self, resource_group_name):
+ try:
+ result = self.hdinsight_client.clusters.list_by_resource_group(resource_group_name)
+ return result
+ except ResourceNotFoundError as err:
+ if err.status_code == 404:
+ return ''
+ except Exception as err:
+ logging.info(
+ "Unable to list hdinsight clusters: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout))
+ append_result(str({"error": "Unable to list hdinsight clusters",
+ "error_message": str(err) + "\n Traceback: " + traceback.print_exc(
+ file=sys.stdout)}))
+ traceback.print_exc(file=sys.stdout)
+
def get_instance_private_ip_address(tag_name, instance_name):
try:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 01/02: [DATALAB-1408]: added password generation
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-1408
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit b72051fd29679690808f6112f2d03fb1e98eefa4
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Tue Sep 6 17:09:46 2022 +0300
[DATALAB-1408]: added password generation
---
.../src/general/scripts/azure/dataengine-service_create.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_create.py b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_create.py
index 8684a329b..fd0aca6a0 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_create.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_create.py
@@ -24,6 +24,7 @@
import argparse
import json
import sys
+import secrets
from datalab.actions_lib import *
from datalab.meta_lib import *
from datalab.logger import logging
@@ -159,7 +160,7 @@ def create_cluster_parameters(location, tags, cluster_version, cluster_login_use
if __name__ == "__main__":
#parser.print_help()
- password = ''
+ password = secrets.token_urlsafe(20)
params = create_cluster_parameters(args.location, json.loads(args.tags), args.cluster_version, 'datalab-user',
password, args.master_instance_type, args.worker_count,
args.worker_instance_type, args.storage_account_name, args.storage_account_key,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org