You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by op...@apache.org on 2022/09/26 13:31:25 UTC
[incubator-datalab] 01/01: [DATALAB-2909]: added scheduler for azure hdinsight
This is an automated email from the ASF dual-hosted git repository.
opolishchuk pushed a commit to branch DATALAB-2909
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 242ec7df9a30de32bd44566814402088cf8d22aa
Author: Oleksandr Polishchuk <po...@gmail.com>
AuthorDate: Mon Sep 26 16:31:10 2022 +0300
[DATALAB-2909]: added scheduler for azure hdinsight
---
.../src/general/lib/azure/meta_lib.py | 26 ++++++++++++++++++++++
.../general/scripts/azure/common_collect_data.py | 5 +++++
2 files changed, 31 insertions(+)
diff --git a/infrastructure-provisioning/src/general/lib/azure/meta_lib.py b/infrastructure-provisioning/src/general/lib/azure/meta_lib.py
index 4aa99032c..9b1cf1f47 100644
--- a/infrastructure-provisioning/src/general/lib/azure/meta_lib.py
+++ b/infrastructure-provisioning/src/general/lib/azure/meta_lib.py
@@ -700,6 +700,32 @@ class AzureMeta:
traceback.print_exc(file=sys.stdout)
+ def list_hdinsight_statuses(self, resource_group_name, cluster_name_list):
+ data = []
+ for cluster_name in cluster_name_list:
+ cluster_name = cluster_name['id']
+ host = {}
+ try:
+ print(cluster_name)
+ request = self.hdinsight_client.clusters.get(resource_group_name, cluster_name)
+ host['id'] = cluster_name
+ print(request.properties.cluster_state)
+ if request.properties.cluster_state == 'Accepted' or request.properties.cluster_state == 'HdInsightConfiguration' or request.properties.cluster_state == 'ClusterStorageProvisioned' or request.properties.cluster_state == 'ReadyForDeployment':
+ host['status'] = 'creating'
+ elif request.properties.cluster_state == 'DeletePending' or request.properties.cluster_state == 'Deleting':
+ host['status'] = 'terminating'
+ elif request.properties.cluster_state == 'Error' or request.properties.cluster_state == 'TimedOut' or request.properties.cluster_state == 'Unknown':
+ host['status'] = 'failed'
+ elif request.properties.cluster_state == 'Running':
+ host['status'] = 'running'
+ data.append(host)
+ except:
+ host['id'] = cluster_name
+ host['status'] = 'terminated'
+ data.append(host)
+ return data
+
+
def get_instance_private_ip_address(tag_name, instance_name):
try:
resource_group_name = os.environ['azure_resource_group_name']
diff --git a/infrastructure-provisioning/src/general/scripts/azure/common_collect_data.py b/infrastructure-provisioning/src/general/scripts/azure/common_collect_data.py
index 89569200e..71cdf86b1 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/common_collect_data.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_collect_data.py
@@ -51,6 +51,11 @@ if __name__ == "__main__":
statuses['image'] = data_images
except:
logging.error("Images JSON wasn't been provided")
+ try:
+ data_clusters = AzureMeta().list_hdinsight_statuses(args.resource_group_name, data.get('cluster'))
+ statuses['cluster'] = data_clusters
+ except:
+ logging.error("Clusters JSON wasn't been provided")
with open('/root/result.json', 'w') as outfile:
json.dump(statuses, outfile)
except Exception as err:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org