You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by op...@apache.org on 2022/09/26 13:31:24 UTC
[incubator-datalab] branch DATALAB-2909 created (now 242ec7df9)
This is an automated email from the ASF dual-hosted git repository.
opolishchuk pushed a change to branch DATALAB-2909
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
at 242ec7df9 [DATALAB-2909]: added scheduler for azure hdinsight
This branch includes the following new commits:
new 242ec7df9 [DATALAB-2909]: added scheduler for azure hdinsight
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 01/01: [DATALAB-2909]: added scheduler for azure hdinsight
Posted by op...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
opolishchuk pushed a commit to branch DATALAB-2909
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 242ec7df9a30de32bd44566814402088cf8d22aa
Author: Oleksandr Polishchuk <po...@gmail.com>
AuthorDate: Mon Sep 26 16:31:10 2022 +0300
[DATALAB-2909]: added scheduler for azure hdinsight
---
.../src/general/lib/azure/meta_lib.py | 26 ++++++++++++++++++++++
.../general/scripts/azure/common_collect_data.py | 5 +++++
2 files changed, 31 insertions(+)
diff --git a/infrastructure-provisioning/src/general/lib/azure/meta_lib.py b/infrastructure-provisioning/src/general/lib/azure/meta_lib.py
index 4aa99032c..9b1cf1f47 100644
--- a/infrastructure-provisioning/src/general/lib/azure/meta_lib.py
+++ b/infrastructure-provisioning/src/general/lib/azure/meta_lib.py
@@ -700,6 +700,32 @@ class AzureMeta:
traceback.print_exc(file=sys.stdout)
+ def list_hdinsight_statuses(self, resource_group_name, cluster_name_list):
+ data = []
+ for cluster_name in cluster_name_list:
+ cluster_name = cluster_name['id']
+ host = {}
+ try:
+ print(cluster_name)
+ request = self.hdinsight_client.clusters.get(resource_group_name, cluster_name)
+ host['id'] = cluster_name
+ print(request.properties.cluster_state)
+ if request.properties.cluster_state == 'Accepted' or request.properties.cluster_state == 'HdInsightConfiguration' or request.properties.cluster_state == 'ClusterStorageProvisioned' or request.properties.cluster_state == 'ReadyForDeployment':
+ host['status'] = 'creating'
+ elif request.properties.cluster_state == 'DeletePending' or request.properties.cluster_state == 'Deleting':
+ host['status'] = 'terminating'
+ elif request.properties.cluster_state == 'Error' or request.properties.cluster_state == 'TimedOut' or request.properties.cluster_state == 'Unknown':
+ host['status'] = 'failed'
+ elif request.properties.cluster_state == 'Running':
+ host['status'] = 'running'
+ data.append(host)
+ except:
+ host['id'] = cluster_name
+ host['status'] = 'terminated'
+ data.append(host)
+ return data
+
+
def get_instance_private_ip_address(tag_name, instance_name):
try:
resource_group_name = os.environ['azure_resource_group_name']
diff --git a/infrastructure-provisioning/src/general/scripts/azure/common_collect_data.py b/infrastructure-provisioning/src/general/scripts/azure/common_collect_data.py
index 89569200e..71cdf86b1 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/common_collect_data.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_collect_data.py
@@ -51,6 +51,11 @@ if __name__ == "__main__":
statuses['image'] = data_images
except:
logging.error("Images JSON wasn't been provided")
+ try:
+ data_clusters = AzureMeta().list_hdinsight_statuses(args.resource_group_name, data.get('cluster'))
+ statuses['cluster'] = data_clusters
+ except:
+ logging.error("Clusters JSON wasn't been provided")
with open('/root/result.json', 'w') as outfile:
json.dump(statuses, outfile)
except Exception as err:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org