You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by op...@apache.org on 2022/09/02 12:38:13 UTC
[incubator-datalab] 01/01: [DATALAB-3004]: added checking image status to status checker
This is an automated email from the ASF dual-hosted git repository.
opolishchuk pushed a commit to branch DATALAB-3004
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit b9f115d4f0cc1bcc52b390dda87a1a7b491d6f8b
Author: Oleksandr Polishchuk <po...@gmail.com>
AuthorDate: Fri Sep 2 15:37:02 2022 +0300
[DATALAB-3004]: added checking image status to status checker
---
.../src/general/lib/aws/meta_lib.py | 20 ++++++++++++++++++++
.../src/general/lib/gcp/meta_lib.py | 16 ++++++++++++++++
.../src/general/scripts/aws/common_collect_data.py | 5 +++++
.../src/general/scripts/azure/common_collect_data.py | 5 +++++
.../src/general/scripts/gcp/common_collect_data.py | 6 ++++++
5 files changed, 52 insertions(+)
diff --git a/infrastructure-provisioning/src/general/lib/aws/meta_lib.py b/infrastructure-provisioning/src/general/lib/aws/meta_lib.py
index 6b6c5cb7a..a6f918b19 100644
--- a/infrastructure-provisioning/src/general/lib/aws/meta_lib.py
+++ b/infrastructure-provisioning/src/general/lib/aws/meta_lib.py
@@ -754,6 +754,26 @@ def get_list_instance_statuses(instance_ids):
return data
+def get_list_image_statuses(image_ids, data=[]):
+ client = boto3.client('ec2')
+ for k in image_ids:
+ host = {}
+ try:
+ if 'id' in k:
+ response = client.describe_images(ImageIds=[k.get('id')]).get('Reservations')
+ for i in response:
+ img = i.get('Images')
+ for j in img:
+ host['id'] = j.get('ImageId')
+ host['status'] = j.get('State').get('Name')
+ data.append(host)
+ except Exception as err:
+ host['id'] = h.get('id')
+ host['status'] = 'terminated'
+ data.append(host)
+ return data
+
+
def get_list_cluster_statuses(cluster_ids, data=[]):
client = boto3.client('emr')
for i in cluster_ids:
diff --git a/infrastructure-provisioning/src/general/lib/gcp/meta_lib.py b/infrastructure-provisioning/src/general/lib/gcp/meta_lib.py
index be5d17b0c..b8f01e764 100644
--- a/infrastructure-provisioning/src/general/lib/gcp/meta_lib.py
+++ b/infrastructure-provisioning/src/general/lib/gcp/meta_lib.py
@@ -673,6 +673,22 @@ class GCPMeta:
data.append(host)
return data
+ def get_list_image_statuses(self, image_name_list):
+ data = []
+ for image in image_name_list:
+ host = {}
+ try:
+ request = self.service.images().get(project=self.project, image=image)
+ result = request.execute()
+ host['id'] = image
+ host['status'] = result.get('status').lower().replace("terminated", "stopped")
+ data.append(host)
+ except:
+ host['id'] = image
+ host['status'] = 'terminated'
+ data.append(host)
+ return data
+
def get_cluster(self, cluster_name):
try:
request = self.dataproc.projects().regions().clusters().get(projectId=self.project,
diff --git a/infrastructure-provisioning/src/general/scripts/aws/common_collect_data.py b/infrastructure-provisioning/src/general/scripts/aws/common_collect_data.py
index a81e37313..6cbb2fc9a 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/common_collect_data.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_collect_data.py
@@ -50,6 +50,11 @@ if __name__ == "__main__":
statuses['cluster'] = data_clusters
except:
logging.info("Clusters JSON wasn't provided")
+ try:
+ data_images = get_list_image_statuses(data.get('image'))
+ statuses['image'] = data_images
+ except:
+ logging.info("Images JSON wasn't been provided")
with open('/root/result.json', 'w') as outfile:
json.dump(statuses, outfile)
except Exception as err:
diff --git a/infrastructure-provisioning/src/general/scripts/azure/common_collect_data.py b/infrastructure-provisioning/src/general/scripts/azure/common_collect_data.py
index ee8eda738..3394f041a 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/common_collect_data.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_collect_data.py
@@ -46,6 +46,11 @@ if __name__ == "__main__":
statuses['host'] = data_instances
except:
logging.error("Hosts JSON wasn't been provided")
+ try:
+ data_images = AzureMeta().list_images(data.get('image'))
+ statuses['image'] = data_images
+ except:
+ logging.error("Images JSON wasn't been provided")
with open('/root/result.json', 'w') as outfile:
json.dump(statuses, outfile)
except Exception as err:
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_collect_data.py b/infrastructure-provisioning/src/general/scripts/gcp/common_collect_data.py
index 1f8527e48..e54828ac0 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/common_collect_data.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/common_collect_data.py
@@ -62,6 +62,12 @@ if __name__ == "__main__":
statuses['cluster'] = data_clusters
except:
logging.error("Clusters JSON wasn't been provided")
+ try:
+ id_images = get_id_resourses(data.get('image'))
+ data_images = GCPMeta().get_list_image_statuses(id_images)
+ statuses['image'] = data_images
+ except:
+ logging.error("Images JSON wasn't been provided")
with open('/root/result.json', 'w') as outfile:
json.dump(statuses, outfile)
except Exception as err:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org