You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by op...@apache.org on 2022/09/02 12:38:12 UTC
[incubator-datalab] branch DATALAB-3004 created (now b9f115d4f)
This is an automated email from the ASF dual-hosted git repository.
opolishchuk pushed a change to branch DATALAB-3004
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
at b9f115d4f [DATALAB-3004]: added checking image status to status checker
This branch includes the following new commits:
new b9f115d4f [DATALAB-3004]: added checking image status to status checker
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 01/01: [DATALAB-3004]: added checking image status to status checker
Posted by op...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
opolishchuk pushed a commit to branch DATALAB-3004
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit b9f115d4f0cc1bcc52b390dda87a1a7b491d6f8b
Author: Oleksandr Polishchuk <po...@gmail.com>
AuthorDate: Fri Sep 2 15:37:02 2022 +0300
[DATALAB-3004]: added checking image status to status checker
---
.../src/general/lib/aws/meta_lib.py | 20 ++++++++++++++++++++
.../src/general/lib/gcp/meta_lib.py | 16 ++++++++++++++++
.../src/general/scripts/aws/common_collect_data.py | 5 +++++
.../src/general/scripts/azure/common_collect_data.py | 5 +++++
.../src/general/scripts/gcp/common_collect_data.py | 6 ++++++
5 files changed, 52 insertions(+)
diff --git a/infrastructure-provisioning/src/general/lib/aws/meta_lib.py b/infrastructure-provisioning/src/general/lib/aws/meta_lib.py
index 6b6c5cb7a..a6f918b19 100644
--- a/infrastructure-provisioning/src/general/lib/aws/meta_lib.py
+++ b/infrastructure-provisioning/src/general/lib/aws/meta_lib.py
@@ -754,6 +754,26 @@ def get_list_instance_statuses(instance_ids):
return data
+def get_list_image_statuses(image_ids, data=[]):
+ client = boto3.client('ec2')
+ for k in image_ids:
+ host = {}
+ try:
+ if 'id' in k:
+ response = client.describe_images(ImageIds=[k.get('id')]).get('Reservations')
+ for i in response:
+ img = i.get('Images')
+ for j in img:
+ host['id'] = j.get('ImageId')
+ host['status'] = j.get('State').get('Name')
+ data.append(host)
+ except Exception as err:
+ host['id'] = h.get('id')
+ host['status'] = 'terminated'
+ data.append(host)
+ return data
+
+
def get_list_cluster_statuses(cluster_ids, data=[]):
client = boto3.client('emr')
for i in cluster_ids:
diff --git a/infrastructure-provisioning/src/general/lib/gcp/meta_lib.py b/infrastructure-provisioning/src/general/lib/gcp/meta_lib.py
index be5d17b0c..b8f01e764 100644
--- a/infrastructure-provisioning/src/general/lib/gcp/meta_lib.py
+++ b/infrastructure-provisioning/src/general/lib/gcp/meta_lib.py
@@ -673,6 +673,22 @@ class GCPMeta:
data.append(host)
return data
+ def get_list_image_statuses(self, image_name_list):
+ data = []
+ for image in image_name_list:
+ host = {}
+ try:
+ request = self.service.images().get(project=self.project, image=image)
+ result = request.execute()
+ host['id'] = image
+ host['status'] = result.get('status').lower().replace("terminated", "stopped")
+ data.append(host)
+ except:
+ host['id'] = image
+ host['status'] = 'terminated'
+ data.append(host)
+ return data
+
def get_cluster(self, cluster_name):
try:
request = self.dataproc.projects().regions().clusters().get(projectId=self.project,
diff --git a/infrastructure-provisioning/src/general/scripts/aws/common_collect_data.py b/infrastructure-provisioning/src/general/scripts/aws/common_collect_data.py
index a81e37313..6cbb2fc9a 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/common_collect_data.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_collect_data.py
@@ -50,6 +50,11 @@ if __name__ == "__main__":
statuses['cluster'] = data_clusters
except:
logging.info("Clusters JSON wasn't provided")
+ try:
+ data_images = get_list_image_statuses(data.get('image'))
+ statuses['image'] = data_images
+ except:
+ logging.info("Images JSON wasn't been provided")
with open('/root/result.json', 'w') as outfile:
json.dump(statuses, outfile)
except Exception as err:
diff --git a/infrastructure-provisioning/src/general/scripts/azure/common_collect_data.py b/infrastructure-provisioning/src/general/scripts/azure/common_collect_data.py
index ee8eda738..3394f041a 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/common_collect_data.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_collect_data.py
@@ -46,6 +46,11 @@ if __name__ == "__main__":
statuses['host'] = data_instances
except:
logging.error("Hosts JSON wasn't been provided")
+ try:
+ data_images = AzureMeta().list_images(data.get('image'))
+ statuses['image'] = data_images
+ except:
+ logging.error("Images JSON wasn't been provided")
with open('/root/result.json', 'w') as outfile:
json.dump(statuses, outfile)
except Exception as err:
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_collect_data.py b/infrastructure-provisioning/src/general/scripts/gcp/common_collect_data.py
index 1f8527e48..e54828ac0 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/common_collect_data.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/common_collect_data.py
@@ -62,6 +62,12 @@ if __name__ == "__main__":
statuses['cluster'] = data_clusters
except:
logging.error("Clusters JSON wasn't been provided")
+ try:
+ id_images = get_id_resourses(data.get('image'))
+ data_images = GCPMeta().get_list_image_statuses(id_images)
+ statuses['image'] = data_images
+ except:
+ logging.error("Images JSON wasn't been provided")
with open('/root/result.json', 'w') as outfile:
json.dump(statuses, outfile)
except Exception as err:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org