You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2022/09/08 10:16:58 UTC
[incubator-datalab] branch DATALAB-1408 updated (2bddbd170 -> a9d0782d4)
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a change to branch DATALAB-1408
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
from 2bddbd170 [DATALAB-1408]: increased check interval during cluster creation and termination
new d9205c9fe [DATALAB-1408]: fixed cluster tags
new d3fa76e4e [DATALAB-3020]: added sbn tag checks during azure project termination
new a9d0782d4 [DATALAB-1408]: fixed azure image list
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../scripts/azure/dataengine-service_prepare.py | 8 +++----
.../src/general/scripts/azure/project_terminate.py | 27 ++++++++++++++--------
.../src/ssn/scripts/docker_build.py | 5 ++--
3 files changed, 23 insertions(+), 17 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 02/03: [DATALAB-3020]: added sbn tag checks during azure project termination
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-1408
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit d3fa76e4e6bbfdb33bd7dbc3b414721ee5e7a9fc
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Thu Sep 8 13:16:06 2022 +0300
[DATALAB-3020]: added sbn tag checks during azure project termination
---
.../src/general/scripts/azure/project_terminate.py | 27 ++++++++++++++--------
1 file changed, 17 insertions(+), 10 deletions(-)
diff --git a/infrastructure-provisioning/src/general/scripts/azure/project_terminate.py b/infrastructure-provisioning/src/general/scripts/azure/project_terminate.py
index 092310b62..8b1696645 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/project_terminate.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/project_terminate.py
@@ -38,8 +38,8 @@ def terminate_edge_node(resource_group_name, service_base_name, project_tag, sub
clusters_list = AzureMeta.list_hdinsight_clusters(resource_group_name)
if clusters_list:
for cluster in clusters_list:
- if "sbn" in cluster.tags and service_base_name == cluster.tags["sbn"] and \
- "project" in cluster.tags and cluster.tags['project'] == project_tag:
+ if "SBN" in cluster.tags and service_base_name == cluster.tags["SBN"] and \
+ "project_tag" in cluster.tags and cluster.tags['project_tag'] == project_tag:
AzureActions.terminate_hdinsight_cluster(resource_group_name, cluster.name)
logging.info('The HDinsight cluster {} has been terminated successfully'.format(cluster.name))
else:
@@ -52,7 +52,8 @@ def terminate_edge_node(resource_group_name, service_base_name, project_tag, sub
try:
for vm in AzureMeta.compute_client.virtual_machines.list(resource_group_name):
try:
- if project_tag == vm.tags["project_tag"]:
+ if "SBN" in vm.tags and service_base_name == vm.tags["SBN"] and \
+ "project_tag" in vm.tags and vm.tags['project_tag'] == project_tag:
AzureActions.remove_instance(resource_group_name, vm.name)
logging.info("Instance {} has been terminated".format(vm.name))
except:
@@ -65,7 +66,8 @@ def terminate_edge_node(resource_group_name, service_base_name, project_tag, sub
try:
for network_interface in AzureMeta.list_network_interfaces(resource_group_name):
try:
- if project_tag == network_interface.tags["project_name"]:
+ if "SBN" in network_interface.tags and service_base_name == network_interface.tags["SBN"] and \
+ "project_tag" in network_interface.tags and network_interface.tags['project_tag'] == project_tag:
AzureActions.delete_network_if(resource_group_name, network_interface.name)
logging.info("Network interface {} has been removed".format(network_interface.name))
except:
@@ -78,7 +80,8 @@ def terminate_edge_node(resource_group_name, service_base_name, project_tag, sub
try:
for static_public_ip in AzureMeta.list_static_ips(resource_group_name):
try:
- if project_tag in static_public_ip.tags["project_tag"]:
+ if "SBN" in static_public_ip.tags and service_base_name == static_public_ip.tags["SBN"] and \
+ "project_tag" in static_public_ip.tags and static_public_ip.tags['project_tag'] == project_tag:
AzureActions.delete_static_public_ip(resource_group_name, static_public_ip.name)
logging.info("Static public IP {} has been removed".format(static_public_ip.name))
except:
@@ -91,7 +94,8 @@ def terminate_edge_node(resource_group_name, service_base_name, project_tag, sub
try:
for disk in AzureMeta.list_disks(resource_group_name):
try:
- if project_tag in disk.tags["project_tag"]:
+ if "SBN" in disk.tags and service_base_name == disk.tags["SBN"] and \
+ "project_tag" in disk.tags and disk.tags['project_tag'] == project_tag:
AzureActions.remove_disk(resource_group_name, disk.name)
logging.info("Disk {} has been removed".format(disk.name))
except:
@@ -104,7 +108,8 @@ def terminate_edge_node(resource_group_name, service_base_name, project_tag, sub
try:
for storage_account in AzureMeta.list_storage_accounts(resource_group_name):
try:
- if project_tag == storage_account.tags["project_tag"]:
+ if "SBN" in storage_account.tags and service_base_name == storage_account.tags["SBN"] and \
+ "project_tag" in storage_account.tags and storage_account.tags['project_tag'] == project_tag:
AzureActions.remove_storage_account(resource_group_name, storage_account.name)
logging.info("Storage account {} has been terminated".format(storage_account.name))
except:
@@ -129,8 +134,9 @@ def terminate_edge_node(resource_group_name, service_base_name, project_tag, sub
logging.info("Removing project specific images")
try:
for image in AzureMeta.list_images():
- if service_base_name == image.tags["SBN"] and project_tag == image.tags["project_tag"] \
- and endpoint_name == image.tags["endpoint_tag"]:
+ if "SBN" in image.tags and service_base_name == image.tags["SBN"] and \
+ "project_tag" in image.tags and image.tags['project_tag'] == project_tag and \
+ "endpoint_tag" in image.tags and endpoint_name == image.tags["endpoint_tag"]:
AzureActions.remove_image(resource_group_name, image.name)
logging.info("Image {} has been removed".format(image.name))
except Exception as err:
@@ -147,7 +153,8 @@ def terminate_edge_node(resource_group_name, service_base_name, project_tag, sub
project_conf['endpoint_name']))
for sg in AzureMeta.network_client.network_security_groups.list(resource_group_name):
try:
- if project_tag == sg.tags["project_tag"]:
+ if "SBN" in sg.tags and service_base_name == sg.tags["SBN"] and \
+ "project_tag" in sg.tags and sg.tags['project_tag'] == project_tag:
AzureActions.remove_security_group(resource_group_name, sg.name)
logging.info("Security group {} has been terminated".format(sg.name))
except:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 03/03: [DATALAB-1408]: fixed azure image list
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-1408
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit a9d0782d470c6e0ee8c08fb7fa0beb57e6397c29
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Thu Sep 8 13:16:27 2022 +0300
[DATALAB-1408]: fixed azure image list
---
infrastructure-provisioning/src/ssn/scripts/docker_build.py | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/infrastructure-provisioning/src/ssn/scripts/docker_build.py b/infrastructure-provisioning/src/ssn/scripts/docker_build.py
index cdb2ee786..9c368b67b 100644
--- a/infrastructure-provisioning/src/ssn/scripts/docker_build.py
+++ b/infrastructure-provisioning/src/ssn/scripts/docker_build.py
@@ -39,7 +39,6 @@ if sys.argv[1] == 'all':
'rstudio',
'zeppelin',
'tensor',
- 'tensor-rstudio',
'deeplearning',
'dataengine',
'dataengine-service']
@@ -54,14 +53,14 @@ def image_build(src_path, node):
os_family = 'redhat'
if subprocess.run("uname -r | awk -F '-' '{print $3}'", capture_output=True, shell=True, check=True).stdout.decode('UTF-8').rstrip("\n\r") == 'aws':
cloud_provider = 'aws'
- node.extend(['tensor-jupyterlab'])
+ node.extend(['tensor-rstudio', 'tensor-jupyterlab'])
elif subprocess.run("uname -r | awk -F '-' '{print $3}'", capture_output=True, shell=True, check=True).stdout.decode('UTF-8').rstrip("\n\r") == 'azure':
cloud_provider = 'azure'
if not path.exists('{}base/azure_auth.json'.format(src_path)):
subprocess.run('cp /home/datalab-user/keys/azure_auth.json {}base/azure_auth.json'.format(src_path), shell=True, check=True)
else:
cloud_provider = 'gcp'
- node.extend(['jupyter-gpu', 'superset'])
+ node.extend(['tensor-rstudio', 'jupyter-gpu', 'superset'])
subprocess.run('cd {2}; docker build --build-arg OS={0} --build-arg SRC_PATH= --file general/files/{1}/base_Dockerfile -t docker.datalab-base:latest .'.format(
os_family, cloud_provider, src_path), shell=True, check=True)
try:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 01/03: [DATALAB-1408]: fixed cluster tags
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-1408
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit d9205c9fee1db8c97d073d5ca697941fd82f01f1
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Thu Sep 8 12:49:55 2022 +0300
[DATALAB-1408]: fixed cluster tags
---
.../src/general/scripts/azure/dataengine-service_prepare.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py
index e0851272b..e93c50b10 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py
@@ -69,13 +69,13 @@ if __name__ == "__main__":
hdinsight_conf['computational_name'])
hdinsight_conf['cluster_tags'] = {
- "name": hdinsight_conf['cluster_name'],
- "sbn": hdinsight_conf['service_base_name'],
+ "Name": hdinsight_conf['cluster_name'],
+ "SBN": hdinsight_conf['service_base_name'],
"notebook_name": os.environ['notebook_instance_name'],
"product": "datalab",
"computational_name": hdinsight_conf['computational_name'],
- "project": hdinsight_conf['project_name'],
- "endpoint": hdinsight_conf['endpoint_name']
+ "project_tag": hdinsight_conf['project_name'],
+ "endpoint_tag": hdinsight_conf['endpoint_name']
}
hdinsight_conf['custom_tag'] = json.loads(os.environ['tags'].replace("'", '"'))['custom_tag']
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org