You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2022/09/06 13:09:23 UTC
[incubator-datalab] 05/06: [DATALAB-1408]: fixed shapes
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-1408
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit a3b6d915e175325064ebcea8583a98547d7c792c
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Tue Sep 6 16:08:25 2022 +0300
[DATALAB-1408]: fixed shapes
---
.../files/azure/dataengine-service_description.json | 18 +++++-------------
.../src/general/lib/azure/actions_lib.py | 1 +
.../general/scripts/azure/dataengine-service_create.py | 4 ++--
.../scripts/azure/dataengine-service_prepare.py | 7 ++++++-
4 files changed, 14 insertions(+), 16 deletions(-)
diff --git a/infrastructure-provisioning/src/general/files/azure/dataengine-service_description.json b/infrastructure-provisioning/src/general/files/azure/dataengine-service_description.json
index c41965004..faa7d1767 100644
--- a/infrastructure-provisioning/src/general/files/azure/dataengine-service_description.json
+++ b/infrastructure-provisioning/src/general/files/azure/dataengine-service_description.json
@@ -5,24 +5,16 @@
"computation_resources_shapes":
{
"For testing" : [
- {"Size": "S", "Description": "Standard_F4s", "Type": "Standard_F4s","Ram": "8.0 GB","Cpu": "4"}
+ {"Size": "S", "Description": "Standard_D12_v2", "Type": "Standard_D12_v2","Ram": "28.0 GB","Cpu": "4"}
],
"Memory optimized" : [
- {"Size": "S", "Description": "Standard_E4s_v3", "Type": "Standard_E4s_v3","Ram": "32 GB","Cpu": "4"},
- {"Size": "M", "Description": "Standard_E16s_v3", "Type": "Standard_E16s_v3","Ram": "128 GB","Cpu": "16"},
- {"Size": "L", "Description": "Standard_E32s_v3", "Type": "Standard_E32s_v3","Ram": "256 GB","Cpu": "32"}
- ],
- "Compute optimized": [
- {"Size": "S", "Description": "Standard_F4s", "Type": "Standard_F4s","Ram": "8.0 GB","Cpu": "4"},
- {"Size": "M", "Description": "Standard_F8s", "Type": "Standard_F8s","Ram": "16.0 GB","Cpu": "8"},
- {"Size": "L", "Description": "Standard_F16s", "Type": "Standard_F16s","Ram": "32.0 GB","Cpu": "16"}
- ],
- "GPU optimized": [
- {"Size": "S", "Description": "Standard_NC6", "Type": "Standard_NC6","Ram": "56.0 GB","Cpu": "6"}
+ {"Size": "S", "Description": "Standard_E4_v3", "Type": "Standard_E4_v3","Ram": "32 GB","Cpu": "4"},
+ {"Size": "M", "Description": "Standard_E16_v3", "Type": "Standard_E16_v3","Ram": "128 GB","Cpu": "16"},
+ {"Size": "L", "Description": "Standard_E32_v3", "Type": "Standard_E32_v3","Ram": "256 GB","Cpu": "32"}
]
},
"templates":
[
- {"version":"4.0", "applications": [{"Name":"Spark", "Version": "x.x.x"}]}
+ {"version":"4.0", "applications": [{"Name":"Spark", "Version": "2.4"}]}
]
}
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
index db9088aa6..1b3086e69 100644
--- a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
@@ -509,6 +509,7 @@ class AzureActions:
resource_group_name,
account_name
)
+ logging.info("Storage account {} was removed.".format(account_name))
return result
except Exception as err:
logging.info(
diff --git a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_create.py b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_create.py
index 741424eed..8684a329b 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_create.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_create.py
@@ -52,7 +52,7 @@ args = parser.parse_args()
def build_hdinsight_cluster(resource_group_name, cluster_name, params):
- logging.info("Will be created cluster: {}".format(cluster_name))
+ logging.info("{} cluster creation".format(cluster_name))
return datalab.actions_lib.AzureActions().create_hdinsight_cluster(resource_group_name, cluster_name, params)
@@ -122,7 +122,7 @@ def create_cluster_parameters(location, tags, cluster_version, cluster_login_use
Role(
name="zookeepernode",
target_instance_count=3,
- hardware_profile=HardwareProfile(vm_size=Small),
+ hardware_profile=HardwareProfile(vm_size="Standard_A4_v2"),
os_profile=OsProfile(
linux_operating_system_profile=LinuxOperatingSystemProfile(
username=cluster_login_username,
diff --git a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py
index 49f636566..af106da59 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py
@@ -23,6 +23,7 @@
import datalab.fab
import datalab.meta_lib
+import datalab.actions_lib
import json
import multiprocessing
import os
@@ -35,6 +36,7 @@ from fabric import *
if __name__ == "__main__":
try:
+ AzureActions = datalab.actions_lib.AzureActions()
AzureMeta = datalab.meta_lib.AzureMeta()
logging.info('Generating infrastructure names and tags')
hdinsight_conf = dict()
@@ -99,6 +101,9 @@ if __name__ == "__main__":
hdinsight_conf['vpc_name'],
hdinsight_conf['subnet_name']).id
+ hdinsight_conf['hdinsight_master_instance_type'] = os.environ['hdinsight_master_instance_type']
+ hdinsight_conf['hdinsight_slave_instance_type'] = os.environ['hdinsight_slave_instance_type']
+
except Exception as err:
datalab.fab.append_result("Failed to generate variables dictionary. Exception:" + str(err))
sys.exit(1)
@@ -137,7 +142,7 @@ if __name__ == "__main__":
"--tags '{}' --public_key '{}' --vpc_id {} --subnet {}"\
.format(hdinsight_conf['resource_group_name'], hdinsight_conf['cluster_name'],
hdinsight_conf['release_label'], hdinsight_conf['region'],
- os.environ['hdinsight_master_instance_type'], os.environ['hdinsight_slave_instance_type'],
+ hdinsight_conf['hdinsight_master_instance_type'], hdinsight_conf['hdinsight_slave_instance_type'],
hdinsight_conf['hdinsight_worker_count'], hdinsight_conf['storage_account_name'],
hdinsight_conf['storage_account_key'], hdinsight_conf['container_name'],
json.dumps(hdinsight_conf['cluster_tags']), ssh_admin_pubkey, hdinsight_conf['vpc_id'],
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org