You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2022/09/13 13:31:53 UTC
[incubator-datalab] 01/01: [DATALAB-1408]: specified spark version 3.1 for hdinsight
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-1408
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 6e54d7f12b490b480040055ce73303669b7224c8
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Tue Sep 13 16:31:40 2022 +0300
[DATALAB-1408]: specified spark version 3.1 for hdinsight
---
.../src/general/files/azure/dataengine-service_description.json | 2 +-
.../src/general/scripts/azure/dataengine-service_create.py | 4 ++++
2 files changed, 5 insertions(+), 1 deletion(-)
diff --git a/infrastructure-provisioning/src/general/files/azure/dataengine-service_description.json b/infrastructure-provisioning/src/general/files/azure/dataengine-service_description.json
index faa7d1767..7128eeff8 100644
--- a/infrastructure-provisioning/src/general/files/azure/dataengine-service_description.json
+++ b/infrastructure-provisioning/src/general/files/azure/dataengine-service_description.json
@@ -15,6 +15,6 @@
},
"templates":
[
- {"version":"4.0", "applications": [{"Name":"Spark", "Version": "2.4"}]}
+ {"version":"4.0", "applications": [{"Name":"Spark", "Version": "3.1"}]}
]
}
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_create.py b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_create.py
index d03518bd5..b4075318b 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_create.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_create.py
@@ -73,6 +73,10 @@ def create_cluster_parameters(location, tags, cluster_version, cluster_login_use
tier=Tier.standard,
cluster_definition=ClusterDefinition(
kind="Spark",
+ component_version=
+ {
+ "Spark": "3.1"
+ },
configurations={
"gateway": {
"restAuthCredential.isEnabled": "true",
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org