You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by yk...@apache.org on 2021/06/16 09:38:04 UTC
[incubator-datalab] branch DATALAB-2430 created (now 97b493c)
This is an automated email from the ASF dual-hosted git repository.
ykinash pushed a change to branch DATALAB-2430
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git.
at 97b493c [DATALAB-2430] -- added missed param
This branch includes the following new commits:
new 97b493c [DATALAB-2430] -- added missed param
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 01/01: [DATALAB-2430] -- added missed param
Posted by yk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ykinash pushed a commit to branch DATALAB-2430
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 97b493c2600b495bc758258f0d3e302259367aa6
Author: KinashYurii <ur...@gmail.com>
AuthorDate: Wed Jun 16 12:36:23 2021 +0300
[DATALAB-2430] -- added missed param
---
.../SparkStandaloneClusterResource.java | 35 ++++++++---
.../computational/SparkComputationalCreateGcp.java | 69 +++++++++++++++-------
.../dto/SparkStandaloneClusterCreateForm.java | 31 ++++++++--
.../service/impl/ComputationalServiceImpl.java | 11 +++-
.../datalab/backendapi/util/RequestBuilder.java | 23 +++++---
5 files changed, 125 insertions(+), 44 deletions(-)
diff --git a/services/datalab-model/src/main/java/com/epam/datalab/dto/computational/SparkStandaloneClusterResource.java b/services/datalab-model/src/main/java/com/epam/datalab/dto/computational/SparkStandaloneClusterResource.java
index f4bbd27..d06e4c9 100644
--- a/services/datalab-model/src/main/java/com/epam/datalab/dto/computational/SparkStandaloneClusterResource.java
+++ b/services/datalab-model/src/main/java/com/epam/datalab/dto/computational/SparkStandaloneClusterResource.java
@@ -39,24 +39,45 @@ import java.util.Map;
@ToString(callSuper = true)
public class SparkStandaloneClusterResource extends UserComputationalResource {
@NotBlank
- @JsonProperty("dataengine_instance_count")
- private String dataEngineInstanceCount;
+ @JsonProperty("master_instance_count")
+ private String masterDataEngineInstanceCount;
@NotBlank
- @JsonProperty("dataengine_instance_shape")
- private String dataEngineInstanceShape;
+ @JsonProperty("master_instance_shape")
+ private String masterDataEngineInstanceShape;
+
+ @NotBlank
+ @JsonProperty("slave_instance_count")
+ private String slaveDataEngineInstanceCount;
+
+ @NotBlank
+ @JsonProperty("slave_instance_shape")
+ private String slaveDtaEngineInstanceShape;
@Builder
public SparkStandaloneClusterResource(String computationalName, String computationalId, String imageName,
String templateName, String status, Date uptime,
SchedulerJobDTO schedulerJobData, boolean reuploadKeyRequired,
- String dataEngineInstanceCount, String dataEngineInstanceShape,
+ String masterDataEngineInstanceCount, String masterDataEngineInstanceShape,
+ String slaveDataEngineInstanceCount, String slaveDataEngineInstanceShape,
+ String masterGpuCount, String masterGpuType,
+ String slaveGpuCount, String slaveGpuType,
+ Boolean enabledGPU,
List<ResourceURL> resourceURL, LocalDateTime lastActivity,
List<ClusterConfig> config, Map<String, String> tags, int totalInstanceCount) {
super(computationalName, computationalId, imageName, templateName, status, uptime, schedulerJobData,
reuploadKeyRequired, resourceURL, lastActivity, tags, totalInstanceCount);
- this.dataEngineInstanceCount = dataEngineInstanceCount;
- this.dataEngineInstanceShape = dataEngineInstanceShape;
+ super.setMasterGpuCount(masterGpuCount);
+ super.setMasterGpuType(masterGpuType);
+ super.setSlaveGpuCount(slaveGpuCount);
+ super.setSlaveGpuType(slaveGpuType);
+ super.setEnabledGPU(enabledGPU);
+
+ this.masterDataEngineInstanceCount = masterDataEngineInstanceCount;
+ this.masterDataEngineInstanceShape = masterDataEngineInstanceShape;
+ this.slaveDataEngineInstanceCount = slaveDataEngineInstanceCount;
+ this.slaveDtaEngineInstanceShape = slaveDataEngineInstanceShape;
+
this.config = config;
}
}
diff --git a/services/datalab-model/src/main/java/com/epam/datalab/dto/gcp/computational/SparkComputationalCreateGcp.java b/services/datalab-model/src/main/java/com/epam/datalab/dto/gcp/computational/SparkComputationalCreateGcp.java
index 90bd51e..57e1d8d 100644
--- a/services/datalab-model/src/main/java/com/epam/datalab/dto/gcp/computational/SparkComputationalCreateGcp.java
+++ b/services/datalab-model/src/main/java/com/epam/datalab/dto/gcp/computational/SparkComputationalCreateGcp.java
@@ -22,60 +22,85 @@ package com.epam.datalab.dto.gcp.computational;
import com.epam.datalab.dto.aws.computational.ClusterConfig;
import com.epam.datalab.dto.base.computational.ComputationalBase;
import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.MoreObjects;
+import lombok.ToString;
import java.util.List;
+@ToString
public class SparkComputationalCreateGcp extends ComputationalBase<SparkComputationalCreateGcp> {
- @JsonProperty("dataengine_instance_count")
- private String dataEngineInstanceCount;
- @JsonProperty("gcp_dataengine_slave_size")
- private String dataEngineSlaveSize;
+ @JsonProperty("dataengine_master_instance_count")
+ private String masterDataEngineInstanceCount;
@JsonProperty("gcp_dataengine_master_size")
- private String dataEngineMasterSize;
+ private String masterDataEngineInstanceShape;
+ @JsonProperty("dataengine_slave_instance_count")
+ private String slaveDataEngineInstanceCount;
+ @JsonProperty("gcp_dataengine_slave_size")
+ private String slaveDtaEngineInstanceShape;
+
+ @JsonProperty("master_gpu_type")
+ private String masterGPUType;
+ @JsonProperty("slave_gpu_type")
+ private String slaveGPUType;
+ @JsonProperty("master_gpu_count")
+ private String masterGPUCount;
+ @JsonProperty("slave_gpu_count")
+ private String slaveGPUCount;
+
+
@JsonProperty("spark_configurations")
private List<ClusterConfig> config;
@JsonProperty("conf_shared_image_enabled")
private String sharedImageEnabled;
public SparkComputationalCreateGcp withDataEngineInstanceCount(String dataEngineInstanceCount) {
- this.dataEngineInstanceCount = dataEngineInstanceCount;
+ this.masterDataEngineInstanceCount = dataEngineInstanceCount;
return this;
}
public SparkComputationalCreateGcp withDataEngineSlaveSize(String dataEngineSlaveSize) {
- this.dataEngineSlaveSize = dataEngineSlaveSize;
+ this.slaveDtaEngineInstanceShape = dataEngineSlaveSize;
return this;
}
public SparkComputationalCreateGcp withDataEngineMasterSize(String dataEngineMasterSize) {
- this.dataEngineMasterSize = dataEngineMasterSize;
+ this.masterDataEngineInstanceShape = dataEngineMasterSize;
return this;
}
- public SparkComputationalCreateGcp withConfig(List<ClusterConfig> config) {
- this.config = config;
+ public SparkComputationalCreateGcp withSlaveDataEngineInstanceCount(String slaveDataEngineInstanceCount) {
+ this.slaveDataEngineInstanceCount = slaveDataEngineInstanceCount;
return this;
}
- public SparkComputationalCreateGcp withSharedImageEnabled(String sharedImageEnabled) {
- this.sharedImageEnabled = sharedImageEnabled;
+ public SparkComputationalCreateGcp withMasterGPUType(String masterGPUType) {
+ this.masterGPUType = masterGPUType;
return this;
}
+ public SparkComputationalCreateGcp withSlaveGPUType(String slaveGPUType) {
+ this.slaveGPUType = slaveGPUType;
+ return this;
+ }
- @Override
- public MoreObjects.ToStringHelper toStringHelper(Object self) {
- return super.toStringHelper(self)
- .add("dataEngineInstanceCount", dataEngineInstanceCount)
- .add("dataEngineSlaveSize", dataEngineSlaveSize)
- .add("dataEngineMasterSize", dataEngineMasterSize);
+ public SparkComputationalCreateGcp withMasterGPUCount(String masterGPUCount) {
+ this.masterGPUCount = masterGPUCount;
+ return this;
}
- @Override
- public String toString() {
- return toStringHelper(this).toString();
+ public SparkComputationalCreateGcp withSlaveGPUCount(String slaveGPUCount) {
+ this.slaveGPUCount = slaveGPUCount;
+ return this;
+ }
+
+ public SparkComputationalCreateGcp withConfig(List<ClusterConfig> config) {
+ this.config = config;
+ return this;
+ }
+
+ public SparkComputationalCreateGcp withSharedImageEnabled(String sharedImageEnabled) {
+ this.sharedImageEnabled = sharedImageEnabled;
+ return this;
}
}
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/dto/SparkStandaloneClusterCreateForm.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/dto/SparkStandaloneClusterCreateForm.java
index 0178dff..4510c1f 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/dto/SparkStandaloneClusterCreateForm.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/dto/SparkStandaloneClusterCreateForm.java
@@ -33,10 +33,33 @@ import org.hibernate.validator.constraints.NotBlank;
public class SparkStandaloneClusterCreateForm extends ComputationalCreateFormDTO {
@NotBlank
- @JsonProperty("dataengine_instance_count")
- private String dataEngineInstanceCount;
+ @JsonProperty("master_instance_count")
+ private String masterDataEngineInstanceCount;
@NotBlank
- @JsonProperty("dataengine_instance_shape")
- private String dataEngineInstanceShape;
+ @JsonProperty("master_instance_shape")
+ private String masterDataEngineInstanceShape;
+
+ @NotBlank
+ @JsonProperty("slave_instance_count")
+ private String slaveDataEngineInstanceCount;
+
+ @NotBlank
+ @JsonProperty("slave_instance_shape")
+ private String slaveDtaEngineInstanceShape;
+
+ @JsonProperty("gpu_enabled")
+ private Boolean enabledGPU;
+
+ @JsonProperty("master_gpu_type")
+ private String masterGpuType;
+
+ @JsonProperty("master_gpu_count")
+ private String masterGpuCount;
+
+ @JsonProperty("slave_gpu_type")
+ private String slaveGpuType;
+
+ @JsonProperty("slave_gpu_count")
+ private String slaveGpuCount;
}
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/ComputationalServiceImpl.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/ComputationalServiceImpl.java
index b75ff62..9d023a9 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/ComputationalServiceImpl.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/ComputationalServiceImpl.java
@@ -359,8 +359,15 @@ public class ComputationalServiceImpl implements ComputationalService {
.imageName(form.getImage())
.templateName(form.getTemplateName())
.status(CREATING.toString())
- .dataEngineInstanceCount(form.getDataEngineInstanceCount())
- .dataEngineInstanceShape(form.getDataEngineInstanceShape())
+ .masterDataEngineInstanceCount(form.getMasterDataEngineInstanceCount())
+ .masterDataEngineInstanceShape(form.getMasterDataEngineInstanceShape())
+ .slaveDataEngineInstanceCount(form.getSlaveDataEngineInstanceCount())
+ .slaveDataEngineInstanceShape(form.getSlaveDtaEngineInstanceShape())
+ .enabledGPU(form.getEnabledGPU())
+ .masterGpuCount(form.getMasterGpuCount())
+ .masterGpuType(form.getMasterGpuType())
+ .slaveGpuCount(form.getSlaveGpuCount())
+ .slaveGpuType(form.getSlaveGpuType())
.config(form.getConfig())
.build();
}
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java
index 5821d76..acb8add 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java
@@ -391,17 +391,17 @@ public class RequestBuilder {
switch (cloudProvider) {
case AWS:
computationalCreate = (T) newResourceSysBaseDTO(userInfo.getName(), cloudProvider, SparkComputationalCreateAws.class)
- .withDataEngineInstanceCount(form.getDataEngineInstanceCount())
- .withDataEngineMasterShape(form.getDataEngineInstanceShape())
- .withDataEngineSlaveShape(form.getDataEngineInstanceShape())
+ .withDataEngineInstanceCount(form.getMasterDataEngineInstanceCount())
+ .withDataEngineMasterShape(form.getMasterDataEngineInstanceShape())
+ .withDataEngineSlaveShape(form.getMasterDataEngineInstanceShape())
.withConfig(form.getConfig())
.withSharedImageEnabled(String.valueOf(projectDTO.isSharedImageEnabled()));
break;
case AZURE:
computationalCreate = (T) newResourceSysBaseDTO(userInfo.getName(), cloudProvider, SparkComputationalCreateAzure.class)
- .withDataEngineInstanceCount(form.getDataEngineInstanceCount())
- .withDataEngineMasterSize(form.getDataEngineInstanceShape())
- .withDataEngineSlaveSize(form.getDataEngineInstanceShape())
+ .withDataEngineInstanceCount(form.getMasterDataEngineInstanceCount())
+ .withDataEngineMasterSize(form.getMasterDataEngineInstanceShape())
+ .withDataEngineSlaveSize(form.getMasterDataEngineInstanceShape())
.withConfig(form.getConfig())
.withSharedImageEnabled(String.valueOf(projectDTO.isSharedImageEnabled()));
if (settingsDAO.isAzureDataLakeEnabled()) {
@@ -415,9 +415,14 @@ public class RequestBuilder {
break;
case GCP:
computationalCreate = (T) newResourceSysBaseDTO(userInfo.getName(), cloudProvider, SparkComputationalCreateGcp.class)
- .withDataEngineInstanceCount(form.getDataEngineInstanceCount())
- .withDataEngineMasterSize(form.getDataEngineInstanceShape())
- .withDataEngineSlaveSize(form.getDataEngineInstanceShape())
+ .withDataEngineInstanceCount(form.getMasterDataEngineInstanceCount())
+ .withSlaveDataEngineInstanceCount(form.getSlaveDataEngineInstanceCount())
+ .withDataEngineMasterSize(form.getMasterDataEngineInstanceShape())
+ .withDataEngineSlaveSize(form.getMasterDataEngineInstanceShape())
+ .withMasterGPUCount(form.getMasterGpuCount())
+ .withMasterGPUType(form.getMasterGpuType())
+ .withSlaveGPUCount(form.getSlaveGpuCount())
+ .withSlaveGPUType(form.getSlaveGpuType())
.withConfig(form.getConfig())
.withSharedImageEnabled(String.valueOf(projectDTO.isSharedImageEnabled()));
break;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org