You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by yk...@apache.org on 2021/06/16 09:38:05 UTC
[incubator-datalab] 01/01: [DATALAB-2430] -- added missed param
This is an automated email from the ASF dual-hosted git repository.
ykinash pushed a commit to branch DATALAB-2430
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 97b493c2600b495bc758258f0d3e302259367aa6
Author: KinashYurii <ur...@gmail.com>
AuthorDate: Wed Jun 16 12:36:23 2021 +0300
[DATALAB-2430] -- added missed param
---
.../SparkStandaloneClusterResource.java | 35 ++++++++---
.../computational/SparkComputationalCreateGcp.java | 69 +++++++++++++++-------
.../dto/SparkStandaloneClusterCreateForm.java | 31 ++++++++--
.../service/impl/ComputationalServiceImpl.java | 11 +++-
.../datalab/backendapi/util/RequestBuilder.java | 23 +++++---
5 files changed, 125 insertions(+), 44 deletions(-)
diff --git a/services/datalab-model/src/main/java/com/epam/datalab/dto/computational/SparkStandaloneClusterResource.java b/services/datalab-model/src/main/java/com/epam/datalab/dto/computational/SparkStandaloneClusterResource.java
index f4bbd27..d06e4c9 100644
--- a/services/datalab-model/src/main/java/com/epam/datalab/dto/computational/SparkStandaloneClusterResource.java
+++ b/services/datalab-model/src/main/java/com/epam/datalab/dto/computational/SparkStandaloneClusterResource.java
@@ -39,24 +39,45 @@ import java.util.Map;
@ToString(callSuper = true)
public class SparkStandaloneClusterResource extends UserComputationalResource {
@NotBlank
- @JsonProperty("dataengine_instance_count")
- private String dataEngineInstanceCount;
+ @JsonProperty("master_instance_count")
+ private String masterDataEngineInstanceCount;
@NotBlank
- @JsonProperty("dataengine_instance_shape")
- private String dataEngineInstanceShape;
+ @JsonProperty("master_instance_shape")
+ private String masterDataEngineInstanceShape;
+
+ @NotBlank
+ @JsonProperty("slave_instance_count")
+ private String slaveDataEngineInstanceCount;
+
+ @NotBlank
+ @JsonProperty("slave_instance_shape")
+ private String slaveDtaEngineInstanceShape;
@Builder
public SparkStandaloneClusterResource(String computationalName, String computationalId, String imageName,
String templateName, String status, Date uptime,
SchedulerJobDTO schedulerJobData, boolean reuploadKeyRequired,
- String dataEngineInstanceCount, String dataEngineInstanceShape,
+ String masterDataEngineInstanceCount, String masterDataEngineInstanceShape,
+ String slaveDataEngineInstanceCount, String slaveDataEngineInstanceShape,
+ String masterGpuCount, String masterGpuType,
+ String slaveGpuCount, String slaveGpuType,
+ Boolean enabledGPU,
List<ResourceURL> resourceURL, LocalDateTime lastActivity,
List<ClusterConfig> config, Map<String, String> tags, int totalInstanceCount) {
super(computationalName, computationalId, imageName, templateName, status, uptime, schedulerJobData,
reuploadKeyRequired, resourceURL, lastActivity, tags, totalInstanceCount);
- this.dataEngineInstanceCount = dataEngineInstanceCount;
- this.dataEngineInstanceShape = dataEngineInstanceShape;
+ super.setMasterGpuCount(masterGpuCount);
+ super.setMasterGpuType(masterGpuType);
+ super.setSlaveGpuCount(slaveGpuCount);
+ super.setSlaveGpuType(slaveGpuType);
+ super.setEnabledGPU(enabledGPU);
+
+ this.masterDataEngineInstanceCount = masterDataEngineInstanceCount;
+ this.masterDataEngineInstanceShape = masterDataEngineInstanceShape;
+ this.slaveDataEngineInstanceCount = slaveDataEngineInstanceCount;
+ this.slaveDtaEngineInstanceShape = slaveDataEngineInstanceShape;
+
this.config = config;
}
}
diff --git a/services/datalab-model/src/main/java/com/epam/datalab/dto/gcp/computational/SparkComputationalCreateGcp.java b/services/datalab-model/src/main/java/com/epam/datalab/dto/gcp/computational/SparkComputationalCreateGcp.java
index 90bd51e..57e1d8d 100644
--- a/services/datalab-model/src/main/java/com/epam/datalab/dto/gcp/computational/SparkComputationalCreateGcp.java
+++ b/services/datalab-model/src/main/java/com/epam/datalab/dto/gcp/computational/SparkComputationalCreateGcp.java
@@ -22,60 +22,85 @@ package com.epam.datalab.dto.gcp.computational;
import com.epam.datalab.dto.aws.computational.ClusterConfig;
import com.epam.datalab.dto.base.computational.ComputationalBase;
import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.MoreObjects;
+import lombok.ToString;
import java.util.List;
+@ToString
public class SparkComputationalCreateGcp extends ComputationalBase<SparkComputationalCreateGcp> {
- @JsonProperty("dataengine_instance_count")
- private String dataEngineInstanceCount;
- @JsonProperty("gcp_dataengine_slave_size")
- private String dataEngineSlaveSize;
+ @JsonProperty("dataengine_master_instance_count")
+ private String masterDataEngineInstanceCount;
@JsonProperty("gcp_dataengine_master_size")
- private String dataEngineMasterSize;
+ private String masterDataEngineInstanceShape;
+ @JsonProperty("dataengine_slave_instance_count")
+ private String slaveDataEngineInstanceCount;
+ @JsonProperty("gcp_dataengine_slave_size")
+ private String slaveDtaEngineInstanceShape;
+
+ @JsonProperty("master_gpu_type")
+ private String masterGPUType;
+ @JsonProperty("slave_gpu_type")
+ private String slaveGPUType;
+ @JsonProperty("master_gpu_count")
+ private String masterGPUCount;
+ @JsonProperty("slave_gpu_count")
+ private String slaveGPUCount;
+
+
@JsonProperty("spark_configurations")
private List<ClusterConfig> config;
@JsonProperty("conf_shared_image_enabled")
private String sharedImageEnabled;
public SparkComputationalCreateGcp withDataEngineInstanceCount(String dataEngineInstanceCount) {
- this.dataEngineInstanceCount = dataEngineInstanceCount;
+ this.masterDataEngineInstanceCount = dataEngineInstanceCount;
return this;
}
public SparkComputationalCreateGcp withDataEngineSlaveSize(String dataEngineSlaveSize) {
- this.dataEngineSlaveSize = dataEngineSlaveSize;
+ this.slaveDtaEngineInstanceShape = dataEngineSlaveSize;
return this;
}
public SparkComputationalCreateGcp withDataEngineMasterSize(String dataEngineMasterSize) {
- this.dataEngineMasterSize = dataEngineMasterSize;
+ this.masterDataEngineInstanceShape = dataEngineMasterSize;
return this;
}
- public SparkComputationalCreateGcp withConfig(List<ClusterConfig> config) {
- this.config = config;
+ public SparkComputationalCreateGcp withSlaveDataEngineInstanceCount(String slaveDataEngineInstanceCount) {
+ this.slaveDataEngineInstanceCount = slaveDataEngineInstanceCount;
return this;
}
- public SparkComputationalCreateGcp withSharedImageEnabled(String sharedImageEnabled) {
- this.sharedImageEnabled = sharedImageEnabled;
+ public SparkComputationalCreateGcp withMasterGPUType(String masterGPUType) {
+ this.masterGPUType = masterGPUType;
return this;
}
+ public SparkComputationalCreateGcp withSlaveGPUType(String slaveGPUType) {
+ this.slaveGPUType = slaveGPUType;
+ return this;
+ }
- @Override
- public MoreObjects.ToStringHelper toStringHelper(Object self) {
- return super.toStringHelper(self)
- .add("dataEngineInstanceCount", dataEngineInstanceCount)
- .add("dataEngineSlaveSize", dataEngineSlaveSize)
- .add("dataEngineMasterSize", dataEngineMasterSize);
+ public SparkComputationalCreateGcp withMasterGPUCount(String masterGPUCount) {
+ this.masterGPUCount = masterGPUCount;
+ return this;
}
- @Override
- public String toString() {
- return toStringHelper(this).toString();
+ public SparkComputationalCreateGcp withSlaveGPUCount(String slaveGPUCount) {
+ this.slaveGPUCount = slaveGPUCount;
+ return this;
+ }
+
+ public SparkComputationalCreateGcp withConfig(List<ClusterConfig> config) {
+ this.config = config;
+ return this;
+ }
+
+ public SparkComputationalCreateGcp withSharedImageEnabled(String sharedImageEnabled) {
+ this.sharedImageEnabled = sharedImageEnabled;
+ return this;
}
}
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/dto/SparkStandaloneClusterCreateForm.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/dto/SparkStandaloneClusterCreateForm.java
index 0178dff..4510c1f 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/dto/SparkStandaloneClusterCreateForm.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/dto/SparkStandaloneClusterCreateForm.java
@@ -33,10 +33,33 @@ import org.hibernate.validator.constraints.NotBlank;
public class SparkStandaloneClusterCreateForm extends ComputationalCreateFormDTO {
@NotBlank
- @JsonProperty("dataengine_instance_count")
- private String dataEngineInstanceCount;
+ @JsonProperty("master_instance_count")
+ private String masterDataEngineInstanceCount;
@NotBlank
- @JsonProperty("dataengine_instance_shape")
- private String dataEngineInstanceShape;
+ @JsonProperty("master_instance_shape")
+ private String masterDataEngineInstanceShape;
+
+ @NotBlank
+ @JsonProperty("slave_instance_count")
+ private String slaveDataEngineInstanceCount;
+
+ @NotBlank
+ @JsonProperty("slave_instance_shape")
+ private String slaveDtaEngineInstanceShape;
+
+ @JsonProperty("gpu_enabled")
+ private Boolean enabledGPU;
+
+ @JsonProperty("master_gpu_type")
+ private String masterGpuType;
+
+ @JsonProperty("master_gpu_count")
+ private String masterGpuCount;
+
+ @JsonProperty("slave_gpu_type")
+ private String slaveGpuType;
+
+ @JsonProperty("slave_gpu_count")
+ private String slaveGpuCount;
}
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/ComputationalServiceImpl.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/ComputationalServiceImpl.java
index b75ff62..9d023a9 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/ComputationalServiceImpl.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/ComputationalServiceImpl.java
@@ -359,8 +359,15 @@ public class ComputationalServiceImpl implements ComputationalService {
.imageName(form.getImage())
.templateName(form.getTemplateName())
.status(CREATING.toString())
- .dataEngineInstanceCount(form.getDataEngineInstanceCount())
- .dataEngineInstanceShape(form.getDataEngineInstanceShape())
+ .masterDataEngineInstanceCount(form.getMasterDataEngineInstanceCount())
+ .masterDataEngineInstanceShape(form.getMasterDataEngineInstanceShape())
+ .slaveDataEngineInstanceCount(form.getSlaveDataEngineInstanceCount())
+ .slaveDataEngineInstanceShape(form.getSlaveDtaEngineInstanceShape())
+ .enabledGPU(form.getEnabledGPU())
+ .masterGpuCount(form.getMasterGpuCount())
+ .masterGpuType(form.getMasterGpuType())
+ .slaveGpuCount(form.getSlaveGpuCount())
+ .slaveGpuType(form.getSlaveGpuType())
.config(form.getConfig())
.build();
}
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java
index 5821d76..acb8add 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java
@@ -391,17 +391,17 @@ public class RequestBuilder {
switch (cloudProvider) {
case AWS:
computationalCreate = (T) newResourceSysBaseDTO(userInfo.getName(), cloudProvider, SparkComputationalCreateAws.class)
- .withDataEngineInstanceCount(form.getDataEngineInstanceCount())
- .withDataEngineMasterShape(form.getDataEngineInstanceShape())
- .withDataEngineSlaveShape(form.getDataEngineInstanceShape())
+ .withDataEngineInstanceCount(form.getMasterDataEngineInstanceCount())
+ .withDataEngineMasterShape(form.getMasterDataEngineInstanceShape())
+ .withDataEngineSlaveShape(form.getMasterDataEngineInstanceShape())
.withConfig(form.getConfig())
.withSharedImageEnabled(String.valueOf(projectDTO.isSharedImageEnabled()));
break;
case AZURE:
computationalCreate = (T) newResourceSysBaseDTO(userInfo.getName(), cloudProvider, SparkComputationalCreateAzure.class)
- .withDataEngineInstanceCount(form.getDataEngineInstanceCount())
- .withDataEngineMasterSize(form.getDataEngineInstanceShape())
- .withDataEngineSlaveSize(form.getDataEngineInstanceShape())
+ .withDataEngineInstanceCount(form.getMasterDataEngineInstanceCount())
+ .withDataEngineMasterSize(form.getMasterDataEngineInstanceShape())
+ .withDataEngineSlaveSize(form.getMasterDataEngineInstanceShape())
.withConfig(form.getConfig())
.withSharedImageEnabled(String.valueOf(projectDTO.isSharedImageEnabled()));
if (settingsDAO.isAzureDataLakeEnabled()) {
@@ -415,9 +415,14 @@ public class RequestBuilder {
break;
case GCP:
computationalCreate = (T) newResourceSysBaseDTO(userInfo.getName(), cloudProvider, SparkComputationalCreateGcp.class)
- .withDataEngineInstanceCount(form.getDataEngineInstanceCount())
- .withDataEngineMasterSize(form.getDataEngineInstanceShape())
- .withDataEngineSlaveSize(form.getDataEngineInstanceShape())
+ .withDataEngineInstanceCount(form.getMasterDataEngineInstanceCount())
+ .withSlaveDataEngineInstanceCount(form.getSlaveDataEngineInstanceCount())
+ .withDataEngineMasterSize(form.getMasterDataEngineInstanceShape())
+ .withDataEngineSlaveSize(form.getMasterDataEngineInstanceShape())
+ .withMasterGPUCount(form.getMasterGpuCount())
+ .withMasterGPUType(form.getMasterGpuType())
+ .withSlaveGPUCount(form.getSlaveGpuCount())
+ .withSlaveGPUType(form.getSlaveGpuType())
.withConfig(form.getConfig())
.withSharedImageEnabled(String.valueOf(projectDTO.isSharedImageEnabled()));
break;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org