You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by of...@apache.org on 2020/06/24 13:38:35 UTC
[incubator-dlab] branch develop updated: [DLAB-1734] Conveyed
amount of nodes for computational resources
This is an automated email from the ASF dual-hosted git repository.
ofuks pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/develop by this push:
new 202e6f2 [DLAB-1734] Conveyed amount of nodes for computational resources
202e6f2 is described below
commit 202e6f2cf7e3d7cbaeafdc1d0f4deae35ccf5bf6
Author: Oleh Fuks <ol...@gmail.com>
AuthorDate: Wed Jun 24 16:37:58 2020 +0300
[DLAB-1734] Conveyed amount of nodes for computational resources
---
.../dto/aws/computational/AwsComputationalResource.java | 5 ++---
.../computational/SparkStandaloneClusterResource.java | 5 ++---
.../dto/computational/UserComputationalResource.java | 4 +++-
.../dto/gcp/computational/GcpComputationalResource.java | 4 ++--
.../resources/aws/ComputationalResourceAws.java | 17 +++++++++--------
.../resources/gcp/ComputationalResourceGcp.java | 9 +++++----
.../com/epam/dlab/backendapi/util/BillingUtils.java | 4 ++--
7 files changed, 25 insertions(+), 23 deletions(-)
diff --git a/services/dlab-model/src/main/java/com/epam/dlab/dto/aws/computational/AwsComputationalResource.java b/services/dlab-model/src/main/java/com/epam/dlab/dto/aws/computational/AwsComputationalResource.java
index 8a268c5..7b8f77b 100644
--- a/services/dlab-model/src/main/java/com/epam/dlab/dto/aws/computational/AwsComputationalResource.java
+++ b/services/dlab-model/src/main/java/com/epam/dlab/dto/aws/computational/AwsComputationalResource.java
@@ -63,10 +63,9 @@ public class AwsComputationalResource extends UserComputationalResource {
String instanceId, String masterShape, String slaveShape, Boolean slaveSpot,
Integer slaveSpotPctPrice, String slaveNumber, String version,
List<ResourceURL> resourceURL, LocalDateTime lastActivity,
- List<ClusterConfig> config, Map<String, String> tags) {
-
+ List<ClusterConfig> config, Map<String, String> tags, String totalInstanceCount) {
super(computationalName, computationalId, imageName, templateName, status, uptime, schedulerJobData,
- reuploadKeyRequired, resourceURL, lastActivity, tags);
+ reuploadKeyRequired, resourceURL, lastActivity, tags, totalInstanceCount);
this.instanceId = instanceId;
this.masterShape = masterShape;
this.slaveShape = slaveShape;
diff --git a/services/dlab-model/src/main/java/com/epam/dlab/dto/computational/SparkStandaloneClusterResource.java b/services/dlab-model/src/main/java/com/epam/dlab/dto/computational/SparkStandaloneClusterResource.java
index 82f5660..80c6d5b 100644
--- a/services/dlab-model/src/main/java/com/epam/dlab/dto/computational/SparkStandaloneClusterResource.java
+++ b/services/dlab-model/src/main/java/com/epam/dlab/dto/computational/SparkStandaloneClusterResource.java
@@ -52,10 +52,9 @@ public class SparkStandaloneClusterResource extends UserComputationalResource {
SchedulerJobDTO schedulerJobData, boolean reuploadKeyRequired,
String dataEngineInstanceCount, String dataEngineInstanceShape,
List<ResourceURL> resourceURL, LocalDateTime lastActivity,
- List<ClusterConfig> config, Map<String, String> tags) {
-
+ List<ClusterConfig> config, Map<String, String> tags, String totalInstanceCount) {
super(computationalName, computationalId, imageName, templateName, status, uptime, schedulerJobData,
- reuploadKeyRequired, resourceURL, lastActivity, tags);
+ reuploadKeyRequired, resourceURL, lastActivity, tags, totalInstanceCount);
this.dataEngineInstanceCount = dataEngineInstanceCount;
this.dataEngineInstanceShape = dataEngineInstanceShape;
this.config = config;
diff --git a/services/dlab-model/src/main/java/com/epam/dlab/dto/computational/UserComputationalResource.java b/services/dlab-model/src/main/java/com/epam/dlab/dto/computational/UserComputationalResource.java
index 9f8c021..df0892c 100644
--- a/services/dlab-model/src/main/java/com/epam/dlab/dto/computational/UserComputationalResource.java
+++ b/services/dlab-model/src/main/java/com/epam/dlab/dto/computational/UserComputationalResource.java
@@ -67,13 +67,14 @@ public class UserComputationalResource {
private int dataengineInstanceCount;
@JsonProperty("instance_id")
private String instanceId;
+ private String totalInstanceCount;
protected List<ClusterConfig> config;
private Map<String, String> tags;
public UserComputationalResource(String computationalName, String computationalId, String imageName,
String templateName, String status, Date uptime, SchedulerJobDTO schedulerData,
boolean reuploadKeyRequired, List<ResourceURL> resourceUrl,
- LocalDateTime lastActivity, Map<String, String> tags) {
+ LocalDateTime lastActivity, Map<String, String> tags, String totalInstanceCount) {
this.computationalName = computationalName;
this.computationalId = computationalId;
this.imageName = imageName;
@@ -85,6 +86,7 @@ public class UserComputationalResource {
this.resourceUrl = resourceUrl;
this.lastActivity = lastActivity;
this.tags = tags;
+ this.totalInstanceCount = totalInstanceCount;
}
public DataEngineType getDataEngineType() {
diff --git a/services/dlab-model/src/main/java/com/epam/dlab/dto/gcp/computational/GcpComputationalResource.java b/services/dlab-model/src/main/java/com/epam/dlab/dto/gcp/computational/GcpComputationalResource.java
index b56bbd8..5f0f103 100644
--- a/services/dlab-model/src/main/java/com/epam/dlab/dto/gcp/computational/GcpComputationalResource.java
+++ b/services/dlab-model/src/main/java/com/epam/dlab/dto/gcp/computational/GcpComputationalResource.java
@@ -63,9 +63,9 @@ public class GcpComputationalResource extends UserComputationalResource {
String instanceId, String masterShape, String slaveShape, String slaveNumber,
String masterNumber, String preemptibleNumber, String version,
List<ResourceURL> resourceURL, LocalDateTime lastActivity,
- Map<String, String> tags) {
+ Map<String, String> tags, String totalInstanceCount) {
super(computationalName, computationalId, imageName, templateName, status, uptime, schedulerJobData,
- reuploadKeyRequired, resourceURL, lastActivity, tags);
+ reuploadKeyRequired, resourceURL, lastActivity, tags, totalInstanceCount);
this.instanceId = instanceId;
this.masterShape = masterShape;
this.slaveShape = slaveShape;
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/aws/ComputationalResourceAws.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/aws/ComputationalResourceAws.java
index 9c82f04..be47eb7 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/aws/ComputationalResourceAws.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/aws/ComputationalResourceAws.java
@@ -97,14 +97,15 @@ public class ComputationalResourceAws implements ComputationalAPI {
.computationalName(form.getName())
.imageName(form.getImage())
.templateName(form.getTemplateName())
- .status(CREATING.toString())
- .masterShape(form.getMasterInstanceType())
- .slaveShape(form.getSlaveInstanceType())
- .slaveSpot(form.getSlaveInstanceSpot())
- .slaveSpotPctPrice(form.getSlaveInstanceSpotPctPrice())
- .slaveNumber(form.getInstanceCount())
- .config(form.getConfig())
- .version(form.getVersion())
+ .status(CREATING.toString())
+ .masterShape(form.getMasterInstanceType())
+ .slaveShape(form.getSlaveInstanceType())
+ .slaveSpot(form.getSlaveInstanceSpot())
+ .slaveSpotPctPrice(form.getSlaveInstanceSpotPctPrice())
+ .slaveNumber(form.getInstanceCount())
+ .config(form.getConfig())
+ .version(form.getVersion())
+ .totalInstanceCount(form.getInstanceCount())
.build();
boolean resourceAdded = computationalService.createDataEngineService(userInfo, form.getName(), form, awsComputationalResource,
form.getProject(), getAuditInfo(form.getNotebookName()));
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/gcp/ComputationalResourceGcp.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/gcp/ComputationalResourceGcp.java
index c5a9b6b..8f9c832 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/gcp/ComputationalResourceGcp.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/gcp/ComputationalResourceGcp.java
@@ -95,10 +95,10 @@ public class ComputationalResourceGcp implements ComputationalAPI {
log.debug("Create computational resources for {} | form is {}", userInfo.getName(), form);
if (DataEngineType.CLOUD_SERVICE == DataEngineType.fromDockerImageName(form.getImage())) {
- validate(userInfo, form);
- GcpComputationalResource gcpComputationalResource = GcpComputationalResource.builder().computationalName
- (form.getName())
- .imageName(form.getImage())
+ validate(userInfo, form);
+ GcpComputationalResource gcpComputationalResource = GcpComputationalResource.builder()
+ .computationalName(form.getName())
+ .imageName(form.getImage())
.templateName(form.getTemplateName())
.status(CREATING.toString())
.masterShape(form.getMasterInstanceType())
@@ -107,6 +107,7 @@ public class ComputationalResourceGcp implements ComputationalAPI {
.masterNumber(form.getMasterInstanceCount())
.preemptibleNumber(form.getPreemptibleCount())
.version(form.getVersion())
+ .totalInstanceCount(form.getMasterInstanceCount() + form.getSlaveInstanceCount())
.build();
boolean resourceAdded = computationalService.createDataEngineService(userInfo, form.getName(), form, gcpComputationalResource,
form.getProject(), getAuditInfo(form.getNotebookName()));
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java
index 72e5a77..bfe704b 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java
@@ -70,7 +70,7 @@ public class BillingUtils {
private static final String IMAGE_NAME = "Image";
private static final String DATAENGINE_NAME_FORMAT = "%d x %s";
- private static final String DATAENGINE_SERVICE_NAME_FORMAT = "Master: %sSlave: %s";
+ private static final String DATAENGINE_SERVICE_NAME_FORMAT = "Master: %sSlave: %s x %s";
public static Stream<BillingReportLine> edgeBillingDataStream(String project, String sbn, String endpoint) {
final String userEdgeId = String.format(EDGE_FORMAT, sbn, project, endpoint).toLowerCase();
@@ -160,7 +160,7 @@ public class BillingUtils {
public static String getComputationalShape(UserComputationalResource resource) {
return DataEngineType.fromDockerImageName(resource.getImageName()) == DataEngineType.SPARK_STANDALONE ?
String.format(DATAENGINE_NAME_FORMAT, resource.getDataengineInstanceCount(), resource.getDataengineShape()) :
- String.format(DATAENGINE_SERVICE_NAME_FORMAT, resource.getMasterNodeShape(), resource.getSlaveNodeShape());
+ String.format(DATAENGINE_SERVICE_NAME_FORMAT, resource.getMasterNodeShape(), resource.getTotalInstanceCount(), resource.getSlaveNodeShape());
}
private static Stream<BillingReportLine> standardImageBillingDataStream(String sbn, String endpoint) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org