You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by yk...@apache.org on 2021/11/08 19:05:16 UTC
[incubator-datalab] 01/03: [DATALAB-2556] -- added logs and small
refactoring
This is an automated email from the ASF dual-hosted git repository.
ykinash pushed a commit to branch DATALAB-2556
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 5337e93ff4d633676fd2617324406948ed5d89fb
Author: KinashYurii <ur...@gmail.com>
AuthorDate: Thu Nov 4 16:36:10 2021 +0200
[DATALAB-2556] -- added logs and small refactoring
---
.../gcp/service/impl/BillingServiceImpl.java | 4 +-
.../epam/datalab/dto/status/EnvResourceList.java | 4 +
.../response/handlers/EdgeCallbackHandler.java | 1 +
.../response/handlers/ProjectCallbackHandler.java | 4 +-
.../response/handlers/ResourceCallbackHandler.java | 5 +-
.../resources/callback/ProjectCallback.java | 35 ++--
.../CheckInfrastructureStatusScheduler.java | 5 +-
.../service/impl/BillingServiceImpl.java | 17 +-
.../impl/InfrastructureInfoServiceImpl.java | 17 +-
.../epam/datalab/backendapi/util/BillingUtils.java | 202 +++++++++++++++------
.../datalab/backendapi/util/RequestBuilder.java | 3 +-
11 files changed, 207 insertions(+), 90 deletions(-)
diff --git a/services/billing-gcp/src/main/java/com/epam/datalab/billing/gcp/service/impl/BillingServiceImpl.java b/services/billing-gcp/src/main/java/com/epam/datalab/billing/gcp/service/impl/BillingServiceImpl.java
index b88caba..f766784 100644
--- a/services/billing-gcp/src/main/java/com/epam/datalab/billing/gcp/service/impl/BillingServiceImpl.java
+++ b/services/billing-gcp/src/main/java/com/epam/datalab/billing/gcp/service/impl/BillingServiceImpl.java
@@ -43,7 +43,9 @@ public class BillingServiceImpl implements BillingService {
@Override
public List<BillingData> getBillingData() {
try {
- return billingDAO.getBillingData();
+ List<BillingData> billingData = billingDAO.getBillingData();
+ log.info("TEST LOG BILLING: billingData: {}", billingData);
+ return billingData;
} catch (Exception e) {
log.error("Can not update billing due to: {}", e.getMessage(), e);
return Collections.emptyList();
diff --git a/services/datalab-model/src/main/java/com/epam/datalab/dto/status/EnvResourceList.java b/services/datalab-model/src/main/java/com/epam/datalab/dto/status/EnvResourceList.java
index 0db1930..620a071 100644
--- a/services/datalab-model/src/main/java/com/epam/datalab/dto/status/EnvResourceList.java
+++ b/services/datalab-model/src/main/java/com/epam/datalab/dto/status/EnvResourceList.java
@@ -46,6 +46,10 @@ public class EnvResourceList {
.add("cluster", clusterList);
}
+ public boolean isEmpty(){
+ return hostList.isEmpty() && clusterList.isEmpty();
+ }
+
@Override
public String toString() {
return toStringHelper(this).toString();
diff --git a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/EdgeCallbackHandler.java b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/EdgeCallbackHandler.java
index 8ebec10..52df8ab 100644
--- a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/EdgeCallbackHandler.java
+++ b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/EdgeCallbackHandler.java
@@ -59,6 +59,7 @@ public class EdgeCallbackHandler<E extends EdgeInfo, T extends UploadFileResult<
}
protected T parseOutResponse(JsonNode resultNode, T baseStatus) {
+ log.info("TEST LOG!!!:resultNode: {}, base status: {} ", resultNode, baseStatus);
if (resultNode != null
&& (getAction() == DockerAction.CREATE || getAction() == DockerAction.START)
&& UserInstanceStatus.of(baseStatus.getStatus()) != UserInstanceStatus.FAILED) {
diff --git a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ProjectCallbackHandler.java b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ProjectCallbackHandler.java
index 1629c05..e8a6566 100644
--- a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ProjectCallbackHandler.java
+++ b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ProjectCallbackHandler.java
@@ -57,10 +57,12 @@ public class ProjectCallbackHandler extends ResourceCallbackHandler<ProjectResul
@Override
protected ProjectResult parseOutResponse(JsonNode resultNode, ProjectResult baseStatus) {
+ log.info("TEST LOG!!!: resultNode: {} , baseStatus: {}", resultNode, baseStatus);
+
baseStatus.setProjectName(projectName);
baseStatus.setEndpointName(endpointName);
if (resultNode != null &&
- Arrays.asList(DockerAction.CREATE, DockerAction.RECREATE, DockerAction.START).contains(getAction()) &&
+ Arrays.asList(DockerAction.CREATE, DockerAction.RECREATE).contains(getAction()) &&
UserInstanceStatus.of(baseStatus.getStatus()) != UserInstanceStatus.FAILED) {
try {
final EdgeInfo projectEdgeInfo = mapper.readValue(resultNode.toString(), clazz);
diff --git a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ResourceCallbackHandler.java b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ResourceCallbackHandler.java
index 5a20b37..6999968 100644
--- a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ResourceCallbackHandler.java
+++ b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ResourceCallbackHandler.java
@@ -30,6 +30,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.Data;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,7 +40,8 @@ import java.util.Date;
public abstract class ResourceCallbackHandler<T extends StatusBaseDTO<?>> implements FileHandlerCallback {
private static final Logger log = LoggerFactory.getLogger(ResourceCallbackHandler.class);
- final ObjectMapper mapper = new ObjectMapper().configure(JsonParser.Feature.AUTO_CLOSE_SOURCE, true);
+ final ObjectMapper mapper = new ObjectMapper()
+ .configure(JsonParser.Feature.AUTO_CLOSE_SOURCE, true);
private static final String STATUS_FIELD = "status";
protected static final String RESPONSE_NODE = "response";
@@ -127,6 +129,7 @@ public abstract class ResourceCallbackHandler<T extends StatusBaseDTO<?>> implem
log.error("Could not {} resource for user: {}, UUID: {}", action, user, uuid);
result.setErrorMessage(getTextValue(resultNode.get(ERROR_NODE)));
}
+ log.info("TEST LOG!!!: resultNode: {}, result: {}", resultNode, result);
result = parseOutResponse(resultNode, result);
selfServicePost(result);
return !UserInstanceStatus.FAILED.equals(status);
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/callback/ProjectCallback.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/callback/ProjectCallback.java
index 07a327f..d5c521a 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/callback/ProjectCallback.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/callback/ProjectCallback.java
@@ -23,7 +23,6 @@ import com.epam.datalab.backendapi.dao.EndpointDAO;
import com.epam.datalab.backendapi.dao.GpuDAO;
import com.epam.datalab.backendapi.dao.ProjectDAO;
import com.epam.datalab.backendapi.domain.RequestId;
-import com.epam.datalab.backendapi.schedulers.CheckInfrastructureStatusScheduler;
import com.epam.datalab.backendapi.service.ExploratoryService;
import com.epam.datalab.dto.UserInstanceStatus;
import com.epam.datalab.dto.base.project.ProjectResult;
@@ -49,39 +48,31 @@ public class ProjectCallback {
private final ExploratoryService exploratoryService;
private final RequestId requestId;
private final GpuDAO gpuDAO;
- private final CheckInfrastructureStatusScheduler scheduler;
@Inject
public ProjectCallback(ProjectDAO projectDAO, EndpointDAO endpointDAO, ExploratoryService exploratoryService, RequestId requestId,
- GpuDAO gpuDAO, CheckInfrastructureStatusScheduler scheduler) {
+ GpuDAO gpuDAO) {
this.projectDAO = projectDAO;
this.exploratoryService = exploratoryService;
this.requestId = requestId;
this.gpuDAO = gpuDAO;
- this.scheduler = scheduler;
}
@POST
public Response updateProjectStatus(ProjectResult projectResult) {
- try {
- requestId.checkAndRemove(projectResult.getRequestId());
- final String projectName = projectResult.getProjectName();
- final UserInstanceStatus status = UserInstanceStatus.of(projectResult.getStatus());
- if (projectResult.getEdgeInfo() != null) {
- saveGpuForProject(projectResult, projectName);
- }
- if (UserInstanceStatus.RUNNING == status && Objects.nonNull(projectResult.getEdgeInfo())) {
- projectDAO.updateEdgeInfo(projectName, projectResult.getEndpointName(), projectResult.getEdgeInfo());
- } else {
- updateExploratoriesStatusIfNeeded(status, projectResult.getProjectName(), projectResult.getEndpointName());
- projectDAO.updateEdgeStatus(projectName, projectResult.getEndpointName(), status);
- }
- } catch (Exception e) {
- log.error(e.toString());
- log.error(e.getMessage());
- log.info("Run scheduler");
- scheduler.execute(null);
+ log.info("TEST LOG!!!: projectResult: {}", projectResult);
+ requestId.checkAndRemove(projectResult.getRequestId());
+ final String projectName = projectResult.getProjectName();
+ final UserInstanceStatus status = UserInstanceStatus.of(projectResult.getStatus());
+ if (projectResult.getEdgeInfo() != null) {
+ saveGpuForProject(projectResult, projectName);
+ }
+ if (UserInstanceStatus.RUNNING == status && Objects.nonNull(projectResult.getEdgeInfo())) {
+ projectDAO.updateEdgeInfo(projectName, projectResult.getEndpointName(), projectResult.getEdgeInfo());
+ } else {
+ updateExploratoriesStatusIfNeeded(status, projectResult.getProjectName(), projectResult.getEndpointName());
+ projectDAO.updateEdgeStatus(projectName, projectResult.getEndpointName(), status);
}
return Response.ok().build();
}
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/CheckInfrastructureStatusScheduler.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/CheckInfrastructureStatusScheduler.java
index 91dda92..5d593db 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/CheckInfrastructureStatusScheduler.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/CheckInfrastructureStatusScheduler.java
@@ -51,7 +51,7 @@ import static com.epam.datalab.dto.UserInstanceStatus.*;
public class CheckInfrastructureStatusScheduler implements Job {
private static final List<UserInstanceStatus> statusesToCheck =
- Arrays.asList(CREATING, RUNNING, STOPPING, RECONFIGURING, STOPPED, TERMINATING, TERMINATED);
+ Arrays.asList(STARTING, CREATING, RUNNING, STOPPING, RECONFIGURING, STOPPED, TERMINATING, TERMINATED);
private final InfrastructureInfoService infrastructureInfoService;
private final SecurityService securityService;
@@ -148,7 +148,8 @@ public class CheckInfrastructureStatusScheduler implements Job {
}
private boolean noEmrCreating(UserComputationalResource c) {
- return ! (c.getStatus().equals(CREATING.name()) && c.getTemplateName().contains(AWS_EMR_CLUSTER));
+ log.info("TEST LOG!!!: status: {}, template name: {}", c.getStatus(), c.getTemplateName());
+ return !(c.getStatus().equals(CREATING.name()) && c.getTemplateName().contains(AWS_EMR_CLUSTER));
}
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/BillingServiceImpl.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/BillingServiceImpl.java
index df800e0..e65901a 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/BillingServiceImpl.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/BillingServiceImpl.java
@@ -22,6 +22,7 @@ package com.epam.datalab.backendapi.service.impl;
import com.epam.datalab.auth.UserInfo;
import com.epam.datalab.backendapi.conf.SelfServiceApplicationConfiguration;
import com.epam.datalab.backendapi.dao.BillingDAO;
+import com.epam.datalab.backendapi.dao.ExploratoryDAO;
import com.epam.datalab.backendapi.dao.ImageExploratoryDAO;
import com.epam.datalab.backendapi.dao.ProjectDAO;
import com.epam.datalab.backendapi.domain.*;
@@ -72,12 +73,13 @@ public class BillingServiceImpl implements BillingService {
private final RESTService billingService;
private final ImageExploratoryDAO imageExploratoryDao;
private final BillingDAO billingDAO;
+ private final ExploratoryDAO exploratoryDAO;
@Inject
public BillingServiceImpl(ProjectService projectService, ProjectDAO projectDAO, EndpointService endpointService,
ExploratoryService exploratoryService, SelfServiceApplicationConfiguration configuration,
@Named(ServiceConsts.BILLING_SERVICE_NAME) RESTService billingService,
- ImageExploratoryDAO imageExploratoryDao, BillingDAO billingDAO) {
+ ImageExploratoryDAO imageExploratoryDao, BillingDAO billingDAO, ExploratoryDAO exploratoryDAO) {
this.projectService = projectService;
this.projectDAO = projectDAO;
this.endpointService = endpointService;
@@ -86,6 +88,7 @@ public class BillingServiceImpl implements BillingService {
this.billingService = billingService;
this.imageExploratoryDao = imageExploratoryDao;
this.billingDAO = billingDAO;
+ this.exploratoryDAO = exploratoryDAO;
}
@Override
@@ -156,6 +159,8 @@ public class BillingServiceImpl implements BillingService {
.stream()
.collect(Collectors.toMap(e -> e, e -> getBillingData(userInfo, e)));
+ log.info("TEST LOG!!!: billingDataMap: {}", billingDataMap);
+
billingDataMap.forEach((endpointDTO, billingData) -> {
log.info("Updating billing information for endpoint {}. Billing data {}", endpointDTO.getName(), billingData);
if (!billingData.isEmpty()) {
@@ -209,10 +214,18 @@ public class BillingServiceImpl implements BillingService {
final Stream<BillingReportLine> billableSharedEndpoints = endpoints
.stream()
.flatMap(endpoint -> BillingUtils.sharedEndpointBillingDataStream(endpoint.getName(), configuration.getServiceBaseName()));
+
+ log.info("TEST LOG!!!: userInstance from service: {}", exploratoryService.findAll(projects));
+ log.info("TEST LOG!!!: userInstance from DB: {}", exploratoryDAO.fetchExploratoryFieldsForProject("proj"));
+
+
final Stream<BillingReportLine> billableUserInstances = exploratoryService.findAll(projects)
.stream()
.filter(userInstance -> Objects.nonNull(userInstance.getExploratoryId()))
.flatMap(ui -> BillingUtils.exploratoryBillingDataStream(ui, configuration.getMaxSparkInstanceCount()));
+
+ log.info("TEST LOG!!!: billableUserInstances: {}", billableUserInstances.collect(Collectors.toList()));
+
final Stream<BillingReportLine> customImages = projects
.stream()
.map(p -> imageExploratoryDao.getImagesForProject(p.getName()))
@@ -222,7 +235,7 @@ public class BillingServiceImpl implements BillingService {
final Map<String, BillingReportLine> billableResources = Stream.of(ssnBillingDataStream, billableEdges, billableSharedEndpoints, billableUserInstances, customImages)
.flatMap(s -> s)
.collect(Collectors.toMap(BillingReportLine::getDatalabId, b -> b));
- log.debug("Billable resources are: {}", billableResources);
+ log.info("Billable resources are: {}", billableResources);
return billableResources;
}
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/InfrastructureInfoServiceImpl.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/InfrastructureInfoServiceImpl.java
index 5ed48f6..4ea885b 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/InfrastructureInfoServiceImpl.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/InfrastructureInfoServiceImpl.java
@@ -146,12 +146,16 @@ public class InfrastructureInfoServiceImpl implements InfrastructureInfoService
.build();
EndpointDTO endpointDTO = endpointService.get(endpoint);
- log.info("Send request to provisioning service:\n POST:{}, with EnvResources: {}", INFRASTRUCTURE_STATUS,
- envResourceList);
- String uuid = provisioningService.post(endpointDTO.getUrl() + INFRASTRUCTURE_STATUS, user.getAccessToken(),
- requestBuilder.newInfrastructureStatus(user.getName(), endpointDTO.getCloudProvider(), envResourceList),
- String.class);
- requestId.put(user.getName(), uuid);
+ if (envResourceList.isEmpty()) {
+ log.info("EnvResources is empty: {} , didn't send request to provisioning service", envResourceList);
+ } else {
+ log.info("Send request to provisioning service:\n POST:{}, with EnvResources: {}", INFRASTRUCTURE_STATUS,
+ envResourceList);
+ String uuid = provisioningService.post(endpointDTO.getUrl() + INFRASTRUCTURE_STATUS, user.getAccessToken(),
+ requestBuilder.newInfrastructureStatus(user.getName(), endpointDTO.getCloudProvider(), envResourceList),
+ String.class);
+ requestId.put(user.getName(), uuid);
+ }
}
private List<BillingReport> getExploratoryBillingData(List<UserInstanceDTO> exploratories) {
@@ -192,6 +196,7 @@ public class InfrastructureInfoServiceImpl implements InfrastructureInfoService
shared.put("status", endpointDTO.getStatus().toString());
shared.put("edge_node_ip", edge.getPublicIp());
+ log.info("TEST LOG!!!: edge: {}", edge);
if (edge instanceof EdgeInfoAws) {
EdgeInfoAws edgeInfoAws = (EdgeInfoAws) edge;
shared.put("user_own_bicket_name", edgeInfoAws.getUserOwnBucketName());
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/util/BillingUtils.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/util/BillingUtils.java
index 83f6774..1e8b31c 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/util/BillingUtils.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/util/BillingUtils.java
@@ -26,32 +26,24 @@ import com.epam.datalab.dto.UserInstanceStatus;
import com.epam.datalab.dto.base.DataEngineType;
import com.epam.datalab.dto.computational.UserComputationalResource;
import jersey.repackaged.com.google.common.collect.Lists;
+import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.time.format.FormatStyle;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Locale;
-import java.util.Objects;
-import java.util.Optional;
+import java.util.*;
+import java.util.stream.Collectors;
import java.util.stream.Stream;
-import static com.epam.datalab.dto.billing.BillingResourceType.BUCKET;
-import static com.epam.datalab.dto.billing.BillingResourceType.COMPUTATIONAL;
-import static com.epam.datalab.dto.billing.BillingResourceType.EDGE;
-import static com.epam.datalab.dto.billing.BillingResourceType.ENDPOINT;
-import static com.epam.datalab.dto.billing.BillingResourceType.EXPLORATORY;
-import static com.epam.datalab.dto.billing.BillingResourceType.IMAGE;
-import static com.epam.datalab.dto.billing.BillingResourceType.SSN;
-import static com.epam.datalab.dto.billing.BillingResourceType.VOLUME;
+import static com.epam.datalab.dto.billing.BillingResourceType.*;
+@Slf4j
public class BillingUtils {
private static final String[] AVAILABLE_NOTEBOOKS = {"zeppelin", "tensor-rstudio", "rstudio", "tensor", "superset", "jupyterlab", "jupyter", "deeplearning"};
private static final String[] BILLING_FILTERED_REPORT_HEADERS = {"DataLab ID", "Project", "DataLab Resource Type", "Status", "Shape", "Product", "Cost"};
private static final String[] COMPLETE_REPORT_REPORT_HEADERS = {"DataLab ID", "User", "Project", "DataLab Resource Type", "Status", "Shape", "Product", "Cost"};
+
private static final String REPORT_FIRST_LINE = "Service base name: %s. Available reporting period from: %s to: %s";
private static final String TOTAL_LINE = "Total: %s %s";
private static final String SSN_FORMAT = "%s-ssn";
@@ -60,10 +52,12 @@ public class BillingUtils {
private static final String EDGE_VOLUME_FORMAT = "%s-%s-%s-edge-volume-primary";
private static final String PROJECT_ENDPOINT_BUCKET_FORMAT = "%s-%s-%s-bucket";
private static final String ENDPOINT_SHARED_BUCKET_FORMAT = "%s-%s-shared-bucket";
+
private static final String VOLUME_PRIMARY_FORMAT = "%s-volume-primary";
private static final String VOLUME_PRIMARY_COMPUTATIONAL_FORMAT = "%s-%s-volume-primary";
private static final String VOLUME_SECONDARY_FORMAT = "%s-volume-secondary";
private static final String VOLUME_SECONDARY_COMPUTATIONAL_FORMAT = "%s-%s-volume-secondary";
+
private static final String IMAGE_STANDARD_FORMAT1 = "%s-%s-%s-%s-notebook-image";
private static final String IMAGE_STANDARD_FORMAT2 = "%s-%s-%s-notebook-image";
private static final String IMAGE_CUSTOM_FORMAT = "%s-%s-%s-%s-%s";
@@ -80,9 +74,27 @@ public class BillingUtils {
final String endpointBucketId = String.format(PROJECT_ENDPOINT_BUCKET_FORMAT, sbn, project, endpoint).toLowerCase();
return Stream.concat(Stream.of(
- BillingReportLine.builder().resourceName(endpoint).user(SHARED_RESOURCE).project(project).datalabId(userEdgeId).resourceType(EDGE).build(),
- BillingReportLine.builder().resourceName("EDGE volume").user(SHARED_RESOURCE).project(project).datalabId(edgeVolumeId).resourceType(VOLUME).build(),
- BillingReportLine.builder().resourceName("Project endpoint shared bucket").user(SHARED_RESOURCE).project(project).datalabId(endpointBucketId).resourceType(BUCKET).build()
+ BillingReportLine.builder()
+ .resourceName(endpoint)
+ .user(SHARED_RESOURCE)
+ .project(project)
+ .datalabId(userEdgeId)
+ .resourceType(EDGE)
+ .build(),
+ BillingReportLine.builder()
+ .resourceName("EDGE volume")
+ .user(SHARED_RESOURCE)
+ .project(project)
+ .datalabId(edgeVolumeId)
+ .resourceType(VOLUME)
+ .build(),
+ BillingReportLine.builder()
+ .resourceName("Project endpoint shared bucket")
+ .user(SHARED_RESOURCE)
+ .project(project)
+ .datalabId(endpointBucketId)
+ .resourceType(BUCKET)
+ .build()
),
standardImageBillingDataStream(sbn, project, endpoint)
);
@@ -91,8 +103,20 @@ public class BillingUtils {
public static Stream<BillingReportLine> ssnBillingDataStream(String sbn) {
final String ssnId = String.format(SSN_FORMAT, sbn);
return Stream.of(
- BillingReportLine.builder().user(SHARED_RESOURCE).project(SHARED_RESOURCE).resourceName("SSN").datalabId(ssnId).resourceType(SSN).build(),
- BillingReportLine.builder().user(SHARED_RESOURCE).project(SHARED_RESOURCE).resourceName("SSN Volume").datalabId(String.format(VOLUME_PRIMARY_FORMAT, ssnId)).resourceType(VOLUME).build()
+ BillingReportLine
+ .builder()
+ .user(SHARED_RESOURCE)
+ .project(SHARED_RESOURCE)
+ .resourceName("SSN")
+ .datalabId(ssnId)
+ .resourceType(SSN)
+ .build(),
+ BillingReportLine.builder()
+ .user(SHARED_RESOURCE)
+ .project(SHARED_RESOURCE)
+ .resourceName("SSN Volume")
+ .datalabId(String.format(VOLUME_PRIMARY_FORMAT, ssnId))
+ .resourceType(VOLUME).build()
);
}
@@ -100,39 +124,93 @@ public class BillingUtils {
final String projectEndpointBucketId = String.format(ENDPOINT_SHARED_BUCKET_FORMAT, sbn, endpoint).toLowerCase();
final String endpointId = String.format(ENDPOINT_FORMAT, sbn, endpoint).toLowerCase();
return Stream.concat(Stream.of(
- BillingReportLine.builder().resourceName("Endpoint shared bucket").user(SHARED_RESOURCE).project(SHARED_RESOURCE).datalabId(projectEndpointBucketId).resourceType(BUCKET).build(),
- BillingReportLine.builder().resourceName("Endpoint").user(SHARED_RESOURCE).project(SHARED_RESOURCE).datalabId(endpointId).resourceType(ENDPOINT).build()
+ BillingReportLine.builder()
+ .resourceName("Endpoint shared bucket")
+ .user(SHARED_RESOURCE)
+ .project(SHARED_RESOURCE)
+ .datalabId(projectEndpointBucketId)
+ .resourceType(BUCKET)
+ .build(),
+ BillingReportLine.builder()
+ .resourceName("Endpoint")
+ .user(SHARED_RESOURCE)
+ .project(SHARED_RESOURCE)
+ .datalabId(endpointId)
+ .resourceType(ENDPOINT).build()
),
standardImageBillingDataStream(sbn, endpoint));
}
public static Stream<BillingReportLine> exploratoryBillingDataStream(UserInstanceDTO userInstance, Integer maxSparkInstanceCount) {
+ log.info("TEST LOG!!!: exploratoryBillingDataStream");
+ log.info("TEST LOG!!!: userInstance: {}", userInstance);
+
+
final Stream<BillingReportLine> computationalStream = userInstance.getResources()
.stream()
.filter(cr -> cr.getComputationalId() != null)
.flatMap(cr -> {
final String computationalId = cr.getComputationalId().toLowerCase();
return Stream.concat(Stream.of(
- withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).datalabId(computationalId).resourceType(COMPUTATIONAL).shape(getComputationalShape(cr))
- .exploratoryName(userInstance.getExploratoryName()).build(),
- withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).datalabId(String.format(VOLUME_PRIMARY_FORMAT, computationalId)).resourceType(VOLUME).build(),
- withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).datalabId(String.format(VOLUME_SECONDARY_FORMAT, computationalId)).resourceType(VOLUME).build(),
- withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).datalabId(String.format(VOLUME_PRIMARY_COMPUTATIONAL_FORMAT, computationalId, "m"))
- .resourceType(VOLUME).build(),
- withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).datalabId(String.format(VOLUME_SECONDARY_COMPUTATIONAL_FORMAT, computationalId, "m"))
- .resourceType(VOLUME).build()
+ withUserProjectEndpoint(userInstance)
+ .resourceName(cr.getComputationalName())
+ .datalabId(computationalId)
+ .resourceType(COMPUTATIONAL)
+ .shape(getComputationalShape(cr))
+ .exploratoryName(userInstance.getExploratoryName())
+ .build(),
+ withUserProjectEndpoint(userInstance)
+ .resourceName(cr.getComputationalName())
+ .datalabId(String.format(VOLUME_PRIMARY_FORMAT, computationalId))
+ .resourceType(VOLUME)
+ .build(),
+ withUserProjectEndpoint(userInstance)
+ .resourceName(cr.getComputationalName())
+ .datalabId(String.format(VOLUME_SECONDARY_FORMAT, computationalId))
+ .resourceType(VOLUME)
+ .build(),
+ withUserProjectEndpoint(userInstance)
+ .resourceName(cr.getComputationalName())
+ .datalabId(String.format(VOLUME_PRIMARY_COMPUTATIONAL_FORMAT, computationalId, "m"))
+ .resourceType(VOLUME)
+ .build(),
+ withUserProjectEndpoint(userInstance)
+ .resourceName(cr.getComputationalName())
+ .datalabId(String.format(VOLUME_SECONDARY_COMPUTATIONAL_FORMAT, computationalId, "m"))
+ .resourceType(VOLUME)
+ .build()
),
getSlaveVolumes(userInstance, cr, maxSparkInstanceCount)
);
});
+ log.info("TEST LOG!!!: computationalStream: {}", computationalStream);
+ log.info("TEST LOG!!!: computationalStream: {}", computationalStream.collect(Collectors.toList()));
+
+
final String exploratoryName = userInstance.getExploratoryName();
final String exploratoryId = userInstance.getExploratoryId().toLowerCase();
final String primaryVolumeId = String.format(VOLUME_PRIMARY_FORMAT, exploratoryId);
final String secondaryVolumeId = String.format(VOLUME_SECONDARY_FORMAT, exploratoryId);
final Stream<BillingReportLine> exploratoryStream = Stream.of(
- withUserProjectEndpoint(userInstance).resourceName(exploratoryName).datalabId(exploratoryId).resourceType(EXPLORATORY).shape(userInstance.getShape()).build(),
- withUserProjectEndpoint(userInstance).resourceName(exploratoryName).datalabId(primaryVolumeId).resourceType(VOLUME).build(),
- withUserProjectEndpoint(userInstance).resourceName(exploratoryName).datalabId(secondaryVolumeId).resourceType(VOLUME).build());
+ withUserProjectEndpoint(userInstance)
+ .resourceName(exploratoryName)
+ .datalabId(exploratoryId)
+ .resourceType(EXPLORATORY)
+ .shape(userInstance.getShape())
+ .build(),
+ withUserProjectEndpoint(userInstance)
+ .resourceName(exploratoryName)
+ .datalabId(primaryVolumeId)
+ .resourceType(VOLUME)
+ .build(),
+ withUserProjectEndpoint(userInstance)
+ .resourceName(exploratoryName)
+ .datalabId(secondaryVolumeId)
+ .resourceType(VOLUME)
+ .build());
+
+ log.info("TEST LOG!!!: exploratoryStream: {}", exploratoryStream);
+ log.info("TEST LOG!!!: exploratoryStream: {}", exploratoryStream.collect(Collectors.toList()));
return Stream.concat(computationalStream, exploratoryStream);
}
@@ -156,10 +234,19 @@ public class BillingUtils {
}
private static BillingReportLine.BillingReportLineBuilder withUserProjectEndpoint(UserInstanceDTO userInstance) {
- return BillingReportLine.builder().user(userInstance.getUser()).project(userInstance.getProject()).endpoint(userInstance.getEndpoint());
+ return BillingReportLine.builder()
+ .user(userInstance.getUser())
+ .project(userInstance.getProject())
+ .endpoint(userInstance.getEndpoint());
}
public static String getComputationalShape(UserComputationalResource resource) {
+ log.info("TEST LOG!!! getComputationalShape");
+ log.info("TEST LOG!!! UserComputationalResource: {}", resource);
+ log.info("TEST LOG!!! getTemplateName: {}", resource.getTemplateName());
+ log.info("TEST LOG!!! getDataengineShape: {}", resource.getDataengineShape());
+ log.info("TEST LOG!!! getMasterNodeShape: {}", resource.getMasterNodeShape());
+
return DataEngineType.fromDockerImageName(resource.getImageName()) == DataEngineType.SPARK_STANDALONE ?
String.format(DATAENGINE_NAME_FORMAT, resource.getDataengineInstanceCount(), resource.getDataengineShape()) :
String.format(DATAENGINE_SERVICE_NAME_FORMAT, resource.getMasterNodeShape(), resource.getTotalInstanceCount() - 1, resource.getSlaveNodeShape());
@@ -168,37 +255,46 @@ public class BillingUtils {
private static Stream<BillingReportLine> standardImageBillingDataStream(String sbn, String endpoint) {
List<BillingReportLine> list = new ArrayList<>();
for (String notebook : AVAILABLE_NOTEBOOKS) {
- list.add(BillingReportLine.builder().resourceName(IMAGE_NAME).datalabId(String.format(IMAGE_STANDARD_FORMAT2, sbn, endpoint, notebook).toLowerCase())
- .user(SHARED_RESOURCE).project(SHARED_RESOURCE).resourceType(IMAGE).build());
+ list.add(BillingReportLine.builder()
+ .resourceName(IMAGE_NAME)
+ .datalabId(String.format(IMAGE_STANDARD_FORMAT2, sbn, endpoint, notebook).toLowerCase())
+ .user(SHARED_RESOURCE)
+ .project(SHARED_RESOURCE)
+ .resourceType(IMAGE)
+ .build());
}
-
return list.stream();
}
private static Stream<BillingReportLine> standardImageBillingDataStream(String sbn, String project, String endpoint) {
List<BillingReportLine> list = new ArrayList<>();
for (String notebook : AVAILABLE_NOTEBOOKS) {
- list.add(BillingReportLine.builder().resourceName(IMAGE_NAME).datalabId(String.format(IMAGE_STANDARD_FORMAT1, sbn, project, endpoint, notebook).toLowerCase())
- .project(project).user(SHARED_RESOURCE).resourceType(IMAGE).build());
+ list.add(BillingReportLine
+ .builder()
+ .resourceName(IMAGE_NAME)
+ .datalabId(String.format(IMAGE_STANDARD_FORMAT1, sbn, project, endpoint, notebook).toLowerCase())
+ .project(project)
+ .user(SHARED_RESOURCE)
+ .resourceType(IMAGE)
+ .build());
}
-
return list.stream();
}
- /**
- * @param sbn Service Base Name
- * @param from formatted date, like 2020-04-07
- * @param to formatted date, like 2020-05-07
- * @param locale user's locale
- * @return line, like:
- * "Service base name: SERVICE_BASE_NAME. Available reporting period from: 2020-04-07 to: 2020-04-07"
- */
- public static String getFirstLine(String sbn, LocalDate from, LocalDate to, String locale) {
- return CSVFormatter.formatLine(Lists.newArrayList(String.format(REPORT_FIRST_LINE, sbn,
- Optional.ofNullable(from).map(date -> date.format(DateTimeFormatter.ofLocalizedDate(FormatStyle.MEDIUM).withLocale(Locale.forLanguageTag(locale)))).orElse(StringUtils.EMPTY),
- Optional.ofNullable(to).map(date -> date.format(DateTimeFormatter.ofLocalizedDate(FormatStyle.MEDIUM).withLocale(Locale.forLanguageTag(locale)))).orElse(StringUtils.EMPTY))),
- CSVFormatter.SEPARATOR, '\"');
- }
+ /**
+ * @param sbn Service Base Name
+ * @param from formatted date, like 2020-04-07
+ * @param to formatted date, like 2020-05-07
+ * @param locale user's locale
+ * @return line, like:
+ * "Service base name: SERVICE_BASE_NAME. Available reporting period from: 2020-04-07 to: 2020-04-07"
+ */
+ public static String getFirstLine(String sbn, LocalDate from, LocalDate to, String locale) {
+ return CSVFormatter.formatLine(Lists.newArrayList(String.format(REPORT_FIRST_LINE, sbn,
+ Optional.ofNullable(from).map(date -> date.format(DateTimeFormatter.ofLocalizedDate(FormatStyle.MEDIUM).withLocale(Locale.forLanguageTag(locale)))).orElse(StringUtils.EMPTY),
+ Optional.ofNullable(to).map(date -> date.format(DateTimeFormatter.ofLocalizedDate(FormatStyle.MEDIUM).withLocale(Locale.forLanguageTag(locale)))).orElse(StringUtils.EMPTY))),
+ CSVFormatter.SEPARATOR, '\"');
+ }
/**
* headerType there are two types of header according user role
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java
index ad50429..7b1ea0a 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java
@@ -171,7 +171,7 @@ public class RequestBuilder {
}
- T t = exploratoryCreate.withExploratoryName(exploratory.getName())
+ return exploratoryCreate.withExploratoryName(exploratory.getName())
.withNotebookImage(exploratory.getDockerImage())
.withApplicationName(getApplicationNameFromImage(exploratory.getDockerImage()))
.withGitCreds(exploratoryGitCredsDTO.getGitCreds())
@@ -184,7 +184,6 @@ public class RequestBuilder {
.withGPUCount(exploratory.getGpuCount())
.withGPUType(exploratory.getGpuType())
.withEnabledGPU(exploratory.getEnabledGPU());
- return t;
}
@SuppressWarnings("unchecked")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org