You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by yk...@apache.org on 2021/11/08 19:05:15 UTC
[incubator-datalab] branch DATALAB-2556 created (now 5891d61)
This is an automated email from the ASF dual-hosted git repository.
ykinash pushed a change to branch DATALAB-2556
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git.
at 5891d61 [DATALAB-2563] -- fixed merge conf
This branch includes the following new commits:
new 5337e93 [DATALAB-2556] -- added logs and small refactoring
new 901a0f1 Merge branch 'develop' of https://github.com/apache/incubator-datalab into DATALAB-2556
new 5891d61 [DATALAB-2563] -- fixed merge conf
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 02/03: Merge branch 'develop' of
https://github.com/apache/incubator-datalab into DATALAB-2556
Posted by yk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ykinash pushed a commit to branch DATALAB-2556
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 901a0f10fad11e47ea3788d65c44d8501299de6e
Merge: 5337e93 1f56b71
Author: KinashYurii <ur...@gmail.com>
AuthorDate: Thu Nov 4 16:38:29 2021 +0200
Merge branch 'develop' of https://github.com/apache/incubator-datalab into DATALAB-2556
.../scripts/deploy_datalab.py | 3 -
.../scripts/deploy_repository/deploy_repository.py | 2 +-
.../src/base/scripts/install_prerequisites.py | 3 +
.../src/general/conf/datalab.ini | 5 +-
.../src/general/lib/os/debian/common_lib.py | 17 +-
.../src/general/lib/os/debian/notebook_lib.py | 4 +-
.../src/general/lib/os/fab.py | 6 +-
.../src/general/scripts/aws/rstudio_configure.py | 4 +-
.../general/templates/azure/interpreter_spark.json | 926 ++++++++++++++++-----
.../src/jupyterlab/Dockerfile_jupyterlab | 9 +-
.../zeppelin/scripts/configure_zeppelin_node.py | 27 +-
services/billing-azure/pom.xml | 14 +-
.../bucket-browser/bucket-browser.component.ts | 6 +-
13 files changed, 761 insertions(+), 265 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 01/03: [DATALAB-2556] -- added logs and small
refactoring
Posted by yk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ykinash pushed a commit to branch DATALAB-2556
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 5337e93ff4d633676fd2617324406948ed5d89fb
Author: KinashYurii <ur...@gmail.com>
AuthorDate: Thu Nov 4 16:36:10 2021 +0200
[DATALAB-2556] -- added logs and small refactoring
---
.../gcp/service/impl/BillingServiceImpl.java | 4 +-
.../epam/datalab/dto/status/EnvResourceList.java | 4 +
.../response/handlers/EdgeCallbackHandler.java | 1 +
.../response/handlers/ProjectCallbackHandler.java | 4 +-
.../response/handlers/ResourceCallbackHandler.java | 5 +-
.../resources/callback/ProjectCallback.java | 35 ++--
.../CheckInfrastructureStatusScheduler.java | 5 +-
.../service/impl/BillingServiceImpl.java | 17 +-
.../impl/InfrastructureInfoServiceImpl.java | 17 +-
.../epam/datalab/backendapi/util/BillingUtils.java | 202 +++++++++++++++------
.../datalab/backendapi/util/RequestBuilder.java | 3 +-
11 files changed, 207 insertions(+), 90 deletions(-)
diff --git a/services/billing-gcp/src/main/java/com/epam/datalab/billing/gcp/service/impl/BillingServiceImpl.java b/services/billing-gcp/src/main/java/com/epam/datalab/billing/gcp/service/impl/BillingServiceImpl.java
index b88caba..f766784 100644
--- a/services/billing-gcp/src/main/java/com/epam/datalab/billing/gcp/service/impl/BillingServiceImpl.java
+++ b/services/billing-gcp/src/main/java/com/epam/datalab/billing/gcp/service/impl/BillingServiceImpl.java
@@ -43,7 +43,9 @@ public class BillingServiceImpl implements BillingService {
@Override
public List<BillingData> getBillingData() {
try {
- return billingDAO.getBillingData();
+ List<BillingData> billingData = billingDAO.getBillingData();
+ log.info("TEST LOG BILLING: billingData: {}", billingData);
+ return billingData;
} catch (Exception e) {
log.error("Can not update billing due to: {}", e.getMessage(), e);
return Collections.emptyList();
diff --git a/services/datalab-model/src/main/java/com/epam/datalab/dto/status/EnvResourceList.java b/services/datalab-model/src/main/java/com/epam/datalab/dto/status/EnvResourceList.java
index 0db1930..620a071 100644
--- a/services/datalab-model/src/main/java/com/epam/datalab/dto/status/EnvResourceList.java
+++ b/services/datalab-model/src/main/java/com/epam/datalab/dto/status/EnvResourceList.java
@@ -46,6 +46,10 @@ public class EnvResourceList {
.add("cluster", clusterList);
}
+ public boolean isEmpty(){
+ return hostList.isEmpty() && clusterList.isEmpty();
+ }
+
@Override
public String toString() {
return toStringHelper(this).toString();
diff --git a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/EdgeCallbackHandler.java b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/EdgeCallbackHandler.java
index 8ebec10..52df8ab 100644
--- a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/EdgeCallbackHandler.java
+++ b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/EdgeCallbackHandler.java
@@ -59,6 +59,7 @@ public class EdgeCallbackHandler<E extends EdgeInfo, T extends UploadFileResult<
}
protected T parseOutResponse(JsonNode resultNode, T baseStatus) {
+ log.info("TEST LOG!!!:resultNode: {}, base status: {} ", resultNode, baseStatus);
if (resultNode != null
&& (getAction() == DockerAction.CREATE || getAction() == DockerAction.START)
&& UserInstanceStatus.of(baseStatus.getStatus()) != UserInstanceStatus.FAILED) {
diff --git a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ProjectCallbackHandler.java b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ProjectCallbackHandler.java
index 1629c05..e8a6566 100644
--- a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ProjectCallbackHandler.java
+++ b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ProjectCallbackHandler.java
@@ -57,10 +57,12 @@ public class ProjectCallbackHandler extends ResourceCallbackHandler<ProjectResul
@Override
protected ProjectResult parseOutResponse(JsonNode resultNode, ProjectResult baseStatus) {
+ log.info("TEST LOG!!!: resultNode: {} , baseStatus: {}", resultNode, baseStatus);
+
baseStatus.setProjectName(projectName);
baseStatus.setEndpointName(endpointName);
if (resultNode != null &&
- Arrays.asList(DockerAction.CREATE, DockerAction.RECREATE, DockerAction.START).contains(getAction()) &&
+ Arrays.asList(DockerAction.CREATE, DockerAction.RECREATE).contains(getAction()) &&
UserInstanceStatus.of(baseStatus.getStatus()) != UserInstanceStatus.FAILED) {
try {
final EdgeInfo projectEdgeInfo = mapper.readValue(resultNode.toString(), clazz);
diff --git a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ResourceCallbackHandler.java b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ResourceCallbackHandler.java
index 5a20b37..6999968 100644
--- a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ResourceCallbackHandler.java
+++ b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ResourceCallbackHandler.java
@@ -30,6 +30,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.Data;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,7 +40,8 @@ import java.util.Date;
public abstract class ResourceCallbackHandler<T extends StatusBaseDTO<?>> implements FileHandlerCallback {
private static final Logger log = LoggerFactory.getLogger(ResourceCallbackHandler.class);
- final ObjectMapper mapper = new ObjectMapper().configure(JsonParser.Feature.AUTO_CLOSE_SOURCE, true);
+ final ObjectMapper mapper = new ObjectMapper()
+ .configure(JsonParser.Feature.AUTO_CLOSE_SOURCE, true);
private static final String STATUS_FIELD = "status";
protected static final String RESPONSE_NODE = "response";
@@ -127,6 +129,7 @@ public abstract class ResourceCallbackHandler<T extends StatusBaseDTO<?>> implem
log.error("Could not {} resource for user: {}, UUID: {}", action, user, uuid);
result.setErrorMessage(getTextValue(resultNode.get(ERROR_NODE)));
}
+ log.info("TEST LOG!!!: resultNode: {}, result: {}", resultNode, result);
result = parseOutResponse(resultNode, result);
selfServicePost(result);
return !UserInstanceStatus.FAILED.equals(status);
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/callback/ProjectCallback.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/callback/ProjectCallback.java
index 07a327f..d5c521a 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/callback/ProjectCallback.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/callback/ProjectCallback.java
@@ -23,7 +23,6 @@ import com.epam.datalab.backendapi.dao.EndpointDAO;
import com.epam.datalab.backendapi.dao.GpuDAO;
import com.epam.datalab.backendapi.dao.ProjectDAO;
import com.epam.datalab.backendapi.domain.RequestId;
-import com.epam.datalab.backendapi.schedulers.CheckInfrastructureStatusScheduler;
import com.epam.datalab.backendapi.service.ExploratoryService;
import com.epam.datalab.dto.UserInstanceStatus;
import com.epam.datalab.dto.base.project.ProjectResult;
@@ -49,39 +48,31 @@ public class ProjectCallback {
private final ExploratoryService exploratoryService;
private final RequestId requestId;
private final GpuDAO gpuDAO;
- private final CheckInfrastructureStatusScheduler scheduler;
@Inject
public ProjectCallback(ProjectDAO projectDAO, EndpointDAO endpointDAO, ExploratoryService exploratoryService, RequestId requestId,
- GpuDAO gpuDAO, CheckInfrastructureStatusScheduler scheduler) {
+ GpuDAO gpuDAO) {
this.projectDAO = projectDAO;
this.exploratoryService = exploratoryService;
this.requestId = requestId;
this.gpuDAO = gpuDAO;
- this.scheduler = scheduler;
}
@POST
public Response updateProjectStatus(ProjectResult projectResult) {
- try {
- requestId.checkAndRemove(projectResult.getRequestId());
- final String projectName = projectResult.getProjectName();
- final UserInstanceStatus status = UserInstanceStatus.of(projectResult.getStatus());
- if (projectResult.getEdgeInfo() != null) {
- saveGpuForProject(projectResult, projectName);
- }
- if (UserInstanceStatus.RUNNING == status && Objects.nonNull(projectResult.getEdgeInfo())) {
- projectDAO.updateEdgeInfo(projectName, projectResult.getEndpointName(), projectResult.getEdgeInfo());
- } else {
- updateExploratoriesStatusIfNeeded(status, projectResult.getProjectName(), projectResult.getEndpointName());
- projectDAO.updateEdgeStatus(projectName, projectResult.getEndpointName(), status);
- }
- } catch (Exception e) {
- log.error(e.toString());
- log.error(e.getMessage());
- log.info("Run scheduler");
- scheduler.execute(null);
+ log.info("TEST LOG!!!: projectResult: {}", projectResult);
+ requestId.checkAndRemove(projectResult.getRequestId());
+ final String projectName = projectResult.getProjectName();
+ final UserInstanceStatus status = UserInstanceStatus.of(projectResult.getStatus());
+ if (projectResult.getEdgeInfo() != null) {
+ saveGpuForProject(projectResult, projectName);
+ }
+ if (UserInstanceStatus.RUNNING == status && Objects.nonNull(projectResult.getEdgeInfo())) {
+ projectDAO.updateEdgeInfo(projectName, projectResult.getEndpointName(), projectResult.getEdgeInfo());
+ } else {
+ updateExploratoriesStatusIfNeeded(status, projectResult.getProjectName(), projectResult.getEndpointName());
+ projectDAO.updateEdgeStatus(projectName, projectResult.getEndpointName(), status);
}
return Response.ok().build();
}
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/CheckInfrastructureStatusScheduler.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/CheckInfrastructureStatusScheduler.java
index 91dda92..5d593db 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/CheckInfrastructureStatusScheduler.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/CheckInfrastructureStatusScheduler.java
@@ -51,7 +51,7 @@ import static com.epam.datalab.dto.UserInstanceStatus.*;
public class CheckInfrastructureStatusScheduler implements Job {
private static final List<UserInstanceStatus> statusesToCheck =
- Arrays.asList(CREATING, RUNNING, STOPPING, RECONFIGURING, STOPPED, TERMINATING, TERMINATED);
+ Arrays.asList(STARTING, CREATING, RUNNING, STOPPING, RECONFIGURING, STOPPED, TERMINATING, TERMINATED);
private final InfrastructureInfoService infrastructureInfoService;
private final SecurityService securityService;
@@ -148,7 +148,8 @@ public class CheckInfrastructureStatusScheduler implements Job {
}
private boolean noEmrCreating(UserComputationalResource c) {
- return ! (c.getStatus().equals(CREATING.name()) && c.getTemplateName().contains(AWS_EMR_CLUSTER));
+ log.info("TEST LOG!!!: status: {}, template name: {}", c.getStatus(), c.getTemplateName());
+ return !(c.getStatus().equals(CREATING.name()) && c.getTemplateName().contains(AWS_EMR_CLUSTER));
}
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/BillingServiceImpl.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/BillingServiceImpl.java
index df800e0..e65901a 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/BillingServiceImpl.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/BillingServiceImpl.java
@@ -22,6 +22,7 @@ package com.epam.datalab.backendapi.service.impl;
import com.epam.datalab.auth.UserInfo;
import com.epam.datalab.backendapi.conf.SelfServiceApplicationConfiguration;
import com.epam.datalab.backendapi.dao.BillingDAO;
+import com.epam.datalab.backendapi.dao.ExploratoryDAO;
import com.epam.datalab.backendapi.dao.ImageExploratoryDAO;
import com.epam.datalab.backendapi.dao.ProjectDAO;
import com.epam.datalab.backendapi.domain.*;
@@ -72,12 +73,13 @@ public class BillingServiceImpl implements BillingService {
private final RESTService billingService;
private final ImageExploratoryDAO imageExploratoryDao;
private final BillingDAO billingDAO;
+ private final ExploratoryDAO exploratoryDAO;
@Inject
public BillingServiceImpl(ProjectService projectService, ProjectDAO projectDAO, EndpointService endpointService,
ExploratoryService exploratoryService, SelfServiceApplicationConfiguration configuration,
@Named(ServiceConsts.BILLING_SERVICE_NAME) RESTService billingService,
- ImageExploratoryDAO imageExploratoryDao, BillingDAO billingDAO) {
+ ImageExploratoryDAO imageExploratoryDao, BillingDAO billingDAO, ExploratoryDAO exploratoryDAO) {
this.projectService = projectService;
this.projectDAO = projectDAO;
this.endpointService = endpointService;
@@ -86,6 +88,7 @@ public class BillingServiceImpl implements BillingService {
this.billingService = billingService;
this.imageExploratoryDao = imageExploratoryDao;
this.billingDAO = billingDAO;
+ this.exploratoryDAO = exploratoryDAO;
}
@Override
@@ -156,6 +159,8 @@ public class BillingServiceImpl implements BillingService {
.stream()
.collect(Collectors.toMap(e -> e, e -> getBillingData(userInfo, e)));
+ log.info("TEST LOG!!!: billingDataMap: {}", billingDataMap);
+
billingDataMap.forEach((endpointDTO, billingData) -> {
log.info("Updating billing information for endpoint {}. Billing data {}", endpointDTO.getName(), billingData);
if (!billingData.isEmpty()) {
@@ -209,10 +214,18 @@ public class BillingServiceImpl implements BillingService {
final Stream<BillingReportLine> billableSharedEndpoints = endpoints
.stream()
.flatMap(endpoint -> BillingUtils.sharedEndpointBillingDataStream(endpoint.getName(), configuration.getServiceBaseName()));
+
+ log.info("TEST LOG!!!: userInstance from service: {}", exploratoryService.findAll(projects));
+ log.info("TEST LOG!!!: userInstance from DB: {}", exploratoryDAO.fetchExploratoryFieldsForProject("proj"));
+
+
final Stream<BillingReportLine> billableUserInstances = exploratoryService.findAll(projects)
.stream()
.filter(userInstance -> Objects.nonNull(userInstance.getExploratoryId()))
.flatMap(ui -> BillingUtils.exploratoryBillingDataStream(ui, configuration.getMaxSparkInstanceCount()));
+
+ log.info("TEST LOG!!!: billableUserInstances: {}", billableUserInstances.collect(Collectors.toList()));
+
final Stream<BillingReportLine> customImages = projects
.stream()
.map(p -> imageExploratoryDao.getImagesForProject(p.getName()))
@@ -222,7 +235,7 @@ public class BillingServiceImpl implements BillingService {
final Map<String, BillingReportLine> billableResources = Stream.of(ssnBillingDataStream, billableEdges, billableSharedEndpoints, billableUserInstances, customImages)
.flatMap(s -> s)
.collect(Collectors.toMap(BillingReportLine::getDatalabId, b -> b));
- log.debug("Billable resources are: {}", billableResources);
+ log.info("Billable resources are: {}", billableResources);
return billableResources;
}
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/InfrastructureInfoServiceImpl.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/InfrastructureInfoServiceImpl.java
index 5ed48f6..4ea885b 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/InfrastructureInfoServiceImpl.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/InfrastructureInfoServiceImpl.java
@@ -146,12 +146,16 @@ public class InfrastructureInfoServiceImpl implements InfrastructureInfoService
.build();
EndpointDTO endpointDTO = endpointService.get(endpoint);
- log.info("Send request to provisioning service:\n POST:{}, with EnvResources: {}", INFRASTRUCTURE_STATUS,
- envResourceList);
- String uuid = provisioningService.post(endpointDTO.getUrl() + INFRASTRUCTURE_STATUS, user.getAccessToken(),
- requestBuilder.newInfrastructureStatus(user.getName(), endpointDTO.getCloudProvider(), envResourceList),
- String.class);
- requestId.put(user.getName(), uuid);
+ if (envResourceList.isEmpty()) {
+ log.info("EnvResources is empty: {} , didn't send request to provisioning service", envResourceList);
+ } else {
+ log.info("Send request to provisioning service:\n POST:{}, with EnvResources: {}", INFRASTRUCTURE_STATUS,
+ envResourceList);
+ String uuid = provisioningService.post(endpointDTO.getUrl() + INFRASTRUCTURE_STATUS, user.getAccessToken(),
+ requestBuilder.newInfrastructureStatus(user.getName(), endpointDTO.getCloudProvider(), envResourceList),
+ String.class);
+ requestId.put(user.getName(), uuid);
+ }
}
private List<BillingReport> getExploratoryBillingData(List<UserInstanceDTO> exploratories) {
@@ -192,6 +196,7 @@ public class InfrastructureInfoServiceImpl implements InfrastructureInfoService
shared.put("status", endpointDTO.getStatus().toString());
shared.put("edge_node_ip", edge.getPublicIp());
+ log.info("TEST LOG!!!: edge: {}", edge);
if (edge instanceof EdgeInfoAws) {
EdgeInfoAws edgeInfoAws = (EdgeInfoAws) edge;
shared.put("user_own_bicket_name", edgeInfoAws.getUserOwnBucketName());
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/util/BillingUtils.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/util/BillingUtils.java
index 83f6774..1e8b31c 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/util/BillingUtils.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/util/BillingUtils.java
@@ -26,32 +26,24 @@ import com.epam.datalab.dto.UserInstanceStatus;
import com.epam.datalab.dto.base.DataEngineType;
import com.epam.datalab.dto.computational.UserComputationalResource;
import jersey.repackaged.com.google.common.collect.Lists;
+import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.time.format.FormatStyle;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Locale;
-import java.util.Objects;
-import java.util.Optional;
+import java.util.*;
+import java.util.stream.Collectors;
import java.util.stream.Stream;
-import static com.epam.datalab.dto.billing.BillingResourceType.BUCKET;
-import static com.epam.datalab.dto.billing.BillingResourceType.COMPUTATIONAL;
-import static com.epam.datalab.dto.billing.BillingResourceType.EDGE;
-import static com.epam.datalab.dto.billing.BillingResourceType.ENDPOINT;
-import static com.epam.datalab.dto.billing.BillingResourceType.EXPLORATORY;
-import static com.epam.datalab.dto.billing.BillingResourceType.IMAGE;
-import static com.epam.datalab.dto.billing.BillingResourceType.SSN;
-import static com.epam.datalab.dto.billing.BillingResourceType.VOLUME;
+import static com.epam.datalab.dto.billing.BillingResourceType.*;
+@Slf4j
public class BillingUtils {
private static final String[] AVAILABLE_NOTEBOOKS = {"zeppelin", "tensor-rstudio", "rstudio", "tensor", "superset", "jupyterlab", "jupyter", "deeplearning"};
private static final String[] BILLING_FILTERED_REPORT_HEADERS = {"DataLab ID", "Project", "DataLab Resource Type", "Status", "Shape", "Product", "Cost"};
private static final String[] COMPLETE_REPORT_REPORT_HEADERS = {"DataLab ID", "User", "Project", "DataLab Resource Type", "Status", "Shape", "Product", "Cost"};
+
private static final String REPORT_FIRST_LINE = "Service base name: %s. Available reporting period from: %s to: %s";
private static final String TOTAL_LINE = "Total: %s %s";
private static final String SSN_FORMAT = "%s-ssn";
@@ -60,10 +52,12 @@ public class BillingUtils {
private static final String EDGE_VOLUME_FORMAT = "%s-%s-%s-edge-volume-primary";
private static final String PROJECT_ENDPOINT_BUCKET_FORMAT = "%s-%s-%s-bucket";
private static final String ENDPOINT_SHARED_BUCKET_FORMAT = "%s-%s-shared-bucket";
+
private static final String VOLUME_PRIMARY_FORMAT = "%s-volume-primary";
private static final String VOLUME_PRIMARY_COMPUTATIONAL_FORMAT = "%s-%s-volume-primary";
private static final String VOLUME_SECONDARY_FORMAT = "%s-volume-secondary";
private static final String VOLUME_SECONDARY_COMPUTATIONAL_FORMAT = "%s-%s-volume-secondary";
+
private static final String IMAGE_STANDARD_FORMAT1 = "%s-%s-%s-%s-notebook-image";
private static final String IMAGE_STANDARD_FORMAT2 = "%s-%s-%s-notebook-image";
private static final String IMAGE_CUSTOM_FORMAT = "%s-%s-%s-%s-%s";
@@ -80,9 +74,27 @@ public class BillingUtils {
final String endpointBucketId = String.format(PROJECT_ENDPOINT_BUCKET_FORMAT, sbn, project, endpoint).toLowerCase();
return Stream.concat(Stream.of(
- BillingReportLine.builder().resourceName(endpoint).user(SHARED_RESOURCE).project(project).datalabId(userEdgeId).resourceType(EDGE).build(),
- BillingReportLine.builder().resourceName("EDGE volume").user(SHARED_RESOURCE).project(project).datalabId(edgeVolumeId).resourceType(VOLUME).build(),
- BillingReportLine.builder().resourceName("Project endpoint shared bucket").user(SHARED_RESOURCE).project(project).datalabId(endpointBucketId).resourceType(BUCKET).build()
+ BillingReportLine.builder()
+ .resourceName(endpoint)
+ .user(SHARED_RESOURCE)
+ .project(project)
+ .datalabId(userEdgeId)
+ .resourceType(EDGE)
+ .build(),
+ BillingReportLine.builder()
+ .resourceName("EDGE volume")
+ .user(SHARED_RESOURCE)
+ .project(project)
+ .datalabId(edgeVolumeId)
+ .resourceType(VOLUME)
+ .build(),
+ BillingReportLine.builder()
+ .resourceName("Project endpoint shared bucket")
+ .user(SHARED_RESOURCE)
+ .project(project)
+ .datalabId(endpointBucketId)
+ .resourceType(BUCKET)
+ .build()
),
standardImageBillingDataStream(sbn, project, endpoint)
);
@@ -91,8 +103,20 @@ public class BillingUtils {
public static Stream<BillingReportLine> ssnBillingDataStream(String sbn) {
final String ssnId = String.format(SSN_FORMAT, sbn);
return Stream.of(
- BillingReportLine.builder().user(SHARED_RESOURCE).project(SHARED_RESOURCE).resourceName("SSN").datalabId(ssnId).resourceType(SSN).build(),
- BillingReportLine.builder().user(SHARED_RESOURCE).project(SHARED_RESOURCE).resourceName("SSN Volume").datalabId(String.format(VOLUME_PRIMARY_FORMAT, ssnId)).resourceType(VOLUME).build()
+ BillingReportLine
+ .builder()
+ .user(SHARED_RESOURCE)
+ .project(SHARED_RESOURCE)
+ .resourceName("SSN")
+ .datalabId(ssnId)
+ .resourceType(SSN)
+ .build(),
+ BillingReportLine.builder()
+ .user(SHARED_RESOURCE)
+ .project(SHARED_RESOURCE)
+ .resourceName("SSN Volume")
+ .datalabId(String.format(VOLUME_PRIMARY_FORMAT, ssnId))
+ .resourceType(VOLUME).build()
);
}
@@ -100,39 +124,93 @@ public class BillingUtils {
final String projectEndpointBucketId = String.format(ENDPOINT_SHARED_BUCKET_FORMAT, sbn, endpoint).toLowerCase();
final String endpointId = String.format(ENDPOINT_FORMAT, sbn, endpoint).toLowerCase();
return Stream.concat(Stream.of(
- BillingReportLine.builder().resourceName("Endpoint shared bucket").user(SHARED_RESOURCE).project(SHARED_RESOURCE).datalabId(projectEndpointBucketId).resourceType(BUCKET).build(),
- BillingReportLine.builder().resourceName("Endpoint").user(SHARED_RESOURCE).project(SHARED_RESOURCE).datalabId(endpointId).resourceType(ENDPOINT).build()
+ BillingReportLine.builder()
+ .resourceName("Endpoint shared bucket")
+ .user(SHARED_RESOURCE)
+ .project(SHARED_RESOURCE)
+ .datalabId(projectEndpointBucketId)
+ .resourceType(BUCKET)
+ .build(),
+ BillingReportLine.builder()
+ .resourceName("Endpoint")
+ .user(SHARED_RESOURCE)
+ .project(SHARED_RESOURCE)
+ .datalabId(endpointId)
+ .resourceType(ENDPOINT).build()
),
standardImageBillingDataStream(sbn, endpoint));
}
public static Stream<BillingReportLine> exploratoryBillingDataStream(UserInstanceDTO userInstance, Integer maxSparkInstanceCount) {
+ log.info("TEST LOG!!!: exploratoryBillingDataStream");
+ log.info("TEST LOG!!!: userInstance: {}", userInstance);
+
+
final Stream<BillingReportLine> computationalStream = userInstance.getResources()
.stream()
.filter(cr -> cr.getComputationalId() != null)
.flatMap(cr -> {
final String computationalId = cr.getComputationalId().toLowerCase();
return Stream.concat(Stream.of(
- withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).datalabId(computationalId).resourceType(COMPUTATIONAL).shape(getComputationalShape(cr))
- .exploratoryName(userInstance.getExploratoryName()).build(),
- withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).datalabId(String.format(VOLUME_PRIMARY_FORMAT, computationalId)).resourceType(VOLUME).build(),
- withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).datalabId(String.format(VOLUME_SECONDARY_FORMAT, computationalId)).resourceType(VOLUME).build(),
- withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).datalabId(String.format(VOLUME_PRIMARY_COMPUTATIONAL_FORMAT, computationalId, "m"))
- .resourceType(VOLUME).build(),
- withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).datalabId(String.format(VOLUME_SECONDARY_COMPUTATIONAL_FORMAT, computationalId, "m"))
- .resourceType(VOLUME).build()
+ withUserProjectEndpoint(userInstance)
+ .resourceName(cr.getComputationalName())
+ .datalabId(computationalId)
+ .resourceType(COMPUTATIONAL)
+ .shape(getComputationalShape(cr))
+ .exploratoryName(userInstance.getExploratoryName())
+ .build(),
+ withUserProjectEndpoint(userInstance)
+ .resourceName(cr.getComputationalName())
+ .datalabId(String.format(VOLUME_PRIMARY_FORMAT, computationalId))
+ .resourceType(VOLUME)
+ .build(),
+ withUserProjectEndpoint(userInstance)
+ .resourceName(cr.getComputationalName())
+ .datalabId(String.format(VOLUME_SECONDARY_FORMAT, computationalId))
+ .resourceType(VOLUME)
+ .build(),
+ withUserProjectEndpoint(userInstance)
+ .resourceName(cr.getComputationalName())
+ .datalabId(String.format(VOLUME_PRIMARY_COMPUTATIONAL_FORMAT, computationalId, "m"))
+ .resourceType(VOLUME)
+ .build(),
+ withUserProjectEndpoint(userInstance)
+ .resourceName(cr.getComputationalName())
+ .datalabId(String.format(VOLUME_SECONDARY_COMPUTATIONAL_FORMAT, computationalId, "m"))
+ .resourceType(VOLUME)
+ .build()
),
getSlaveVolumes(userInstance, cr, maxSparkInstanceCount)
);
});
+ log.info("TEST LOG!!!: computationalStream: {}", computationalStream);
+ log.info("TEST LOG!!!: computationalStream: {}", computationalStream.collect(Collectors.toList()));
+
+
final String exploratoryName = userInstance.getExploratoryName();
final String exploratoryId = userInstance.getExploratoryId().toLowerCase();
final String primaryVolumeId = String.format(VOLUME_PRIMARY_FORMAT, exploratoryId);
final String secondaryVolumeId = String.format(VOLUME_SECONDARY_FORMAT, exploratoryId);
final Stream<BillingReportLine> exploratoryStream = Stream.of(
- withUserProjectEndpoint(userInstance).resourceName(exploratoryName).datalabId(exploratoryId).resourceType(EXPLORATORY).shape(userInstance.getShape()).build(),
- withUserProjectEndpoint(userInstance).resourceName(exploratoryName).datalabId(primaryVolumeId).resourceType(VOLUME).build(),
- withUserProjectEndpoint(userInstance).resourceName(exploratoryName).datalabId(secondaryVolumeId).resourceType(VOLUME).build());
+ withUserProjectEndpoint(userInstance)
+ .resourceName(exploratoryName)
+ .datalabId(exploratoryId)
+ .resourceType(EXPLORATORY)
+ .shape(userInstance.getShape())
+ .build(),
+ withUserProjectEndpoint(userInstance)
+ .resourceName(exploratoryName)
+ .datalabId(primaryVolumeId)
+ .resourceType(VOLUME)
+ .build(),
+ withUserProjectEndpoint(userInstance)
+ .resourceName(exploratoryName)
+ .datalabId(secondaryVolumeId)
+ .resourceType(VOLUME)
+ .build());
+
+ log.info("TEST LOG!!!: exploratoryStream: {}", exploratoryStream);
+ log.info("TEST LOG!!!: exploratoryStream: {}", exploratoryStream.collect(Collectors.toList()));
return Stream.concat(computationalStream, exploratoryStream);
}
@@ -156,10 +234,19 @@ public class BillingUtils {
}
private static BillingReportLine.BillingReportLineBuilder withUserProjectEndpoint(UserInstanceDTO userInstance) {
- return BillingReportLine.builder().user(userInstance.getUser()).project(userInstance.getProject()).endpoint(userInstance.getEndpoint());
+ return BillingReportLine.builder()
+ .user(userInstance.getUser())
+ .project(userInstance.getProject())
+ .endpoint(userInstance.getEndpoint());
}
public static String getComputationalShape(UserComputationalResource resource) {
+ log.info("TEST LOG!!! getComputationalShape");
+ log.info("TEST LOG!!! UserComputationalResource: {}", resource);
+ log.info("TEST LOG!!! getTemplateName: {}", resource.getTemplateName());
+ log.info("TEST LOG!!! getDataengineShape: {}", resource.getDataengineShape());
+ log.info("TEST LOG!!! getMasterNodeShape: {}", resource.getMasterNodeShape());
+
return DataEngineType.fromDockerImageName(resource.getImageName()) == DataEngineType.SPARK_STANDALONE ?
String.format(DATAENGINE_NAME_FORMAT, resource.getDataengineInstanceCount(), resource.getDataengineShape()) :
String.format(DATAENGINE_SERVICE_NAME_FORMAT, resource.getMasterNodeShape(), resource.getTotalInstanceCount() - 1, resource.getSlaveNodeShape());
@@ -168,37 +255,46 @@ public class BillingUtils {
private static Stream<BillingReportLine> standardImageBillingDataStream(String sbn, String endpoint) {
List<BillingReportLine> list = new ArrayList<>();
for (String notebook : AVAILABLE_NOTEBOOKS) {
- list.add(BillingReportLine.builder().resourceName(IMAGE_NAME).datalabId(String.format(IMAGE_STANDARD_FORMAT2, sbn, endpoint, notebook).toLowerCase())
- .user(SHARED_RESOURCE).project(SHARED_RESOURCE).resourceType(IMAGE).build());
+ list.add(BillingReportLine.builder()
+ .resourceName(IMAGE_NAME)
+ .datalabId(String.format(IMAGE_STANDARD_FORMAT2, sbn, endpoint, notebook).toLowerCase())
+ .user(SHARED_RESOURCE)
+ .project(SHARED_RESOURCE)
+ .resourceType(IMAGE)
+ .build());
}
-
return list.stream();
}
private static Stream<BillingReportLine> standardImageBillingDataStream(String sbn, String project, String endpoint) {
List<BillingReportLine> list = new ArrayList<>();
for (String notebook : AVAILABLE_NOTEBOOKS) {
- list.add(BillingReportLine.builder().resourceName(IMAGE_NAME).datalabId(String.format(IMAGE_STANDARD_FORMAT1, sbn, project, endpoint, notebook).toLowerCase())
- .project(project).user(SHARED_RESOURCE).resourceType(IMAGE).build());
+ list.add(BillingReportLine
+ .builder()
+ .resourceName(IMAGE_NAME)
+ .datalabId(String.format(IMAGE_STANDARD_FORMAT1, sbn, project, endpoint, notebook).toLowerCase())
+ .project(project)
+ .user(SHARED_RESOURCE)
+ .resourceType(IMAGE)
+ .build());
}
-
return list.stream();
}
- /**
- * @param sbn Service Base Name
- * @param from formatted date, like 2020-04-07
- * @param to formatted date, like 2020-05-07
- * @param locale user's locale
- * @return line, like:
- * "Service base name: SERVICE_BASE_NAME. Available reporting period from: 2020-04-07 to: 2020-04-07"
- */
- public static String getFirstLine(String sbn, LocalDate from, LocalDate to, String locale) {
- return CSVFormatter.formatLine(Lists.newArrayList(String.format(REPORT_FIRST_LINE, sbn,
- Optional.ofNullable(from).map(date -> date.format(DateTimeFormatter.ofLocalizedDate(FormatStyle.MEDIUM).withLocale(Locale.forLanguageTag(locale)))).orElse(StringUtils.EMPTY),
- Optional.ofNullable(to).map(date -> date.format(DateTimeFormatter.ofLocalizedDate(FormatStyle.MEDIUM).withLocale(Locale.forLanguageTag(locale)))).orElse(StringUtils.EMPTY))),
- CSVFormatter.SEPARATOR, '\"');
- }
+ /**
+ * @param sbn Service Base Name
+ * @param from formatted date, like 2020-04-07
+ * @param to formatted date, like 2020-05-07
+ * @param locale user's locale
+ * @return line, like:
+ * "Service base name: SERVICE_BASE_NAME. Available reporting period from: 2020-04-07 to: 2020-04-07"
+ */
+ public static String getFirstLine(String sbn, LocalDate from, LocalDate to, String locale) {
+ return CSVFormatter.formatLine(Lists.newArrayList(String.format(REPORT_FIRST_LINE, sbn,
+ Optional.ofNullable(from).map(date -> date.format(DateTimeFormatter.ofLocalizedDate(FormatStyle.MEDIUM).withLocale(Locale.forLanguageTag(locale)))).orElse(StringUtils.EMPTY),
+ Optional.ofNullable(to).map(date -> date.format(DateTimeFormatter.ofLocalizedDate(FormatStyle.MEDIUM).withLocale(Locale.forLanguageTag(locale)))).orElse(StringUtils.EMPTY))),
+ CSVFormatter.SEPARATOR, '\"');
+ }
/**
* headerType there are two types of header according user role
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java
index ad50429..7b1ea0a 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java
@@ -171,7 +171,7 @@ public class RequestBuilder {
}
- T t = exploratoryCreate.withExploratoryName(exploratory.getName())
+ return exploratoryCreate.withExploratoryName(exploratory.getName())
.withNotebookImage(exploratory.getDockerImage())
.withApplicationName(getApplicationNameFromImage(exploratory.getDockerImage()))
.withGitCreds(exploratoryGitCredsDTO.getGitCreds())
@@ -184,7 +184,6 @@ public class RequestBuilder {
.withGPUCount(exploratory.getGpuCount())
.withGPUType(exploratory.getGpuType())
.withEnabledGPU(exploratory.getEnabledGPU());
- return t;
}
@SuppressWarnings("unchecked")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 03/03: [DATALAB-2563] -- fixed merge conf
Posted by yk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ykinash pushed a commit to branch DATALAB-2556
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 5891d61799ddcc906935eadbf90f25e699fe870a
Author: KinashYurii <ur...@gmail.com>
AuthorDate: Mon Nov 8 21:02:04 2021 +0200
[DATALAB-2563] -- fixed merge conf
---
services/billing-azure/pom.xml | 33 +++++++++++++++++----------------
1 file changed, 17 insertions(+), 16 deletions(-)
diff --git a/services/billing-azure/pom.xml b/services/billing-azure/pom.xml
index 80caf44..b5a4098 100644
--- a/services/billing-azure/pom.xml
+++ b/services/billing-azure/pom.xml
@@ -105,11 +105,11 @@
<version>${org.mongodb.version}</version>
</dependency>
- <!--dependency>
- <groupId>com.fasterxml.jackson.dataformat</groupId>
- <artifactId>jackson-dataformat-yaml</artifactId>
- <version>2.9.5</version>
- </dependency-->
+<!-- <dependency>-->
+<!-- <groupId>com.fasterxml.jackson.dataformat</groupId>-->
+<!-- <artifactId>jackson-dataformat-yaml</artifactId>-->
+<!-- <version>2.12.2</version>-->
+<!-- </dependency>-->
<dependency>
<groupId>ch.qos.logback</groupId>
@@ -121,16 +121,17 @@
<artifactId>httpclient</artifactId>
<version>4.5.5</version>
</dependency>
- <!--dependency>
- <groupId>com.fasterxml.jackson.datatype</groupId>
- <artifactId>jackson-datatype-guava</artifactId>
- <version>2.11.0</version>
- </dependency-->
- <!--dependency>
- <groupId>com.fasterxml.jackson.datatype</groupId>
- <artifactId>jackson-datatype-jsr310</artifactId>
- <version>2.11.0</version>
- </dependency-->
+
+<!-- <dependency>-->
+<!-- <groupId>com.fasterxml.jackson.datatype</groupId>-->
+<!-- <artifactId>jackson-datatype-guava</artifactId>-->
+<!-- <version>2.11.0</version>-->
+<!-- </dependency>-->
+<!-- <dependency>-->
+<!-- <groupId>com.fasterxml.jackson.datatype</groupId>-->
+<!-- <artifactId>jackson-datatype-jsr310</artifactId>-->
+<!-- <version>2.11.0</version>-->
+<!-- </dependency>-->
</dependencies>
@@ -150,4 +151,4 @@
</plugins>
</build>
-</project>
+</project>
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org