You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by yk...@apache.org on 2021/11/10 21:08:10 UTC
[incubator-datalab] 01/01: [DATALAB] -- fixed jackson dependency,
added test log, fixed billing null value,
This is an automated email from the ASF dual-hosted git repository.
ykinash pushed a commit to branch DATALAB-2556
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 66455cb133ca49c782a4a6af513320b184727552
Author: KinashYurii <ur...@gmail.com>
AuthorDate: Wed Nov 10 23:06:14 2021 +0200
[DATALAB] -- fixed jackson dependency, added test log, fixed billing null value,
---
services/datalab-model/pom.xml | 6 +++++
.../epam/datalab/dto/UserEnvironmentResources.java | 5 +++-
.../epam/datalab/dto/project/ProjectCreateDTO.java | 2 ++
.../handlers/ResourcesStatusCallbackHandler.java | 7 +++++
.../resources/aws/InfrastructureResourceAws.java | 1 +
.../CheckInfrastructureStatusScheduler.java | 9 +++++--
.../schedulers/billing/BillingScheduler.java | 2 +-
.../service/impl/BillingServiceImpl.java | 30 +++++++++++++++++++++-
.../epam/datalab/backendapi/util/BillingUtils.java | 6 ++++-
.../datalab/backendapi/util/RequestBuilder.java | 4 ++-
10 files changed, 65 insertions(+), 7 deletions(-)
diff --git a/services/datalab-model/pom.xml b/services/datalab-model/pom.xml
index 2f7878a..a27ff0b 100644
--- a/services/datalab-model/pom.xml
+++ b/services/datalab-model/pom.xml
@@ -74,6 +74,12 @@
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>2.11.2</version>
+ </dependency>
+
</dependencies>
</project>
\ No newline at end of file
diff --git a/services/datalab-model/src/main/java/com/epam/datalab/dto/UserEnvironmentResources.java b/services/datalab-model/src/main/java/com/epam/datalab/dto/UserEnvironmentResources.java
index 3c74d45..0fc9c27 100644
--- a/services/datalab-model/src/main/java/com/epam/datalab/dto/UserEnvironmentResources.java
+++ b/services/datalab-model/src/main/java/com/epam/datalab/dto/UserEnvironmentResources.java
@@ -20,11 +20,13 @@
package com.epam.datalab.dto;
import com.epam.datalab.dto.status.EnvResourceList;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.MoreObjects.ToStringHelper;
import lombok.EqualsAndHashCode;
-@EqualsAndHashCode
+@EqualsAndHashCode(callSuper = false)
+@JsonIgnoreProperties
public class UserEnvironmentResources extends ResourceSysBaseDTO<UserEnvironmentResources> {
@JsonProperty("edge_list_resources")
private EnvResourceList resourceList;
@@ -46,6 +48,7 @@ public class UserEnvironmentResources extends ResourceSysBaseDTO<UserEnvironment
/**
* Set the list of resources (hosts, clusters, storages).
*/
+ @JsonProperty("edge_list_resources")
public UserEnvironmentResources withResourceList(EnvResourceList resourceList) {
setResourceList(resourceList);
return this;
diff --git a/services/datalab-model/src/main/java/com/epam/datalab/dto/project/ProjectCreateDTO.java b/services/datalab-model/src/main/java/com/epam/datalab/dto/project/ProjectCreateDTO.java
index 09cbd46..c780d4d 100644
--- a/services/datalab-model/src/main/java/com/epam/datalab/dto/project/ProjectCreateDTO.java
+++ b/services/datalab-model/src/main/java/com/epam/datalab/dto/project/ProjectCreateDTO.java
@@ -23,7 +23,9 @@ import com.epam.datalab.dto.ResourceBaseDTO;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Builder;
import lombok.Data;
+import lombok.EqualsAndHashCode;
+@EqualsAndHashCode(callSuper = true)
@Data
@Builder
public class ProjectCreateDTO extends ResourceBaseDTO<ProjectCreateDTO> {
diff --git a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ResourcesStatusCallbackHandler.java b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ResourcesStatusCallbackHandler.java
index 0acb3a4..778b774 100644
--- a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ResourcesStatusCallbackHandler.java
+++ b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ResourcesStatusCallbackHandler.java
@@ -52,6 +52,13 @@ public class ResourcesStatusCallbackHandler extends ResourceCallbackHandler<EnvS
@JsonCreator
public ResourcesStatusCallbackHandler(@JacksonInject RESTService selfService, @JsonProperty("action") DockerAction action,
+ @JsonProperty("uuid") String uuid, @JsonProperty("user") String user) {
+ super(selfService, user, uuid, action);
+ this.datalabHostResources = getEnvResources(null);
+ }
+
+ @JsonCreator
+ public ResourcesStatusCallbackHandler(@JacksonInject RESTService selfService, @JsonProperty("action") DockerAction action,
@JsonProperty("uuid") String uuid, @JsonProperty("user") String user,
EnvResourceList resourceList) {
super(selfService, user, uuid, action);
diff --git a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/resources/aws/InfrastructureResourceAws.java b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/resources/aws/InfrastructureResourceAws.java
index c88002c..2df471e 100644
--- a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/resources/aws/InfrastructureResourceAws.java
+++ b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/resources/aws/InfrastructureResourceAws.java
@@ -44,6 +44,7 @@ public class InfrastructureResourceAws extends InfrastructureService {
@Path("/status")
@POST
public String status(@Auth UserInfo ui, UserEnvironmentResources dto) {
+ log.info("TEST LOG!!! UserEnvironmentResources:{}", dto);
return action(ui.getName(), dto, dto.getCloudSettings().getIamUser(), DockerAction.STATUS);
}
}
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/CheckInfrastructureStatusScheduler.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/CheckInfrastructureStatusScheduler.java
index 5d593db..018176a 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/CheckInfrastructureStatusScheduler.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/CheckInfrastructureStatusScheduler.java
@@ -110,7 +110,8 @@ public class CheckInfrastructureStatusScheduler implements Job {
.stream()
.filter(c -> DataEngineType.SPARK_STANDALONE == DataEngineType.fromDockerImageName(c.getImageName()))
.filter(c -> statusesToCheck.contains(UserInstanceStatus.of(c.getStatus())))
-// .filter(c -> c.getComputationalId() != null && c.getInstanceId() != null)
+ .filter(this::noEmrCreating)
+ .filter(c -> c.getInstanceId() != null)
.map(r -> new EnvResource()
.withId(r.getInstanceId())
.withName(r.getComputationalName())
@@ -135,7 +136,7 @@ public class CheckInfrastructureStatusScheduler implements Job {
return userInstanceDTO.getResources().stream()
.filter(c -> DataEngineType.CLOUD_SERVICE == DataEngineType.fromDockerImageName(c.getImageName()))
.filter(c -> statusesToCheck.contains(UserInstanceStatus.of(c.getStatus())))
-// .filter(c -> c.getComputationalId() != null && c.getInstanceId() != null)
+ .filter(c -> c.getInstanceId() != null)
.filter(this::noEmrCreating)
.map(r -> new EnvResource()
.withId(r.getInstanceId())
@@ -167,6 +168,10 @@ public class CheckInfrastructureStatusScheduler implements Job {
private List<EnvResource> getEdgeInstances(String endpoint, Map.Entry<String, List<ProjectEndpointDTO>> entry) {
return entry.getValue()
.stream()
+ .peek(e -> log.info("TEST LOG!!!: status: {}", e.getStatus()))
+ .peek(e -> log.info("TEST LOG!!!: e: {}", e))
+ .peek(e -> log.info("TEST LOG!!!: statusesToCheck: {}", statusesToCheck))
+
.filter(e -> statusesToCheck.contains(e.getStatus()))
.filter(e -> e.getName().equals(endpoint))
.filter(e -> Objects.nonNull(e.getEdgeInfo()))
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/billing/BillingScheduler.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/billing/BillingScheduler.java
index c4dc177..fbc369c 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/billing/BillingScheduler.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/billing/BillingScheduler.java
@@ -46,7 +46,7 @@ public class BillingScheduler implements Job {
try {
billingService.updateRemoteBillingData(securityService.getServiceAccountInfo("admin"));
} catch (Exception e) {
- log.error("Something went wrong {}", e.getMessage(), e);
+ log.error("Something wents wrong {}", e.getMessage(), e);
}
}
}
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/BillingServiceImpl.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/BillingServiceImpl.java
index e65901a..2a6908c 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/BillingServiceImpl.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/BillingServiceImpl.java
@@ -46,6 +46,8 @@ import com.epam.datalab.rest.client.RESTService;
import com.google.common.collect.Lists;
import com.google.inject.Inject;
import com.google.inject.name.Named;
+import lombok.AllArgsConstructor;
+import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.http.client.utils.URIBuilder;
@@ -99,6 +101,24 @@ public class BillingServiceImpl implements BillingService {
.peek(this::appendStatuses)
.filter(bd -> CollectionUtils.isEmpty(filter.getStatuses()) || filter.getStatuses().contains(bd.getStatus()))
.collect(Collectors.toList());
+
+ //TEST
+ List<ProjectResources> collect = billingReportLines.stream()
+ .filter(x -> x.getShape().equals("0 x null"))
+ .map(x -> new ProjectResources(x.getResourceName(), x.getProject(), x))
+ .collect(Collectors.toList());
+
+ collect.stream()
+ .peek((
+ x -> System.out.println(exploratoryService.getUserInstance(user.getName(), x.getProject(), x.getResName()))
+ ))
+ .peek(
+ x -> System.out.println(
+ exploratoryService.getUserInstance(user.getName(), x.getProject(), x.getBillingReportLine().getExploratoryName()))
+
+ ).count();
+
+//TEST
final LocalDate min = billingReportLines.stream().min(Comparator.comparing(BillingReportLine::getUsageDateFrom)).map(BillingReportLine::getUsageDateFrom).orElse(null);
final LocalDate max = billingReportLines.stream().max(Comparator.comparing(BillingReportLine::getUsageDateTo)).map(BillingReportLine::getUsageDateTo).orElse(null);
final double sum = billingReportLines.stream().mapToDouble(BillingReportLine::getCost).sum();
@@ -385,4 +405,12 @@ public class BillingServiceImpl implements BillingService {
.map(Double::intValue)
.orElse(BigDecimal.ZERO.intValue());
}
-}
+
+ @Data
+ @AllArgsConstructor
+ private class ProjectResources {
+ private String resName;
+ private String project;
+ private BillingReportLine billingReportLine;
+ }
+}
\ No newline at end of file
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/util/BillingUtils.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/util/BillingUtils.java
index 1e8b31c..3ca334f 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/util/BillingUtils.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/util/BillingUtils.java
@@ -151,6 +151,7 @@ public class BillingUtils {
.filter(cr -> cr.getComputationalId() != null)
.flatMap(cr -> {
final String computationalId = cr.getComputationalId().toLowerCase();
+ log.info("TEST LOG!!!: cr: {}", cr);
return Stream.concat(Stream.of(
withUserProjectEndpoint(userInstance)
.resourceName(cr.getComputationalName())
@@ -244,10 +245,13 @@ public class BillingUtils {
log.info("TEST LOG!!! getComputationalShape");
log.info("TEST LOG!!! UserComputationalResource: {}", resource);
log.info("TEST LOG!!! getTemplateName: {}", resource.getTemplateName());
+ log.info("TEST LOG!!! getTotalInstanceCount: {}", resource.getTotalInstanceCount());
log.info("TEST LOG!!! getDataengineShape: {}", resource.getDataengineShape());
log.info("TEST LOG!!! getMasterNodeShape: {}", resource.getMasterNodeShape());
+ log.info("TEST LOG!!!: DataEngineType.fromDockerImageName(resource.getImageName()) == DataEngineType.SPARK_STANDALONE: {} == {}",
+ DataEngineType.fromDockerImageName(resource.getImageName()), DataEngineType.SPARK_STANDALONE);
- return DataEngineType.fromDockerImageName(resource.getImageName()) == DataEngineType.SPARK_STANDALONE ?
+ return DataEngineType.fromDockerImageName(resource.getImageName()) != DataEngineType.SPARK_STANDALONE ?
String.format(DATAENGINE_NAME_FORMAT, resource.getDataengineInstanceCount(), resource.getDataengineShape()) :
String.format(DATAENGINE_SERVICE_NAME_FORMAT, resource.getMasterNodeShape(), resource.getTotalInstanceCount() - 1, resource.getSlaveNodeShape());
}
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java
index 7b1ea0a..9a77442 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java
@@ -654,8 +654,10 @@ public class RequestBuilder {
}
public UserEnvironmentResources newInfrastructureStatus(String user, CloudProvider cloudProvider, EnvResourceList resourceList) {
- return newResourceSysBaseDTO(user, cloudProvider, UserEnvironmentResources.class)
+ UserEnvironmentResources userEnvironmentResources = newResourceSysBaseDTO(user, cloudProvider, UserEnvironmentResources.class)
.withResourceList(resourceList);
+ log.info("TEST LOG!!! userEnvironmentResources: {}", userEnvironmentResources);
+ return userEnvironmentResources;
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org