You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by yk...@apache.org on 2021/11/12 14:55:40 UTC
[incubator-datalab] branch develop updated: [DATALAB] -- clean up
code
This is an automated email from the ASF dual-hosted git repository.
ykinash pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
The following commit(s) were added to refs/heads/develop by this push:
new f3b9500 [DATALAB] -- clean up code
f3b9500 is described below
commit f3b9500ed82e3f661da53d41f27ae79e980bcb9f
Author: KinashYurii <ur...@gmail.com>
AuthorDate: Fri Nov 12 16:34:01 2021 +0200
[DATALAB] -- clean up code
---
services/datalab-model/pom.xml | 6 ------
.../epam/datalab/dto/status/EnvResourceList.java | 5 +----
.../handlers/ComputationalCallbackHandler.java | 2 +-
.../core/response/handlers/EdgeCallbackHandler.java | 1 -
.../response/handlers/ProjectCallbackHandler.java | 2 --
.../response/handlers/ResourceCallbackHandler.java | 1 -
.../resources/aws/InfrastructureResourceAws.java | 1 -
.../resources/callback/ProjectCallback.java | 1 -
.../CheckInfrastructureStatusScheduler.java | 5 -----
.../backendapi/service/impl/BillingServiceImpl.java | 8 --------
.../service/impl/InfrastructureInfoServiceImpl.java | 7 +++++--
.../impl/InfrastructureTemplateServiceImpl.java | 5 +----
.../epam/datalab/backendapi/util/BillingUtils.java | 21 ---------------------
.../datalab/backendapi/util/RequestBuilder.java | 4 +---
14 files changed, 9 insertions(+), 60 deletions(-)
diff --git a/services/datalab-model/pom.xml b/services/datalab-model/pom.xml
index a27ff0b..2f7878a 100644
--- a/services/datalab-model/pom.xml
+++ b/services/datalab-model/pom.xml
@@ -74,12 +74,6 @@
<scope>provided</scope>
</dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- <version>2.11.2</version>
- </dependency>
-
</dependencies>
</project>
\ No newline at end of file
diff --git a/services/datalab-model/src/main/java/com/epam/datalab/dto/status/EnvResourceList.java b/services/datalab-model/src/main/java/com/epam/datalab/dto/status/EnvResourceList.java
index 620a071..4cfc2c1 100644
--- a/services/datalab-model/src/main/java/com/epam/datalab/dto/status/EnvResourceList.java
+++ b/services/datalab-model/src/main/java/com/epam/datalab/dto/status/EnvResourceList.java
@@ -19,6 +19,7 @@
package com.epam.datalab.dto.status;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.MoreObjects;
import com.google.common.base.MoreObjects.ToStringHelper;
@@ -46,10 +47,6 @@ public class EnvResourceList {
.add("cluster", clusterList);
}
- public boolean isEmpty(){
- return hostList.isEmpty() && clusterList.isEmpty();
- }
-
@Override
public String toString() {
return toStringHelper(this).toString();
diff --git a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ComputationalCallbackHandler.java b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ComputationalCallbackHandler.java
index 416eda7..7e24d11 100644
--- a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ComputationalCallbackHandler.java
+++ b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ComputationalCallbackHandler.java
@@ -50,7 +50,7 @@ public class ComputationalCallbackHandler extends ResourceCallbackHandler<Comput
@JsonProperty
private final ComputationalBase<?> dto;
- private ComputationalConfigure computationalConfigure;
+ private final ComputationalConfigure computationalConfigure;
@JsonCreator
public ComputationalCallbackHandler(@JacksonInject ComputationalConfigure computationalConfigure,
diff --git a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/EdgeCallbackHandler.java b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/EdgeCallbackHandler.java
index 52df8ab..8ebec10 100644
--- a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/EdgeCallbackHandler.java
+++ b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/EdgeCallbackHandler.java
@@ -59,7 +59,6 @@ public class EdgeCallbackHandler<E extends EdgeInfo, T extends UploadFileResult<
}
protected T parseOutResponse(JsonNode resultNode, T baseStatus) {
- log.info("TEST LOG!!!:resultNode: {}, base status: {} ", resultNode, baseStatus);
if (resultNode != null
&& (getAction() == DockerAction.CREATE || getAction() == DockerAction.START)
&& UserInstanceStatus.of(baseStatus.getStatus()) != UserInstanceStatus.FAILED) {
diff --git a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ProjectCallbackHandler.java b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ProjectCallbackHandler.java
index e8a6566..426c03d 100644
--- a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ProjectCallbackHandler.java
+++ b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ProjectCallbackHandler.java
@@ -57,8 +57,6 @@ public class ProjectCallbackHandler extends ResourceCallbackHandler<ProjectResul
@Override
protected ProjectResult parseOutResponse(JsonNode resultNode, ProjectResult baseStatus) {
- log.info("TEST LOG!!!: resultNode: {} , baseStatus: {}", resultNode, baseStatus);
-
baseStatus.setProjectName(projectName);
baseStatus.setEndpointName(endpointName);
if (resultNode != null &&
diff --git a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ResourceCallbackHandler.java b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ResourceCallbackHandler.java
index 6999968..137a81b 100644
--- a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ResourceCallbackHandler.java
+++ b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ResourceCallbackHandler.java
@@ -129,7 +129,6 @@ public abstract class ResourceCallbackHandler<T extends StatusBaseDTO<?>> implem
log.error("Could not {} resource for user: {}, UUID: {}", action, user, uuid);
result.setErrorMessage(getTextValue(resultNode.get(ERROR_NODE)));
}
- log.info("TEST LOG!!!: resultNode: {}, result: {}", resultNode, result);
result = parseOutResponse(resultNode, result);
selfServicePost(result);
return !UserInstanceStatus.FAILED.equals(status);
diff --git a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/resources/aws/InfrastructureResourceAws.java b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/resources/aws/InfrastructureResourceAws.java
index 2df471e..c88002c 100644
--- a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/resources/aws/InfrastructureResourceAws.java
+++ b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/resources/aws/InfrastructureResourceAws.java
@@ -44,7 +44,6 @@ public class InfrastructureResourceAws extends InfrastructureService {
@Path("/status")
@POST
public String status(@Auth UserInfo ui, UserEnvironmentResources dto) {
- log.info("TEST LOG!!! UserEnvironmentResources:{}", dto);
return action(ui.getName(), dto, dto.getCloudSettings().getIamUser(), DockerAction.STATUS);
}
}
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/callback/ProjectCallback.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/callback/ProjectCallback.java
index d5c521a..23459ea 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/callback/ProjectCallback.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/callback/ProjectCallback.java
@@ -61,7 +61,6 @@ public class ProjectCallback {
@POST
public Response updateProjectStatus(ProjectResult projectResult) {
- log.info("TEST LOG!!!: projectResult: {}", projectResult);
requestId.checkAndRemove(projectResult.getRequestId());
final String projectName = projectResult.getProjectName();
final UserInstanceStatus status = UserInstanceStatus.of(projectResult.getStatus());
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/CheckInfrastructureStatusScheduler.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/CheckInfrastructureStatusScheduler.java
index 018176a..ab11001 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/CheckInfrastructureStatusScheduler.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/CheckInfrastructureStatusScheduler.java
@@ -149,7 +149,6 @@ public class CheckInfrastructureStatusScheduler implements Job {
}
private boolean noEmrCreating(UserComputationalResource c) {
- log.info("TEST LOG!!!: status: {}, template name: {}", c.getStatus(), c.getTemplateName());
return !(c.getStatus().equals(CREATING.name()) && c.getTemplateName().contains(AWS_EMR_CLUSTER));
}
@@ -168,10 +167,6 @@ public class CheckInfrastructureStatusScheduler implements Job {
private List<EnvResource> getEdgeInstances(String endpoint, Map.Entry<String, List<ProjectEndpointDTO>> entry) {
return entry.getValue()
.stream()
- .peek(e -> log.info("TEST LOG!!!: status: {}", e.getStatus()))
- .peek(e -> log.info("TEST LOG!!!: e: {}", e))
- .peek(e -> log.info("TEST LOG!!!: statusesToCheck: {}", statusesToCheck))
-
.filter(e -> statusesToCheck.contains(e.getStatus()))
.filter(e -> e.getName().equals(endpoint))
.filter(e -> Objects.nonNull(e.getEdgeInfo()))
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/BillingServiceImpl.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/BillingServiceImpl.java
index 2a6908c..4b6152f 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/BillingServiceImpl.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/BillingServiceImpl.java
@@ -179,8 +179,6 @@ public class BillingServiceImpl implements BillingService {
.stream()
.collect(Collectors.toMap(e -> e, e -> getBillingData(userInfo, e)));
- log.info("TEST LOG!!!: billingDataMap: {}", billingDataMap);
-
billingDataMap.forEach((endpointDTO, billingData) -> {
log.info("Updating billing information for endpoint {}. Billing data {}", endpointDTO.getName(), billingData);
if (!billingData.isEmpty()) {
@@ -235,17 +233,11 @@ public class BillingServiceImpl implements BillingService {
.stream()
.flatMap(endpoint -> BillingUtils.sharedEndpointBillingDataStream(endpoint.getName(), configuration.getServiceBaseName()));
- log.info("TEST LOG!!!: userInstance from service: {}", exploratoryService.findAll(projects));
- log.info("TEST LOG!!!: userInstance from DB: {}", exploratoryDAO.fetchExploratoryFieldsForProject("proj"));
-
-
final Stream<BillingReportLine> billableUserInstances = exploratoryService.findAll(projects)
.stream()
.filter(userInstance -> Objects.nonNull(userInstance.getExploratoryId()))
.flatMap(ui -> BillingUtils.exploratoryBillingDataStream(ui, configuration.getMaxSparkInstanceCount()));
- log.info("TEST LOG!!!: billableUserInstances: {}", billableUserInstances.collect(Collectors.toList()));
-
final Stream<BillingReportLine> customImages = projects
.stream()
.map(p -> imageExploratoryDao.getImagesForProject(p.getName()))
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/InfrastructureInfoServiceImpl.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/InfrastructureInfoServiceImpl.java
index 4ea885b..2aca621 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/InfrastructureInfoServiceImpl.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/InfrastructureInfoServiceImpl.java
@@ -146,7 +146,7 @@ public class InfrastructureInfoServiceImpl implements InfrastructureInfoService
.build();
EndpointDTO endpointDTO = endpointService.get(endpoint);
- if (envResourceList.isEmpty()) {
+ if (isEmpty(envResourceList)) {
log.info("EnvResources is empty: {} , didn't send request to provisioning service", envResourceList);
} else {
log.info("Send request to provisioning service:\n POST:{}, with EnvResources: {}", INFRASTRUCTURE_STATUS,
@@ -158,6 +158,10 @@ public class InfrastructureInfoServiceImpl implements InfrastructureInfoService
}
}
+ private boolean isEmpty(EnvResourceList envResourceList) {
+ return envResourceList.getClusterList().isEmpty() && envResourceList.getHostList().isEmpty();
+ }
+
private List<BillingReport> getExploratoryBillingData(List<UserInstanceDTO> exploratories) {
return exploratories
.stream()
@@ -196,7 +200,6 @@ public class InfrastructureInfoServiceImpl implements InfrastructureInfoService
shared.put("status", endpointDTO.getStatus().toString());
shared.put("edge_node_ip", edge.getPublicIp());
- log.info("TEST LOG!!!: edge: {}", edge);
if (edge instanceof EdgeInfoAws) {
EdgeInfoAws edgeInfoAws = (EdgeInfoAws) edge;
shared.put("user_own_bicket_name", edgeInfoAws.getUserOwnBucketName());
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/InfrastructureTemplateServiceImpl.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/InfrastructureTemplateServiceImpl.java
index 4508397..da13cf1 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/InfrastructureTemplateServiceImpl.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/InfrastructureTemplateServiceImpl.java
@@ -108,15 +108,12 @@ public class InfrastructureTemplateServiceImpl implements InfrastructureTemplate
final Set<String> roles = userGroupDao.getUserGroups(user.getName());
- List<FullComputationalTemplate> s = Arrays.stream(array)
+ return Arrays.stream(array)
.peek(e -> e.setImage(getSimpleImageName(e.getImage())))
.peek(e -> filterShapes(user, e.getComputationResourceShapes(), RoleType.COMPUTATIONAL_SHAPES, roles))
.filter(e -> UserRoles.checkAccess(user, RoleType.COMPUTATIONAL, e.getImage(), roles))
.map(comp -> fullComputationalTemplate(comp, endpointDTO.getCloudProvider(), project))
.collect(Collectors.toList());
- log.info("changed: {}", s);
-
- return s;
} catch (DatalabException e) {
log.error("Could not load list of computational templates for user: {}", user.getName(), e);
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/util/BillingUtils.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/util/BillingUtils.java
index 3ca334f..fe9575f 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/util/BillingUtils.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/util/BillingUtils.java
@@ -33,7 +33,6 @@ import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.time.format.FormatStyle;
import java.util.*;
-import java.util.stream.Collectors;
import java.util.stream.Stream;
import static com.epam.datalab.dto.billing.BillingResourceType.*;
@@ -142,16 +141,11 @@ public class BillingUtils {
}
public static Stream<BillingReportLine> exploratoryBillingDataStream(UserInstanceDTO userInstance, Integer maxSparkInstanceCount) {
- log.info("TEST LOG!!!: exploratoryBillingDataStream");
- log.info("TEST LOG!!!: userInstance: {}", userInstance);
-
-
final Stream<BillingReportLine> computationalStream = userInstance.getResources()
.stream()
.filter(cr -> cr.getComputationalId() != null)
.flatMap(cr -> {
final String computationalId = cr.getComputationalId().toLowerCase();
- log.info("TEST LOG!!!: cr: {}", cr);
return Stream.concat(Stream.of(
withUserProjectEndpoint(userInstance)
.resourceName(cr.getComputationalName())
@@ -184,9 +178,6 @@ public class BillingUtils {
getSlaveVolumes(userInstance, cr, maxSparkInstanceCount)
);
});
- log.info("TEST LOG!!!: computationalStream: {}", computationalStream);
- log.info("TEST LOG!!!: computationalStream: {}", computationalStream.collect(Collectors.toList()));
-
final String exploratoryName = userInstance.getExploratoryName();
final String exploratoryId = userInstance.getExploratoryId().toLowerCase();
@@ -210,9 +201,6 @@ public class BillingUtils {
.resourceType(VOLUME)
.build());
- log.info("TEST LOG!!!: exploratoryStream: {}", exploratoryStream);
- log.info("TEST LOG!!!: exploratoryStream: {}", exploratoryStream.collect(Collectors.toList()));
-
return Stream.concat(computationalStream, exploratoryStream);
}
@@ -242,15 +230,6 @@ public class BillingUtils {
}
public static String getComputationalShape(UserComputationalResource resource) {
- log.info("TEST LOG!!! getComputationalShape");
- log.info("TEST LOG!!! UserComputationalResource: {}", resource);
- log.info("TEST LOG!!! getTemplateName: {}", resource.getTemplateName());
- log.info("TEST LOG!!! getTotalInstanceCount: {}", resource.getTotalInstanceCount());
- log.info("TEST LOG!!! getDataengineShape: {}", resource.getDataengineShape());
- log.info("TEST LOG!!! getMasterNodeShape: {}", resource.getMasterNodeShape());
- log.info("TEST LOG!!!: DataEngineType.fromDockerImageName(resource.getImageName()) == DataEngineType.SPARK_STANDALONE: {} == {}",
- DataEngineType.fromDockerImageName(resource.getImageName()), DataEngineType.SPARK_STANDALONE);
-
return DataEngineType.fromDockerImageName(resource.getImageName()) != DataEngineType.SPARK_STANDALONE ?
String.format(DATAENGINE_NAME_FORMAT, resource.getDataengineInstanceCount(), resource.getDataengineShape()) :
String.format(DATAENGINE_SERVICE_NAME_FORMAT, resource.getMasterNodeShape(), resource.getTotalInstanceCount() - 1, resource.getSlaveNodeShape());
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java
index 9a77442..7b1ea0a 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java
@@ -654,10 +654,8 @@ public class RequestBuilder {
}
public UserEnvironmentResources newInfrastructureStatus(String user, CloudProvider cloudProvider, EnvResourceList resourceList) {
- UserEnvironmentResources userEnvironmentResources = newResourceSysBaseDTO(user, cloudProvider, UserEnvironmentResources.class)
+ return newResourceSysBaseDTO(user, cloudProvider, UserEnvironmentResources.class)
.withResourceList(resourceList);
- log.info("TEST LOG!!! userEnvironmentResources: {}", userEnvironmentResources);
- return userEnvironmentResources;
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org