You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by of...@apache.org on 2020/11/04 17:15:02 UTC
[incubator-datalab] branch DATALAB-1895 updated: [DATALAB-1895]
Sync up DataLab instance statuses with cloud statuses
This is an automated email from the ASF dual-hosted git repository.
ofuks pushed a commit to branch DATALAB-1895
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
The following commit(s) were added to refs/heads/DATALAB-1895 by this push:
new 8a40f7a [DATALAB-1895] Sync up DataLab instance statuses with cloud statuses
new 6badb37 Merge remote-tracking branch 'origin/DATALAB-1895' into DATALAB-1895
8a40f7a is described below
commit 8a40f7a86e7ef801df1c1ecee39be3883ddfb186
Author: Oleh Fuks <ol...@gmail.com>
AuthorDate: Wed Nov 4 12:20:50 2020 +0200
[DATALAB-1895] Sync up DataLab instance statuses with cloud statuses
---
.../java/com/epam/datalab/dto/UserInstanceDTO.java | 2 +
.../handlers/ResourcesStatusCallbackHandler.java | 31 ++---
.../resources/base/InfrastructureService.java | 7 +-
services/self-service/self-service.yml | 3 +
.../datalab/backendapi/dao/ExploratoryDAO.java | 22 +++-
.../resources/InfrastructureInfoResource.java | 4 +-
.../callback/EnvironmentStatusCallback.java | 25 ++--
.../CheckInfrastructureStatusScheduler.java | 130 +++++++++++++++++++++
.../service/InfrastructureInfoService.java | 3 +
.../service/impl/EndpointServiceImpl.java | 2 +-
.../impl/InfrastructureInfoServiceImpl.java | 32 ++++-
.../datalab/backendapi/util/RequestBuilder.java | 41 ++++---
.../service/impl/EndpointServiceImplTest.java | 26 +++--
13 files changed, 259 insertions(+), 69 deletions(-)
diff --git a/services/datalab-model/src/main/java/com/epam/datalab/dto/UserInstanceDTO.java b/services/datalab-model/src/main/java/com/epam/datalab/dto/UserInstanceDTO.java
index 12f7a20..45607f1 100644
--- a/services/datalab-model/src/main/java/com/epam/datalab/dto/UserInstanceDTO.java
+++ b/services/datalab-model/src/main/java/com/epam/datalab/dto/UserInstanceDTO.java
@@ -42,6 +42,8 @@ import java.util.Map;
public class UserInstanceDTO {
@JsonProperty("_id")
private String id;
+ @JsonProperty("instance_id")
+ private String instanceId;
@JsonProperty
private String user;
@JsonProperty("exploratory_name")
diff --git a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ResourcesStatusCallbackHandler.java b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ResourcesStatusCallbackHandler.java
index 53e8d8a..9ca477b 100644
--- a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ResourcesStatusCallbackHandler.java
+++ b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ResourcesStatusCallbackHandler.java
@@ -41,16 +41,14 @@ import static com.epam.datalab.rest.contracts.ApiCallbacks.STATUS_URI;
@Slf4j
public class ResourcesStatusCallbackHandler extends ResourceCallbackHandler<EnvStatusDTO> {
+ private EnvResourceList datalabResourceList;
+
@JsonCreator
public ResourcesStatusCallbackHandler(
- @JacksonInject RESTService selfService, @JsonProperty("action") DockerAction
- action, @JsonProperty("uuid") String uuid, @JsonProperty("user") String user) {
+ @JacksonInject RESTService selfService, @JsonProperty("action") DockerAction action, @JsonProperty("uuid") String uuid,
+ @JsonProperty("user") String user, EnvResourceList resourceList) {
super(selfService, user, uuid, action);
- }
-
- @Override
- protected String getCallbackURI() {
- return INFRASTRUCTURE + STATUS_URI;
+ this.datalabResourceList = resourceList;
}
@Override
@@ -59,15 +57,15 @@ public class ResourcesStatusCallbackHandler extends ResourceCallbackHandler<EnvS
return baseStatus;
}
- EnvResourceList resourceList;
+ EnvResourceList cloudResourceList;
try {
- resourceList = mapper.readValue(resultNode.toString(), EnvResourceList.class);
+ cloudResourceList = mapper.readValue(resultNode.toString(), EnvResourceList.class);
} catch (IOException e) {
- throw new DatalabException("Docker response for UUID " + getUUID() + " not valid: " + e.getLocalizedMessage()
- , e);
+ throw new DatalabException("Docker response for UUID " + getUUID() + " not valid: " + e.getLocalizedMessage(), e);
}
- baseStatus.withResourceList(resourceList)
+ baseStatus
+ .withResourceList(cloudResourceList)
.withUptime(Date.from(Instant.now()));
log.trace("Inner status {}", baseStatus);
@@ -76,17 +74,22 @@ public class ResourcesStatusCallbackHandler extends ResourceCallbackHandler<EnvS
}
@Override
- public boolean handle(String fileName, byte[] content) throws Exception {
+ public boolean handle(String fileName, byte[] content) {
try {
return super.handle(fileName, content);
} catch (Exception e) {
- log.warn("Could not retrive the status of resources for UUID {} and user {}: {}",
+ log.warn("Could not retrieve the status of resources for UUID {} and user {}: {}",
getUUID(), getUser(), e.getLocalizedMessage(), e);
}
return true; // Always necessary return true for status response
}
@Override
+ protected String getCallbackURI() {
+ return INFRASTRUCTURE + STATUS_URI;
+ }
+
+ @Override
public void handleError(String errorMessage) {
// Nothing action for status response
}
diff --git a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/resources/base/InfrastructureService.java b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/resources/base/InfrastructureService.java
index e9c29ce..6a2d870 100644
--- a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/resources/base/InfrastructureService.java
+++ b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/resources/base/InfrastructureService.java
@@ -33,6 +33,7 @@ import com.epam.datalab.backendapi.core.response.handlers.ResourcesStatusCallbac
import com.epam.datalab.cloud.CloudProvider;
import com.epam.datalab.dto.UserEnvironmentResources;
import com.epam.datalab.dto.status.EnvResource;
+import com.epam.datalab.dto.status.EnvResourceList;
import com.epam.datalab.exceptions.DatalabException;
import com.epam.datalab.process.model.ProcessInfo;
import com.epam.datalab.rest.client.RESTService;
@@ -68,7 +69,7 @@ public abstract class InfrastructureService implements DockerCommands {
String uuid = DockerCommands.generateUUID();
folderListenerExecutor.start(configuration.getImagesDirectory(),
configuration.getRequestEnvStatusTimeout(),
- getFileHandlerCallback(dockerAction, uuid, iamUser));
+ getFileHandlerCallback(dockerAction, uuid, iamUser, dto.getResourceList()));
try {
removeResourcesWithRunningContainers(username, dto);
@@ -143,8 +144,8 @@ public abstract class InfrastructureService implements DockerCommands {
return runningContainerNames.stream().noneMatch(container -> container.matches(regex));
}
- protected FileHandlerCallback getFileHandlerCallback(DockerAction action, String uuid, String user) {
- return new ResourcesStatusCallbackHandler(selfService, action, uuid, user);
+ protected FileHandlerCallback getFileHandlerCallback(DockerAction action, String uuid, String user, EnvResourceList resourceList) {
+ return new ResourcesStatusCallbackHandler(selfService, action, uuid, user, resourceList);
}
private String nameContainer(String user, DockerAction action, String name) {
diff --git a/services/self-service/self-service.yml b/services/self-service/self-service.yml
index 1ef10a5..fb2e5cf 100644
--- a/services/self-service/self-service.yml
+++ b/services/self-service/self-service.yml
@@ -126,6 +126,9 @@ schedulers:
inactivity:
enabled: false
cron: "0 0 0/2 ? * * *"
+ checkInfrastructureStatusScheduler:
+ enabled: true
+ cron: "0 0/15 * ? * *"
startComputationalScheduler:
enabled: true
cron: "*/20 * * ? * * *"
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/dao/ExploratoryDAO.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/dao/ExploratoryDAO.java
index 599acaa..0657107 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/dao/ExploratoryDAO.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/dao/ExploratoryDAO.java
@@ -218,18 +218,30 @@ public class ExploratoryDAO extends BaseDAO {
false);
}
- public List<UserInstanceDTO> fetchExploratoriesByEndpointWhereStatusNotIn(String endpoint,
- List<UserInstanceStatus> statuses) {
+ public List<UserInstanceDTO> fetchExploratoriesByEndpointWhereStatusIn(List<String> endpoints, List<UserInstanceStatus> statuses,
+ boolean computationalFieldsRequired) {
final List<String> exploratoryStatusList = statusList(statuses);
return getUserInstances(
and(
- eq(ENDPOINT, endpoint),
- not(in(STATUS, exploratoryStatusList))
+ in(ENDPOINT, endpoints),
+ in(STATUS, exploratoryStatusList)
),
- false);
+ computationalFieldsRequired);
}
+ public List<UserInstanceDTO> fetchExploratoriesByEndpointWhereStatusNotIn(String endpoint, List<UserInstanceStatus> statuses,
+ boolean computationalFieldsRequired) {
+ final List<String> exploratoryStatusList = statusList(statuses);
+
+ return getUserInstances(
+ and(
+ eq(ENDPOINT, endpoint),
+ not(in(STATUS, exploratoryStatusList))
+ ),
+ computationalFieldsRequired);
+ }
+
private List<UserInstanceDTO> getUserInstances(Bson condition, boolean computationalFieldsRequired) {
return stream(getCollection(USER_INSTANCES)
.find(condition)
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/InfrastructureInfoResource.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/InfrastructureInfoResource.java
index 725e369..c477c1e 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/InfrastructureInfoResource.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/InfrastructureInfoResource.java
@@ -46,7 +46,7 @@ import java.util.List;
@Slf4j
public class InfrastructureInfoResource {
- private InfrastructureInfoService infrastructureInfoService;
+ private final InfrastructureInfoService infrastructureInfoService;
@Inject
public InfrastructureInfoResource(InfrastructureInfoService infrastructureInfoService) {
@@ -82,7 +82,6 @@ public class InfrastructureInfoResource {
@Path("/info")
public List<ProjectInfrastructureInfo> getUserResources(@Auth UserInfo userInfo) {
return infrastructureInfoService.getUserResources(userInfo);
-
}
@GET
@@ -90,6 +89,5 @@ public class InfrastructureInfoResource {
public Response getVersion(@Auth UserInfo userInfo) {
return Response.ok(infrastructureInfoService.getInfrastructureMetaInfo())
.build();
-
}
}
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/callback/EnvironmentStatusCallback.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/callback/EnvironmentStatusCallback.java
index e9a8202..7775164 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/callback/EnvironmentStatusCallback.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/callback/EnvironmentStatusCallback.java
@@ -21,9 +21,7 @@ package com.epam.datalab.backendapi.resources.callback;
import com.epam.datalab.backendapi.dao.EnvDAO;
import com.epam.datalab.backendapi.domain.RequestId;
-import com.epam.datalab.dto.UserInstanceStatus;
import com.epam.datalab.dto.status.EnvStatusDTO;
-import com.epam.datalab.exceptions.DatalabException;
import com.epam.datalab.rest.contracts.ApiCallbacks;
import com.google.inject.Inject;
import lombok.extern.slf4j.Slf4j;
@@ -55,17 +53,18 @@ public class EnvironmentStatusCallback {
@POST
@Path(ApiCallbacks.STATUS_URI)
public Response status(EnvStatusDTO dto) {
- log.trace("Updating the status of resources for user {}: {}", dto.getUser(), dto);
- requestId.checkAndRemove(dto.getRequestId());
- try {
- if (UserInstanceStatus.FAILED == UserInstanceStatus.of(dto.getStatus())) {
- log.warn("Request for the status of resources for user {} fails: {}", dto.getUser(), dto.getErrorMessage());
- } else {
- envDAO.updateEnvStatus(dto.getUser(), null, dto.getResourceList());
- }
- } catch (DatalabException e) {
- log.warn("Could not update status of resources for user {}: {}", dto.getUser(), e.getLocalizedMessage(), e);
- }
+ log.info("EnvStatusDTO__ {} ", dto);
+// log.trace("Updating the status of resources for user {}: {}", dto.getUser(), dto);
+// requestId.checkAndRemove(dto.getRequestId());
+// try {
+// if (UserInstanceStatus.FAILED == UserInstanceStatus.of(dto.getStatus())) {
+// log.warn("Request for the status of resources for user {} fails: {}", dto.getUser(), dto.getErrorMessage());
+// } else {
+// envDAO.updateEnvStatus(dto.getUser(), null, dto.getResourceList());
+// }
+// } catch (DatalabException e) {
+// log.warn("Could not update status of resources for user {}: {}", dto.getUser(), e.getLocalizedMessage(), e);
+// }
return Response.ok().build();
}
}
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/CheckInfrastructureStatusScheduler.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/CheckInfrastructureStatusScheduler.java
new file mode 100644
index 0000000..55caf85
--- /dev/null
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/CheckInfrastructureStatusScheduler.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.datalab.backendapi.schedulers;
+
+import com.epam.datalab.auth.UserInfo;
+import com.epam.datalab.backendapi.dao.ExploratoryDAO;
+import com.epam.datalab.backendapi.domain.EndpointDTO;
+import com.epam.datalab.backendapi.domain.ProjectDTO;
+import com.epam.datalab.backendapi.schedulers.internal.Scheduled;
+import com.epam.datalab.backendapi.service.EndpointService;
+import com.epam.datalab.backendapi.service.InfrastructureInfoService;
+import com.epam.datalab.backendapi.service.ProjectService;
+import com.epam.datalab.backendapi.service.SecurityService;
+import com.epam.datalab.dto.UserInstanceDTO;
+import com.epam.datalab.dto.UserInstanceStatus;
+import com.epam.datalab.dto.base.DataEngineType;
+import com.epam.datalab.dto.status.EnvResource;
+import com.epam.datalab.model.ResourceType;
+import com.google.inject.Inject;
+import org.quartz.Job;
+import org.quartz.JobExecutionContext;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+@Scheduled("checkInfrastructureStatusScheduler")
+public class CheckInfrastructureStatusScheduler implements Job {
+
+ private static final List<UserInstanceStatus> statusesToCheck = Arrays.asList(UserInstanceStatus.RUNNING, UserInstanceStatus.STOPPED);
+ private final InfrastructureInfoService infrastructureInfoService;
+ private final SecurityService securityService;
+ private final EndpointService endpointService;
+ private final ExploratoryDAO exploratoryDAO;
+ private final ProjectService projectService;
+
+ @Inject
+ public CheckInfrastructureStatusScheduler(InfrastructureInfoService infrastructureInfoService, SecurityService securityService,
+ EndpointService endpointService, ExploratoryDAO exploratoryDAO, ProjectService projectService) {
+ this.infrastructureInfoService = infrastructureInfoService;
+ this.securityService = securityService;
+ this.endpointService = endpointService;
+ this.exploratoryDAO = exploratoryDAO;
+ this.projectService = projectService;
+ }
+
+ @Override
+ public void execute(JobExecutionContext context) {
+ UserInfo serviceUser = securityService.getServiceAccountInfo("admin");
+
+ List<String> activeEndpoints = endpointService.getEndpointsWithStatus(EndpointDTO.EndpointStatus.ACTIVE)
+ .stream()
+ .map(EndpointDTO::getName)
+ .collect(Collectors.toList());
+
+ List<UserInstanceDTO> userInstanceDTOS = exploratoryDAO.fetchExploratoriesByEndpointWhereStatusIn(activeEndpoints, statusesToCheck, Boolean.TRUE);
+
+ Map<String, List<EnvResource>> exploratoryAndSparkInstances = userInstanceDTOS
+ .stream()
+ .collect(Collectors.toMap(UserInstanceDTO::getEndpoint, this::getExploratoryAndSparkInstances));
+
+ activeEndpoints.forEach(e -> {
+ List<EnvResource> hostInstances = Stream.of(getEdgeInstances(e), exploratoryAndSparkInstances.get(e))
+ .flatMap(Collection::stream)
+ .collect(Collectors.toList());
+ infrastructureInfoService.updateInfrastructureStatuses(serviceUser, e, hostInstances, Collections.emptyList());
+ }
+ );
+ }
+
+ private List<EnvResource> getExploratoryAndSparkInstances(UserInstanceDTO userInstanceDTO) {
+ List<EnvResource> instances = userInstanceDTO.getResources()
+ .stream()
+ .filter(c -> DataEngineType.SPARK_STANDALONE == DataEngineType.fromDockerImageName(c.getImageName()))
+ .filter(c -> statusesToCheck.contains(UserInstanceStatus.of(c.getStatus())))
+ .map(r -> new EnvResource()
+ .withId(r.getInstanceId())
+ .withName(r.getComputationalName())
+ .withStatus(r.getStatus())
+ .withResourceType(ResourceType.COMPUTATIONAL))
+ .collect(Collectors.toList());
+
+ instances.add(new EnvResource()
+ .withId(userInstanceDTO.getInstanceId())
+ .withName(userInstanceDTO.getExploratoryName())
+ .withStatus(userInstanceDTO.getStatus())
+ .withResourceType(ResourceType.EXPLORATORY));
+
+ return instances;
+ }
+
+ private List<EnvResource> getEdgeInstances(String endpoint) {
+ return projectService.getProjectsByEndpoint(endpoint)
+ .stream()
+ .map(ProjectDTO::getEndpoints)
+ .flatMap(Collection::stream)
+ .filter(e -> statusesToCheck.contains(e.getStatus()))
+ .filter(e -> e.getName().equals(endpoint))
+ .filter(e -> Objects.nonNull(e.getEdgeInfo()))
+ .map(e -> new EnvResource()
+ .withId(e.getEdgeInfo().getInstanceId())
+ .withName(e.getName())
+ .withStatus(e.getStatus().toString())
+ .withResourceType(ResourceType.EDGE)
+ )
+ .collect(Collectors.toList());
+ }
+}
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/InfrastructureInfoService.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/InfrastructureInfoService.java
index c64b99e..b8950ef 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/InfrastructureInfoService.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/InfrastructureInfoService.java
@@ -23,6 +23,7 @@ import com.epam.datalab.auth.UserInfo;
import com.epam.datalab.backendapi.resources.dto.HealthStatusPageDTO;
import com.epam.datalab.backendapi.resources.dto.ProjectInfrastructureInfo;
import com.epam.datalab.dto.InfrastructureMetaInfoDTO;
+import com.epam.datalab.dto.status.EnvResource;
import java.util.List;
@@ -32,4 +33,6 @@ public interface InfrastructureInfoService {
HealthStatusPageDTO getHeathStatus(UserInfo user);
InfrastructureMetaInfoDTO getInfrastructureMetaInfo();
+
+ void updateInfrastructureStatuses(UserInfo user, String endpoint, List<EnvResource> hostInstanceIds, List<EnvResource> clusterInstanceIds);
}
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/EndpointServiceImpl.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/EndpointServiceImpl.java
index cede329..d284c96 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/EndpointServiceImpl.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/EndpointServiceImpl.java
@@ -89,7 +89,7 @@ public class EndpointServiceImpl implements EndpointService {
@Override
public EndpointResourcesDTO getEndpointResources(String endpoint) {
List<UserInstanceDTO> exploratories = exploratoryDAO.fetchExploratoriesByEndpointWhereStatusNotIn(endpoint,
- Arrays.asList(UserInstanceStatus.TERMINATED, UserInstanceStatus.FAILED));
+ Arrays.asList(UserInstanceStatus.TERMINATED, UserInstanceStatus.FAILED), Boolean.FALSE);
List<ProjectDTO> projects = projectService.getProjectsByEndpoint(endpoint);
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/InfrastructureInfoServiceImpl.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/InfrastructureInfoServiceImpl.java
index e1ddbe0..efd4088 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/InfrastructureInfoServiceImpl.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/InfrastructureInfoServiceImpl.java
@@ -26,6 +26,7 @@ import com.epam.datalab.backendapi.domain.BillingReport;
import com.epam.datalab.backendapi.domain.EndpointDTO;
import com.epam.datalab.backendapi.domain.ProjectDTO;
import com.epam.datalab.backendapi.domain.ProjectEndpointDTO;
+import com.epam.datalab.backendapi.domain.RequestId;
import com.epam.datalab.backendapi.resources.dto.HealthStatusEnum;
import com.epam.datalab.backendapi.resources.dto.HealthStatusPageDTO;
import com.epam.datalab.backendapi.resources.dto.ProjectInfrastructureInfo;
@@ -35,6 +36,8 @@ import com.epam.datalab.backendapi.service.BillingService;
import com.epam.datalab.backendapi.service.EndpointService;
import com.epam.datalab.backendapi.service.InfrastructureInfoService;
import com.epam.datalab.backendapi.service.ProjectService;
+import com.epam.datalab.backendapi.util.RequestBuilder;
+import com.epam.datalab.constants.ServiceConsts;
import com.epam.datalab.dto.InfrastructureMetaInfoDTO;
import com.epam.datalab.dto.UserInstanceDTO;
import com.epam.datalab.dto.aws.edge.EdgeInfoAws;
@@ -42,7 +45,11 @@ import com.epam.datalab.dto.azure.edge.EdgeInfoAzure;
import com.epam.datalab.dto.base.edge.EdgeInfo;
import com.epam.datalab.dto.computational.UserComputationalResource;
import com.epam.datalab.dto.gcp.edge.EdgeInfoGcp;
+import com.epam.datalab.dto.status.EnvResource;
+import com.epam.datalab.dto.status.EnvResourceList;
+import com.epam.datalab.rest.client.RESTService;
import com.google.inject.Inject;
+import com.google.inject.name.Named;
import com.jcabi.manifests.Manifests;
import lombok.extern.slf4j.Slf4j;
@@ -55,26 +62,36 @@ import java.util.stream.Collectors;
@Slf4j
public class InfrastructureInfoServiceImpl implements InfrastructureInfoService {
+
private static final String RELEASE_NOTES_FORMAT = "https://github.com/apache/incubator-datalab/blob/%s/RELEASE_NOTES.md";
private static final String PERMISSION_VIEW = "/api/bucket/view";
private static final String PERMISSION_UPLOAD = "/api/bucket/upload";
private static final String PERMISSION_DOWNLOAD = "/api/bucket/download";
private static final String PERMISSION_DELETE = "/api/bucket/delete";
+ private static final String INFRASTRUCTURE_STAUS = "infrastructure/status";
private final ExploratoryDAO expDAO;
private final SelfServiceApplicationConfiguration configuration;
private final ProjectService projectService;
private final EndpointService endpointService;
private final BillingService billingService;
+ private final RequestBuilder requestBuilder;
+ private final RESTService provisioningService;
+ private final RequestId requestId;
+
@Inject
public InfrastructureInfoServiceImpl(ExploratoryDAO expDAO, SelfServiceApplicationConfiguration configuration, ProjectService projectService,
- EndpointService endpointService, BillingService billingService) {
+ EndpointService endpointService, BillingService billingService, RequestBuilder requestBuilder,
+ @Named(ServiceConsts.PROVISIONING_SERVICE_NAME) RESTService provisioningService, RequestId requestId) {
this.expDAO = expDAO;
this.configuration = configuration;
this.projectService = projectService;
this.endpointService = endpointService;
this.billingService = billingService;
+ this.requestBuilder = requestBuilder;
+ this.provisioningService = provisioningService;
+ this.requestId = requestId;
}
@Override
@@ -128,6 +145,19 @@ public class InfrastructureInfoServiceImpl implements InfrastructureInfoService
.build();
}
+ @Override
+ public void updateInfrastructureStatuses(UserInfo user, String endpoint, List<EnvResource> hostInstances, List<EnvResource> clusterInstances) {
+ EnvResourceList envResourceList = new EnvResourceList()
+ .withClusterList(clusterInstances)
+ .withHostList(hostInstances);
+
+ EndpointDTO endpointDTO = endpointService.get(endpoint);
+ String uuid = provisioningService.post(endpointDTO.getUrl() + INFRASTRUCTURE_STAUS, user.getAccessToken(),
+ requestBuilder.newInfrastructureStatus(user.getName(), endpointDTO.getCloudProvider(), envResourceList),
+ String.class);
+ requestId.put(user.getName(), uuid);
+ }
+
private List<BillingReport> getExploratoryBillingData(List<UserInstanceDTO> exploratories) {
return exploratories
.stream()
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java
index 8e0845d..6ece8db 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java
@@ -34,6 +34,7 @@ import com.epam.datalab.dto.LibListComputationalDTO;
import com.epam.datalab.dto.LibListExploratoryDTO;
import com.epam.datalab.dto.ResourceBaseDTO;
import com.epam.datalab.dto.ResourceSysBaseDTO;
+import com.epam.datalab.dto.UserEnvironmentResources;
import com.epam.datalab.dto.UserInstanceDTO;
import com.epam.datalab.dto.aws.AwsCloudSettings;
import com.epam.datalab.dto.aws.computational.AwsComputationalTerminateDTO;
@@ -72,6 +73,7 @@ import com.epam.datalab.dto.gcp.computational.SparkComputationalCreateGcp;
import com.epam.datalab.dto.gcp.exploratory.ExploratoryCreateGcp;
import com.epam.datalab.dto.project.ProjectActionDTO;
import com.epam.datalab.dto.project.ProjectCreateDTO;
+import com.epam.datalab.dto.status.EnvResourceList;
import com.epam.datalab.exceptions.DatalabException;
import com.epam.datalab.model.exploratory.Exploratory;
import com.epam.datalab.util.UsernameUtils;
@@ -616,25 +618,30 @@ public class RequestBuilder {
.key(projectDTO.getKey().replace("\n", ""))
.name(projectDTO.getName())
.tag(projectDTO.getTag())
- .endpoint(endpointDTO.getName())
- .build()
- .withCloudSettings(cloudSettings(userInfo.getName(), endpointDTO.getCloudProvider()));
+ .endpoint(endpointDTO.getName())
+ .build()
+ .withCloudSettings(cloudSettings(userInfo.getName(), endpointDTO.getCloudProvider()));
}
- public ProjectActionDTO newProjectAction(UserInfo userInfo, String project, EndpointDTO endpointDTO) {
- return new ProjectActionDTO(project, endpointDTO.getName())
- .withCloudSettings(cloudSettings(userInfo.getName(), endpointDTO.getCloudProvider()));
- }
-
- /**
- * Returns application name basing on docker image
- *
- * @param imageName docker image name
- * @return application name
- */
- private String getApplicationNameFromImage(String imageName) {
- if (imageName != null) {
- int pos = imageName.indexOf('-');
+ public ProjectActionDTO newProjectAction(UserInfo userInfo, String project, EndpointDTO endpointDTO) {
+ return new ProjectActionDTO(project, endpointDTO.getName())
+ .withCloudSettings(cloudSettings(userInfo.getName(), endpointDTO.getCloudProvider()));
+ }
+
+ public UserEnvironmentResources newInfrastructureStatus(String user, CloudProvider cloudProvider, EnvResourceList resourceList) {
+ return newResourceSysBaseDTO(user, cloudProvider, UserEnvironmentResources.class)
+ .withResourceList(resourceList);
+ }
+
+ /**
+ * Returns application name basing on docker image
+ *
+ * @param imageName docker image name
+ * @return application name
+ */
+ private String getApplicationNameFromImage(String imageName) {
+ if (imageName != null) {
+ int pos = imageName.indexOf('-');
if (pos > 0) {
return imageName.substring(pos + 1);
}
diff --git a/services/self-service/src/test/java/com/epam/datalab/backendapi/service/impl/EndpointServiceImplTest.java b/services/self-service/src/test/java/com/epam/datalab/backendapi/service/impl/EndpointServiceImplTest.java
index 5b978b7..a00a2f9 100644
--- a/services/self-service/src/test/java/com/epam/datalab/backendapi/service/impl/EndpointServiceImplTest.java
+++ b/services/self-service/src/test/java/com/epam/datalab/backendapi/service/impl/EndpointServiceImplTest.java
@@ -50,6 +50,7 @@ import java.util.Optional;
import static org.junit.Assert.assertEquals;
import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.anyListOf;
import static org.mockito.Matchers.anyString;
import static org.mockito.Mockito.mock;
@@ -105,18 +106,19 @@ public class EndpointServiceImplTest extends TestBase {
@Test
public void getEndpointResources() {
- List<UserInstanceDTO> userInstances = getUserInstances();
- List<ProjectDTO> projectDTOs = getProjectDTOs();
- when(exploratoryDAO.fetchExploratoriesByEndpointWhereStatusNotIn(anyString(), anyListOf(UserInstanceStatus.class)))
- .thenReturn(userInstances);
- when(projectService.getProjectsByEndpoint(anyString())).thenReturn(projectDTOs);
-
- EndpointResourcesDTO actualEndpointResources = endpointService.getEndpointResources(ENDPOINT_NAME);
-
- assertEquals("objects should be equal", new EndpointResourcesDTO(userInstances, projectDTOs), actualEndpointResources);
- verify(exploratoryDAO).fetchExploratoriesByEndpointWhereStatusNotIn(ENDPOINT_NAME, Arrays.asList(UserInstanceStatus.TERMINATED, UserInstanceStatus.FAILED));
- verify(projectService).getProjectsByEndpoint(ENDPOINT_NAME);
- verifyNoMoreInteractions(exploratoryDAO, projectService);
+ List<UserInstanceDTO> userInstances = getUserInstances();
+ List<ProjectDTO> projectDTOs = getProjectDTOs();
+ when(exploratoryDAO.fetchExploratoriesByEndpointWhereStatusNotIn(anyString(), anyListOf(UserInstanceStatus.class), anyBoolean()))
+ .thenReturn(userInstances);
+ when(projectService.getProjectsByEndpoint(anyString())).thenReturn(projectDTOs);
+
+ EndpointResourcesDTO actualEndpointResources = endpointService.getEndpointResources(ENDPOINT_NAME);
+
+ assertEquals("objects should be equal", new EndpointResourcesDTO(userInstances, projectDTOs), actualEndpointResources);
+ verify(exploratoryDAO).fetchExploratoriesByEndpointWhereStatusNotIn(ENDPOINT_NAME, Arrays.asList(UserInstanceStatus.TERMINATED,
+ UserInstanceStatus.FAILED), Boolean.FALSE);
+ verify(projectService).getProjectsByEndpoint(ENDPOINT_NAME);
+ verifyNoMoreInteractions(exploratoryDAO, projectService);
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org