You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by of...@apache.org on 2020/04/10 14:30:20 UTC
[incubator-dlab] branch DLAB-1571 updated: remote billing
This is an automated email from the ASF dual-hosted git repository.
ofuks pushed a commit to branch DLAB-1571
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/DLAB-1571 by this push:
new ab0dea3 remote billing
ab0dea3 is described below
commit ab0dea360312f3660d6958337905252b3440d7eb
Author: Oleh Fuks <ol...@gmail.com>
AuthorDate: Fri Apr 10 17:29:57 2020 +0300
remote billing
---
.../epam/dlab/backendapi/dao/BaseBillingDAO.java | 15 ++--
.../com/epam/dlab/backendapi/dao/BillingDAO.java | 2 +-
.../epam/dlab/backendapi/domain/BillingReport.java | 1 +
.../dlab/backendapi/domain/BillingReportLine.java | 2 +
.../resources/dto/ProjectInfrastructureInfo.java | 4 +-
.../dlab/backendapi/service/BillingService.java | 3 +-
.../backendapi/service/ExploratoryService.java | 2 +
.../service/impl/BillingServiceImpl.java | 83 +++++++++++++++-------
.../service/impl/ExploratoryServiceImpl.java | 12 +++-
.../impl/InfrastructureInfoServiceImpl.java | 15 ++--
.../epam/dlab/backendapi/util/BillingUtils.java | 41 +++++------
11 files changed, 111 insertions(+), 69 deletions(-)
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/BaseBillingDAO.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/BaseBillingDAO.java
index 4c1612f..565a9d4 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/BaseBillingDAO.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/BaseBillingDAO.java
@@ -71,6 +71,10 @@ public class BaseBillingDAO extends BaseDAO implements BillingDAO {
private static final String PRODUCT = "product";
private static final String CURRENCY = "currency";
private static final String COST = "cost";
+ private static final String RESOURCE_NAME = "resource_name";
+ private static final String ENDPOINT = "endpoint";
+ private static final String SHAPE = "shape";
+ private static final String EXPLORATORY = "exploratoryName";
@Inject
protected SettingsDAO settings;
@@ -130,8 +134,8 @@ public class BaseBillingDAO extends BaseDAO implements BillingDAO {
}
@Override
- public List<BillingReportLine> findBillingData(List<String> dlabIds) {
- return find(BILLING, in(DLAB_ID, dlabIds), BillingReportLine.class);
+ public List<BillingReportLine> findBillingData(String project, String endpoint, List<String> resourceNames) {
+ return find(BILLING, and(eq(PROJECT, project), eq(ENDPOINT, endpoint), in(RESOURCE_NAME, resourceNames)), BillingReportLine.class);
}
@Override
@@ -177,7 +181,7 @@ public class BaseBillingDAO extends BaseDAO implements BillingDAO {
}
private Bson groupCriteria() {
- return group(getGroupingFields(USER, DLAB_ID, RESOURCE_TYPE, PROJECT, PRODUCT, CURRENCY),
+ return group(getGroupingFields(USER, DLAB_ID, RESOURCE_TYPE, RESOURCE_NAME, PROJECT, PRODUCT, CURRENCY, SHAPE, EXPLORATORY),
sum(COST, "$" + COST),
min(FROM, "$" + FROM),
max(TO, "$" + TO));
@@ -216,9 +220,12 @@ public class BaseBillingDAO extends BaseDAO implements BillingDAO {
return BillingReportLine.builder()
.dlabId(id.getString(DLAB_ID))
.project(id.getString(PROJECT))
+ .resourceName(id.getString(RESOURCE_NAME))
+ .exploratoryName(id.getString(EXPLORATORY))
+ .shape(id.getString(SHAPE))
.user(id.getString(USER))
.product(id.getString(PRODUCT))
- .resourceType(BillingResourceType.valueOf(id.getString(RESOURCE_TYPE)))
+ .resourceType(Optional.ofNullable(id.getString(RESOURCE_TYPE)).map(BillingResourceType::valueOf).orElse(null))
.usageDateFrom(d.getDate(FROM).toInstant().atZone(ZoneId.systemDefault()).toLocalDate())
.usageDateTo(d.getDate(TO).toInstant().atZone(ZoneId.systemDefault()).toLocalDate())
.cost(BigDecimal.valueOf(d.getDouble(COST)).setScale(2, BigDecimal.ROUND_HALF_UP).doubleValue())
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/BillingDAO.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/BillingDAO.java
index 3a00d5b..2e4e945 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/BillingDAO.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/BillingDAO.java
@@ -42,7 +42,7 @@ public interface BillingDAO {
boolean isProjectQuoteReached(String project);
- List<BillingReportLine> findBillingData(List<String> dlabIds);
+ List<BillingReportLine> findBillingData(String project, String endpoint, List<String> resourceNames);
List<BillingReportLine> aggregateBillingData(BillingFilter filter);
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/BillingReport.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/BillingReport.java
index 8722e73..2bb2062 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/BillingReport.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/BillingReport.java
@@ -30,6 +30,7 @@ import java.util.List;
@Builder
public class BillingReport {
private String sbn;
+ private String name;
@JsonProperty("report_lines")
private List<BillingReportLine> reportLines;
@JsonProperty("from")
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/BillingReportLine.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/BillingReportLine.java
index 123a5c3..a9cdd12 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/BillingReportLine.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/BillingReportLine.java
@@ -37,6 +37,7 @@ public class BillingReportLine {
@JsonProperty("resource_name")
private String resourceName;
private String project;
+ private String endpoint;
private String user;
@JsonProperty("from")
private LocalDate usageDateFrom;
@@ -51,4 +52,5 @@ public class BillingReportLine {
private BillingResourceType resourceType;
private UserInstanceStatus status;
private String shape;
+ private String exploratoryName;
}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/ProjectInfrastructureInfo.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/ProjectInfrastructureInfo.java
index d215ac2..b9dfd89 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/ProjectInfrastructureInfo.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/ProjectInfrastructureInfo.java
@@ -19,7 +19,7 @@
package com.epam.dlab.backendapi.resources.dto;
-import com.epam.dlab.backendapi.domain.BillingReportLine;
+import com.epam.dlab.backendapi.domain.BillingReport;
import com.epam.dlab.backendapi.domain.EndpointDTO;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor;
@@ -41,7 +41,7 @@ public class ProjectInfrastructureInfo {
@JsonProperty
private Iterable<Document> exploratory;
@JsonProperty
- private List<BillingReportLine> exploratoryBilling;
+ private List<BillingReport> exploratoryBilling;
@JsonProperty
private List<EndpointDTO> endpoints;
}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/BillingService.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/BillingService.java
index 7d30f74..b76b141 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/BillingService.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/BillingService.java
@@ -21,7 +21,6 @@ package com.epam.dlab.backendapi.service;
import com.epam.dlab.auth.UserInfo;
import com.epam.dlab.backendapi.domain.BillingReport;
-import com.epam.dlab.backendapi.domain.BillingReportLine;
import com.epam.dlab.backendapi.resources.dto.BillingFilter;
import java.util.List;
@@ -31,7 +30,7 @@ public interface BillingService {
String downloadReport(UserInfo userInfo, BillingFilter filter);
- List<BillingReportLine> getExploratoryBillingData(String exploratoryId, List<String> resources);
+ BillingReport getExploratoryBillingData(String project, String endpoint, String exploratoryName, List<String> compNames);
void updateRemoteBillingData(UserInfo userInfo);
}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/ExploratoryService.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/ExploratoryService.java
index 5a43fa6..807df17 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/ExploratoryService.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/ExploratoryService.java
@@ -48,6 +48,8 @@ public interface ExploratoryService {
Optional<UserInstanceDTO> getUserInstance(String user, String project, String exploratoryName);
+ Optional<UserInstanceDTO> getUserInstance(String user, String project, String exploratoryName, boolean includeCompResources);
+
List<UserInstanceDTO> findAll();
List<UserInstanceDTO> findAll(Set<ProjectDTO> projects);
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BillingServiceImpl.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BillingServiceImpl.java
index a129fc6..f8bd787 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BillingServiceImpl.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BillingServiceImpl.java
@@ -38,7 +38,9 @@ import com.epam.dlab.backendapi.service.ProjectService;
import com.epam.dlab.backendapi.util.BillingUtils;
import com.epam.dlab.cloud.CloudProvider;
import com.epam.dlab.constants.ServiceConsts;
+import com.epam.dlab.dto.UserInstanceStatus;
import com.epam.dlab.dto.billing.BillingData;
+import com.epam.dlab.dto.billing.BillingResourceType;
import com.epam.dlab.exceptions.DlabException;
import com.epam.dlab.rest.client.RESTService;
import com.google.common.collect.Lists;
@@ -53,6 +55,7 @@ import java.math.BigDecimal;
import java.net.URI;
import java.net.URISyntaxException;
import java.time.LocalDate;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
@@ -96,12 +99,16 @@ public class BillingServiceImpl implements BillingService {
@Override
public BillingReport getBillingReport(UserInfo user, BillingFilter filter) {
setUserFilter(user, filter);
- List<BillingReportLine> billingReportLines = billingDAO.aggregateBillingData(filter);
- LocalDate min = billingReportLines.stream().min(Comparator.comparing(BillingReportLine::getUsageDateFrom)).map(BillingReportLine::getUsageDateFrom).orElse(null);
- LocalDate max = billingReportLines.stream().max(Comparator.comparing(BillingReportLine::getUsageDateTo)).map(BillingReportLine::getUsageDateTo).orElse(null);
- double sum = billingReportLines.stream().mapToDouble(BillingReportLine::getCost).sum();
- String currency = billingReportLines.stream().map(BillingReportLine::getCurrency).distinct().count() == 1 ? billingReportLines.get(0).getCurrency() : null;
+ List<BillingReportLine> billingReportLines = billingDAO.aggregateBillingData(filter)
+ .stream()
+ .peek(this::appendStatuses)
+ .collect(Collectors.toList());
+ final LocalDate min = billingReportLines.stream().min(Comparator.comparing(BillingReportLine::getUsageDateFrom)).map(BillingReportLine::getUsageDateFrom).orElse(null);
+ final LocalDate max = billingReportLines.stream().max(Comparator.comparing(BillingReportLine::getUsageDateTo)).map(BillingReportLine::getUsageDateTo).orElse(null);
+ final double sum = billingReportLines.stream().mapToDouble(BillingReportLine::getCost).sum();
+ final String currency = billingReportLines.stream().map(BillingReportLine::getCurrency).distinct().count() == 1 ? billingReportLines.get(0).getCurrency() : null;
return BillingReport.builder()
+ .name("Billing report")
.sbn(sbn)
.reportLines(billingReportLines)
.usageDateFrom(min)
@@ -127,23 +134,21 @@ public class BillingServiceImpl implements BillingService {
}
}
- public List<BillingReportLine> getExploratoryBillingData(String exploratoryId, List<String> resources) {
- List<String> dlabIds = null;
- try {
- dlabIds = Stream.concat(
- BillingUtils.getExploratoryIds(exploratoryId).stream(),
- resources
- .stream()
- .map(BillingUtils::getComputationalIds)
- .flatMap(Collection::stream)
- )
- .collect(Collectors.toList());
-
- return billingDAO.findBillingData(dlabIds);
- } catch (Exception e) {
- log.error("Cannot retrieve billing information for {} {}", dlabIds, e.getMessage());
- return Collections.emptyList();
- }
+ public BillingReport getExploratoryBillingData(String project, String endpoint, String exploratoryName, List<String> compNames) {
+ List<String> resourceNames = new ArrayList<>(compNames);
+ resourceNames.add(exploratoryName);
+ List<BillingReportLine> billingData = billingDAO.findBillingData(project, endpoint, resourceNames)
+ .stream()
+ .peek(bd -> bd.setCost(BigDecimal.valueOf(bd.getCost()).setScale(2, BigDecimal.ROUND_HALF_UP).doubleValue()))
+ .collect(Collectors.toList());
+ final double sum = billingData.stream().mapToDouble(BillingReportLine::getCost).sum();
+ final String currency = billingData.stream().map(BillingReportLine::getCurrency).distinct().count() == 1 ? billingData.get(0).getCurrency() : null;
+ return BillingReport.builder()
+ .name(exploratoryName)
+ .reportLines(billingData)
+ .totalCost(new BigDecimal(sum).setScale(2, BigDecimal.ROUND_HALF_UP).doubleValue())
+ .currency(currency)
+ .build();
}
public void updateRemoteBillingData(UserInfo userInfo) {
@@ -185,7 +190,7 @@ public class BillingServiceImpl implements BillingService {
final Stream<BillingReportLine> billableUserInstances = exploratoryService.findAll(projects)
.stream()
.filter(userInstance -> Objects.nonNull(userInstance.getExploratoryId()))
- .flatMap(ui -> BillingUtils.exploratoryBillingDataStream(ui, configuration.getMaxSparkInstanceCount(), sbn));
+ .flatMap(ui -> BillingUtils.exploratoryBillingDataStream(ui, configuration.getMaxSparkInstanceCount()));
final Stream<BillingReportLine> customImages = projects
.stream()
.map(p -> imageExploratoryDao.getImagesForProject(p.getName()))
@@ -203,8 +208,7 @@ public class BillingServiceImpl implements BillingService {
private Stream<BillingReportLine> projectEdges(String serviceBaseName, String projectName, List<ProjectEndpointDTO> endpoints) {
return endpoints
.stream()
- .flatMap(endpoint -> BillingUtils.edgeBillingDataStream(projectName, serviceBaseName, endpoint.getName(),
- endpoint.getStatus().toString()));
+ .flatMap(endpoint -> BillingUtils.edgeBillingDataStream(projectName, serviceBaseName, endpoint.getName()));
}
private void updateBillingData(UserInfo userInfo, EndpointDTO endpointDTO, List<BillingData> billingData) {
@@ -214,9 +218,8 @@ public class BillingServiceImpl implements BillingService {
final Stream<BillingReportLine> billingReportLineStream = billingData
.stream()
- .filter(bd -> billableResources.containsKey(bd.getTag()))
.peek(bd -> bd.setApplication(endpointName))
- .map(bd -> toBillingReport(bd, billableResources.get(bd.getTag())));
+ .map(bd -> toBillingReport(bd, getOrDefault(billableResources, bd.getTag())));
if (cloudProvider == CloudProvider.GCP) {
final Map<String, List<BillingReportLine>> gcpBillingData = billingReportLineStream
@@ -229,6 +232,10 @@ public class BillingServiceImpl implements BillingService {
}
}
+ private BillingReportLine getOrDefault(Map<String, BillingReportLine> billableResources, String tag) {
+ return billableResources.getOrDefault(tag, BillingReportLine.builder().dlabId(tag).build());
+ }
+
private void updateGcpBillingData(String endpointName, Map<String, List<BillingReportLine>> billingData) {
billingData.forEach((usageDate, billingReportLines) -> {
billingDAO.deleteByUsageDate(endpointName, usageDate);
@@ -267,6 +274,27 @@ public class BillingServiceImpl implements BillingService {
.toString();
}
+ private void appendStatuses(BillingReportLine br) {
+ BillingResourceType resourceType = br.getResourceType();
+ if (BillingResourceType.EDGE == resourceType) {
+ projectService.get(br.getProject()).getEndpoints()
+ .stream()
+ .filter(e -> e.getName().equals(br.getResourceName()))
+ .findAny()
+ .ifPresent(e -> br.setStatus(e.getStatus()));
+ } else if (BillingResourceType.EXPLORATORY == resourceType) {
+ exploratoryService.getUserInstance(br.getUser(), br.getProject(), br.getResourceName())
+ .ifPresent(ui -> br.setStatus(UserInstanceStatus.of(ui.getStatus())));
+ } else if (BillingResourceType.COMPUTATIONAL == resourceType) {
+ exploratoryService.getUserInstance(br.getUser(), br.getProject(), br.getExploratoryName(), true)
+ .flatMap(ui -> ui.getResources()
+ .stream()
+ .filter(cr -> cr.getComputationalName().equals(br.getResourceName()))
+ .findAny())
+ .ifPresent(cr -> br.setStatus(UserInstanceStatus.of(cr.getStatus())));
+ }
+ }
+
private boolean isFullReport(UserInfo userInfo) {
return UserRoles.checkAccess(userInfo, RoleType.PAGE, "/api/infrastructure_provision/billing",
userInfo.getRoles());
@@ -285,6 +313,7 @@ public class BillingServiceImpl implements BillingService {
.currency(billingData.getCurrency())
.product(billingData.getProduct())
.project(billingReportLine.getProject())
+ .endpoint(billingReportLine.getEndpoint())
.usageDateFrom(billingData.getUsageDateFrom())
.usageDateTo(billingData.getUsageDateTo())
.usageDate(billingData.getUsageDate())
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/ExploratoryServiceImpl.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/ExploratoryServiceImpl.java
index 6a2a615..9f6be91 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/ExploratoryServiceImpl.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/ExploratoryServiceImpl.java
@@ -189,7 +189,17 @@ public class ExploratoryServiceImpl implements ExploratoryService {
try {
return Optional.of(exploratoryDAO.fetchExploratoryFields(user, project, exploratoryName));
} catch (DlabException e) {
- log.warn("User instance with exploratory name {} for user {} not found.", exploratoryName, user);
+ log.warn("User instance with exploratory {}, project {} for user {} not found.", exploratoryName, project, user);
+ }
+ return Optional.empty();
+ }
+
+ @Override
+ public Optional<UserInstanceDTO> getUserInstance(String user, String project, String exploratoryName, boolean includeCompResources) {
+ try {
+ return Optional.of(exploratoryDAO.fetchExploratoryFields(user, project, exploratoryName, includeCompResources));
+ } catch (DlabException e) {
+ log.warn("User instance with exploratory {}, project {} for user {} not found.", exploratoryName, project, user);
}
return Optional.empty();
}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/InfrastructureInfoServiceImpl.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/InfrastructureInfoServiceImpl.java
index 88f4b9a..b1eac51 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/InfrastructureInfoServiceImpl.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/InfrastructureInfoServiceImpl.java
@@ -23,7 +23,7 @@ import com.epam.dlab.auth.UserInfo;
import com.epam.dlab.backendapi.conf.SelfServiceApplicationConfiguration;
import com.epam.dlab.backendapi.dao.BillingDAO;
import com.epam.dlab.backendapi.dao.ExploratoryDAO;
-import com.epam.dlab.backendapi.domain.BillingReportLine;
+import com.epam.dlab.backendapi.domain.BillingReport;
import com.epam.dlab.backendapi.domain.EndpointDTO;
import com.epam.dlab.backendapi.domain.ProjectEndpointDTO;
import com.epam.dlab.backendapi.resources.dto.HealthStatusEnum;
@@ -45,7 +45,6 @@ import com.jcabi.manifests.Manifests;
import lombok.extern.slf4j.Slf4j;
import org.bson.Document;
-import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -95,24 +94,24 @@ public class InfrastructureInfoServiceImpl implements InfrastructureInfoService
.anyMatch(endpoint1 -> endpoint1.getName().equals(endpoint.getName())))
.collect(Collectors.toList());
- List<BillingReportLine> billingData = e.getValue()
+ List<BillingReport> billingData = e.getValue()
.stream()
.map(exp ->
- billingService.getExploratoryBillingData(exp.getString("exploratory_id"),
+ billingService.getExploratoryBillingData(exp.getString("project"), exp.getString("endpoint"),
+ exp.getString("exploratory_name"),
Optional.ofNullable(exp.get("computational_resources")).map(cr -> (List<Document>) cr).get()
.stream()
- .map(cr -> cr.getString("computational_id"))
+ .map(cr -> cr.getString("computational_name"))
.collect(Collectors.toList()))
)
- .flatMap(Collection::stream)
.collect(Collectors.toList());
final Map<String, Map<String, String>> projectEdges =
endpoints
.stream()
.collect(Collectors.toMap(ProjectEndpointDTO::getName, this::getSharedInfo));
- return new ProjectInfrastructureInfo(e.getKey(),
- billingDAO.getBillingProjectQuoteUsed(e.getKey()), projectEdges, e.getValue(), billingData, endpointResult);
+ return new ProjectInfrastructureInfo(e.getKey(), billingDAO.getBillingProjectQuoteUsed(e.getKey()),
+ projectEdges, e.getValue(), billingData, endpointResult);
})
.collect(Collectors.toList());
} catch (Exception e) {
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java
index 16b828c..da59725 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java
@@ -63,21 +63,19 @@ public class BillingUtils {
private static final String IMAGE_STANDARD_FORMAT1 = "%s-%s-%s-%s-notebook-image";
private static final String IMAGE_CUSTOM_FORMAT = "%s-%s-%s-%s-%s";
- private static final String VOLUME_PRIMARY = "Volume primary";
- private static final String VOLUME_SECONDARY = "Volume secondary";
private static final String SHARED_RESOURCE = "Shared resource";
private static final String IMAGE_NAME = "Image";
private static final String DATAENGINE_NAME_FORMAT = "%d x %s";
private static final String DATAENGINE_SERVICE_NAME_FORMAT = "Master: %s%sSlave: %d x %s";
- public static Stream<BillingReportLine> edgeBillingDataStream(String project, String sbn, String endpoint, String status) {
+ public static Stream<BillingReportLine> edgeBillingDataStream(String project, String sbn, String endpoint) {
final String userEdgeId = String.format(EDGE_FORMAT, sbn, project.toLowerCase(), endpoint);
final String edgeVolumeId = String.format(EDGE_VOLUME_FORMAT, sbn, project.toLowerCase(), endpoint);
final String endpointBucketId = String.format(PROJECT_ENDPOINT_BUCKET_FORMAT, sbn, project.toLowerCase(), endpoint);
return Stream.concat(Stream.of(
- BillingReportLine.builder().resourceName("EDGE node").user(SHARED_RESOURCE).project(project).dlabId(userEdgeId).resourceType(EDGE).build(),
+ BillingReportLine.builder().resourceName(endpoint).user(SHARED_RESOURCE).project(project).dlabId(userEdgeId).resourceType(EDGE).build(),
BillingReportLine.builder().resourceName("EDGE volume").user(SHARED_RESOURCE).project(project).dlabId(edgeVolumeId).resourceType(VOLUME).build(),
BillingReportLine.builder().resourceName("Project endpoint shared bucket").user(SHARED_RESOURCE).project(project).dlabId(endpointBucketId).resourceType(BUCKET).build()
),
@@ -102,26 +100,29 @@ public class BillingUtils {
);
}
- public static Stream<BillingReportLine> exploratoryBillingDataStream(UserInstanceDTO userInstance, Integer maxSparkInstanceCount, String sbn) {
+ public static Stream<BillingReportLine> exploratoryBillingDataStream(UserInstanceDTO userInstance, Integer maxSparkInstanceCount) {
final Stream<BillingReportLine> computationalStream = userInstance.getResources()
.stream()
.filter(cr -> cr.getComputationalId() != null)
.flatMap(cr -> Stream.concat(Stream.of(
- withUserProject(userInstance).dlabId(cr.getComputationalId()).resourceName(cr.getComputationalName()).resourceType(COMPUTATIONAL).shape(getComputationalShape(cr)).build(),
- withUserProject(userInstance).resourceName(cr.getComputationalName() + ":" + VOLUME_PRIMARY).dlabId(String.format(VOLUME_PRIMARY_COMPUTATIONAL_FORMAT, cr.getComputationalId(), "m"))
+ withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).dlabId(cr.getComputationalId()).resourceType(COMPUTATIONAL).shape(getComputationalShape(cr))
+ .exploratoryName(userInstance.getExploratoryName()).build(),
+ withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).dlabId(String.format(VOLUME_PRIMARY_FORMAT, cr.getComputationalId())).resourceType(VOLUME).build(),
+ withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).dlabId(String.format(VOLUME_PRIMARY_COMPUTATIONAL_FORMAT, cr.getComputationalId(), "m"))
.resourceType(VOLUME).build(),
- withUserProject(userInstance).resourceName(cr.getComputationalName() + ":" + VOLUME_SECONDARY).dlabId(String.format(VOLUME_SECONDARY_COMPUTATIONAL_FORMAT, cr.getComputationalId(), "m"))
+ withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).dlabId(String.format(VOLUME_SECONDARY_COMPUTATIONAL_FORMAT, cr.getComputationalId(), "m"))
.resourceType(VOLUME).build()
),
getSlaveVolumes(userInstance, cr, maxSparkInstanceCount)
));
+ final String exploratoryName = userInstance.getExploratoryName();
final String exploratoryId = userInstance.getExploratoryId();
final String primaryVolumeId = String.format(VOLUME_PRIMARY_FORMAT, exploratoryId);
final String secondaryVolumeId = String.format(VOLUME_SECONDARY_FORMAT, exploratoryId);
final Stream<BillingReportLine> exploratoryStream = Stream.of(
- withUserProject(userInstance).resourceName(userInstance.getExploratoryName()).dlabId(exploratoryId).resourceType(EXPLORATORY).shape(userInstance.getShape()).build(),
- withUserProject(userInstance).resourceName(VOLUME_PRIMARY).dlabId(primaryVolumeId).resourceType(VOLUME).build(),
- withUserProject(userInstance).resourceName(VOLUME_SECONDARY).dlabId(secondaryVolumeId).resourceType(VOLUME).build());
+ withUserProjectEndpoint(userInstance).resourceName(exploratoryName).dlabId(exploratoryId).resourceType(EXPLORATORY).shape(userInstance.getShape()).build(),
+ withUserProjectEndpoint(userInstance).resourceName(exploratoryName).dlabId(primaryVolumeId).resourceType(VOLUME).build(),
+ withUserProjectEndpoint(userInstance).resourceName(exploratoryName).dlabId(secondaryVolumeId).resourceType(VOLUME).build());
return Stream.concat(computationalStream, exploratoryStream);
}
@@ -136,27 +137,19 @@ public class BillingUtils {
private static Stream<BillingReportLine> getSlaveVolumes(UserInstanceDTO userInstance, UserComputationalResource cr, Integer maxSparkInstanceCount) {
List<BillingReportLine> list = new ArrayList<>();
for (int i = 1; i <= maxSparkInstanceCount; i++) {
- list.add(withUserProject(userInstance).resourceName(cr.getComputationalName() + ":" + VOLUME_PRIMARY).dlabId(String.format(VOLUME_PRIMARY_COMPUTATIONAL_FORMAT, cr.getComputationalId(), "s" + i))
+ list.add(withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).dlabId(String.format(VOLUME_PRIMARY_COMPUTATIONAL_FORMAT, cr.getComputationalId(), "s" + i))
.resourceType(VOLUME).build());
- list.add(withUserProject(userInstance).resourceName(cr.getComputationalName() + ":" + VOLUME_PRIMARY).dlabId(String.format(VOLUME_SECONDARY_COMPUTATIONAL_FORMAT, cr.getComputationalId(), "s" + i))
+ list.add(withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).dlabId(String.format(VOLUME_SECONDARY_COMPUTATIONAL_FORMAT, cr.getComputationalId(), "s" + i))
.resourceType(VOLUME).build());
}
return list.stream();
}
- private static BillingReportLine.BillingReportLineBuilder withUserProject(UserInstanceDTO userInstance) {
- return BillingReportLine.builder().user(userInstance.getUser()).project(userInstance.getProject());
+ private static BillingReportLine.BillingReportLineBuilder withUserProjectEndpoint(UserInstanceDTO userInstance) {
+ return BillingReportLine.builder().user(userInstance.getUser()).project(userInstance.getProject()).endpoint(userInstance.getEndpoint());
}
- public static List<String> getComputationalIds(String computationalId) {
- return Arrays.asList(computationalId, String.format(VOLUME_PRIMARY_FORMAT, computationalId));
- }
-
- public static List<String> getExploratoryIds(String exploratoryId) {
- return Arrays.asList(exploratoryId, String.format(VOLUME_PRIMARY_FORMAT, exploratoryId), String.format(VOLUME_SECONDARY_FORMAT, exploratoryId));
- }
-
- private static String getComputationalShape(UserComputationalResource resource) {
+ public static String getComputationalShape(UserComputationalResource resource) {
return DataEngineType.fromDockerImageName(resource.getImageName()) == DataEngineType.SPARK_STANDALONE ?
String.format(DATAENGINE_NAME_FORMAT, resource.getDataengineInstanceCount(), resource.getDataengineShape()) :
String.format(DATAENGINE_SERVICE_NAME_FORMAT, resource.getMasterNodeShape(), System.lineSeparator(), null, null);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org