You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by of...@apache.org on 2020/03/03 11:39:16 UTC
[incubator-dlab] 01/08: [DLAB-1571] Convey billing for remote gcp
endpoint
This is an automated email from the ASF dual-hosted git repository.
ofuks pushed a commit to branch DLAB-1571
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 72a2f1c2d1fc951d65696272651ccf17f68315c9
Author: Oleh Fuks <ol...@gmail.com>
AuthorDate: Tue Feb 25 17:31:21 2020 +0200
[DLAB-1571] Convey billing for remote gcp endpoint
---
services/billing-gcp/pom.xml | 5 +
.../gcp/conf/BillingApplicationConfiguration.java | 2 +-
.../BillingController.java} | 23 ++-
.../com/epam/dlab/billing/gcp/dao/BillingDAO.java | 3 +
.../billing/gcp/dao/impl/BigQueryBillingDAO.java | 58 ++++++-
.../epam/dlab/billing/gcp/model/BillingData.java | 21 +--
.../dlab/billing/gcp/model/GcpBillingData.java | 7 +
.../billing/gcp/repository/BillingRepository.java | 3 +-
.../billing/gcp/scheduler/BillingScheduler.java | 2 +-
.../dlab/billing/gcp/service/BillingService.java | 2 +
.../billing/gcp/service/BillingServiceImpl.java | 45 +++--
.../epam/dlab/billing/gcp/util/BillingUtils.java | 33 ++--
.../billing-gcp/src/main/resources/application.yml | 12 +-
.../com/epam/dlab/dto/billing/BillingData.java} | 22 ++-
.../dlab/dto/billing/BillingResourceType.java} | 14 +-
.../com/epam/dlab/rest/client/RESTService.java | 14 +-
.../epam/dlab/backendapi/dao/BaseBillingDAO.java | 16 +-
.../dlab/backendapi/dao/azure/AzureBillingDAO.java | 4 +-
.../dlab/backendapi/domain/BillingReportDTO.java} | 47 ++----
.../backendapi/modules/CloudProviderModule.java | 3 +
.../dlab/backendapi/resources/BillingResource.java | 16 +-
.../backendapi/resources/dto/BillingFilter.java | 37 ++--
.../dlab/backendapi/service/BillingService.java | 2 +-
.../backendapi/service/BillingServiceNew.java} | 10 +-
.../backendapi/service/ExploratoryService.java | 2 +
.../service/impl/BillingServiceImplNew.java | 187 +++++++++++++++++++++
.../service/impl/ExploratoryServiceImpl.java | 5 +
.../epam/dlab/backendapi/util/BillingUtils.java | 95 +++++++++++
.../InfrastructureTemplateServiceBaseTest.java | 10 +-
29 files changed, 549 insertions(+), 151 deletions(-)
diff --git a/services/billing-gcp/pom.xml b/services/billing-gcp/pom.xml
index 114b25d..6ae8114 100644
--- a/services/billing-gcp/pom.xml
+++ b/services/billing-gcp/pom.xml
@@ -73,6 +73,11 @@
<version>${org.mockito.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>com.epam.dlab</groupId>
+ <artifactId>dlab-model</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
</dependencies>
<build>
diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/conf/BillingApplicationConfiguration.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/conf/BillingApplicationConfiguration.java
index f565c6f..5cb1ec5 100644
--- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/conf/BillingApplicationConfiguration.java
+++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/conf/BillingApplicationConfiguration.java
@@ -28,7 +28,7 @@ import org.springframework.context.annotation.Configuration;
public class BillingApplicationConfiguration {
- @Bean
+// @Bean
public BigQuery bigQueryService() {
return BigQueryOptions.getDefaultInstance().getService();
}
diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/BillingDAO.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/controller/BillingController.java
similarity index 53%
copy from services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/BillingDAO.java
copy to services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/controller/BillingController.java
index 7e6b0b7..dbcfa10 100644
--- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/BillingDAO.java
+++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/controller/BillingController.java
@@ -17,13 +17,28 @@
* under the License.
*/
-package com.epam.dlab.billing.gcp.dao;
+package com.epam.dlab.billing.gcp.controller;
-import com.epam.dlab.billing.gcp.model.GcpBillingData;
+import com.epam.dlab.billing.gcp.dao.BillingDAO;
+import com.epam.dlab.dto.billing.BillingData;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RestController;
import java.util.List;
-public interface BillingDAO {
+@RestController
+public class BillingController {
- List<GcpBillingData> getBillingData() throws InterruptedException;
+ private final BillingDAO billingDAO;
+
+ public BillingController(BillingDAO billingDAO) {
+ this.billingDAO = billingDAO;
+ }
+
+ @GetMapping("/report")
+ public ResponseEntity<List<BillingData>> getBilling() {
+ return new ResponseEntity<>(billingDAO.getBillingReport(), HttpStatus.OK);
+ }
}
diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/BillingDAO.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/BillingDAO.java
index 7e6b0b7..55c2c2d 100644
--- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/BillingDAO.java
+++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/BillingDAO.java
@@ -20,10 +20,13 @@
package com.epam.dlab.billing.gcp.dao;
import com.epam.dlab.billing.gcp.model.GcpBillingData;
+import com.epam.dlab.dto.billing.BillingData;
import java.util.List;
public interface BillingDAO {
List<GcpBillingData> getBillingData() throws InterruptedException;
+
+ List<BillingData> getBillingReport();
}
diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/impl/BigQueryBillingDAO.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/impl/BigQueryBillingDAO.java
index b0ece02..990399a 100644
--- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/impl/BigQueryBillingDAO.java
+++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/impl/BigQueryBillingDAO.java
@@ -24,27 +24,43 @@ import com.epam.dlab.billing.gcp.dao.BillingDAO;
import com.epam.dlab.billing.gcp.model.BillingHistory;
import com.epam.dlab.billing.gcp.model.GcpBillingData;
import com.epam.dlab.billing.gcp.repository.BillingHistoryRepository;
-import com.google.cloud.bigquery.*;
+import com.epam.dlab.dto.billing.BillingData;
+import com.google.cloud.bigquery.BigQuery;
+import com.google.cloud.bigquery.FieldValueList;
+import com.google.cloud.bigquery.QueryJobConfiguration;
+import com.google.cloud.bigquery.QueryParameterValue;
+import com.google.cloud.bigquery.Table;
+import com.google.cloud.bigquery.TableInfo;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.data.mongodb.core.MongoTemplate;
+import org.springframework.data.mongodb.core.aggregation.Aggregation;
+import org.springframework.data.mongodb.core.aggregation.GroupOperation;
import org.springframework.stereotype.Component;
+import java.math.BigDecimal;
import java.time.Instant;
import java.time.LocalDate;
import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
+import static org.springframework.data.mongodb.core.aggregation.Aggregation.group;
+import static org.springframework.data.mongodb.core.aggregation.Aggregation.newAggregation;
+
@Component
@Slf4j
public class BigQueryBillingDAO implements BillingDAO {
+ private static final String DATE_FORMAT = "yyyy-MM-dd";
private static final String SBN_PARAM = "sbn";
private static final String DATASET_PARAM = "dataset";
private final BillingHistoryRepository billingHistoryRepo;
+ private final MongoTemplate mongoTemplate;
private final String sbn;
private static final String GET_BILLING_DATA_QUERY = "SELECT b.sku.description usageType," +
@@ -55,15 +71,17 @@ public class BigQueryBillingDAO implements BillingDAO {
"CROSS JOIN UNNEST(b.labels) as label\n" +
"where label.key = 'name' and cost != 0 and label.value like @sbn\n" +
"group by usageType, usage_date_from, usage_date_to, product, value, currency";
- private final BigQuery service;
+ private BigQuery service = null;
private final String dataset;
@Autowired
- public BigQueryBillingDAO(DlabConfiguration conf, BigQuery service, BillingHistoryRepository billingHistoryRepo) {
+ public BigQueryBillingDAO(DlabConfiguration conf, BillingHistoryRepository billingHistoryRepo,
+ MongoTemplate mongoTemplate) {
dataset = conf.getBigQueryDataset();
sbn = conf.getSbn();
- this.service = service;
+// this.service = null;
this.billingHistoryRepo = billingHistoryRepo;
+ this.mongoTemplate = mongoTemplate;
}
@Override
@@ -82,6 +100,19 @@ public class BigQueryBillingDAO implements BillingDAO {
.collect(Collectors.toList());
}
+ @Override
+ public List<BillingData> getBillingReport() {
+ GroupOperation groupOperation = group("product", "currency", "usageType", "dlabId")
+ .min("from").as("from")
+ .max("to").as("to")
+ .sum("cost").as("cost");
+ Aggregation aggregation = newAggregation(groupOperation);
+
+ return mongoTemplate.aggregate(aggregation, "billing", GcpBillingData.class).getMappedResults().stream()
+ .map(this::toBillingReportDTO)
+ .collect(Collectors.toList());
+ }
+
private Stream<? extends GcpBillingData> bigQueryResultSetStream(Table table) {
try {
final String tableName = table.getTableId().getTable();
@@ -102,19 +133,32 @@ public class BigQueryBillingDAO implements BillingDAO {
}
private GcpBillingData toBillingData(FieldValueList fields) {
-
return GcpBillingData.builder()
.usageDateFrom(toLocalDate(fields, "usage_date_from"))
.usageDateTo(toLocalDate(fields, "usage_date_to"))
- .cost(fields.get("cost").getNumericValue())
+ .cost(fields.get("cost").getNumericValue().setScale(3, BigDecimal.ROUND_HALF_UP))
.product(fields.get("product").getStringValue())
.usageType(fields.get("usageType").getStringValue())
.currency(fields.get("currency").getStringValue())
- .tag(fields.get("value").getStringValue()).build();
+ .tag(fields.get("value").getStringValue())
+ .usageDate(toLocalDate(fields, "usage_date_from").format((DateTimeFormatter.ofPattern(DATE_FORMAT))))
+ .build();
}
private LocalDate toLocalDate(FieldValueList fieldValues, String timestampFieldName) {
return LocalDate.from(Instant.ofEpochMilli(fieldValues.get(timestampFieldName).getTimestampValue() / 1000)
.atZone(ZoneId.systemDefault()));
}
+
+ private BillingData toBillingReportDTO(GcpBillingData billingData) {
+ return BillingData.builder()
+ .usageDateFrom(billingData.getUsageDateFrom())
+ .usageDateTo(billingData.getUsageDateTo())
+ .product(billingData.getProduct())
+ .usageType(billingData.getUsageType())
+ .cost(billingData.getCost().setScale(3, BigDecimal.ROUND_HALF_UP).doubleValue())
+ .currency(billingData.getCurrency())
+ .tag(billingData.getTag())
+ .build();
+ }
}
diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/model/BillingData.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/model/BillingData.java
index 32a98ed..773d381 100644
--- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/model/BillingData.java
+++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/model/BillingData.java
@@ -19,23 +19,24 @@
package com.epam.dlab.billing.gcp.model;
+import com.epam.dlab.dto.billing.BillingResourceType;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Builder;
import lombok.Data;
import org.springframework.data.annotation.Id;
-import org.springframework.data.mongodb.core.mapping.Document;
import org.springframework.data.mongodb.core.mapping.Field;
import java.time.LocalDate;
@Data
@Builder
-@Document(collection = "billing")
+//@Document(collection = "billing")
+@JsonIgnoreProperties(ignoreUnknown = true)
public class BillingData {
@Id
private String id;
private String user;
@Field("resource_name")
- private String displayName;
private String resourceName;
@Field("from")
private LocalDate usageDateFrom;
@@ -54,17 +55,5 @@ public class BillingData {
@Field("dlab_id")
private String dlabId;
@Field("dlab_resource_type")
- private ResourceType resourceType;
-
-
- public enum ResourceType {
- EDGE,
- SSN,
- SHARED_BUCKET,
- SSN_BUCKET,
- EDGE_BUCKET,
- VOLUME,
- EXPLORATORY,
- COMPUTATIONAL
- }
+ private BillingResourceType resourceType;
}
diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/model/GcpBillingData.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/model/GcpBillingData.java
index a11dcce..d688198 100644
--- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/model/GcpBillingData.java
+++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/model/GcpBillingData.java
@@ -21,18 +21,25 @@ package com.epam.dlab.billing.gcp.model;
import lombok.Builder;
import lombok.Data;
+import org.springframework.data.mongodb.core.mapping.Document;
+import org.springframework.data.mongodb.core.mapping.Field;
import java.math.BigDecimal;
import java.time.LocalDate;
@Data
@Builder
+@Document(collection = "billing")
public class GcpBillingData {
+ @Field("from")
private final LocalDate usageDateFrom;
+ @Field("to")
private final LocalDate usageDateTo;
private final String product;
private final String usageType;
private final BigDecimal cost;
private final String currency;
+ @Field("dlabId")
private final String tag;
+ private final String usageDate;
}
diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/repository/BillingRepository.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/repository/BillingRepository.java
index 9dbfe98..77c362a 100644
--- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/repository/BillingRepository.java
+++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/repository/BillingRepository.java
@@ -20,8 +20,9 @@
package com.epam.dlab.billing.gcp.repository;
import com.epam.dlab.billing.gcp.model.BillingData;
+import com.epam.dlab.billing.gcp.model.GcpBillingData;
import org.springframework.data.mongodb.repository.MongoRepository;
-public interface BillingRepository extends MongoRepository<BillingData, String> {
+public interface BillingRepository extends MongoRepository<GcpBillingData, String> {
void deleteByUsageDateRegex(String usageDateRegex);
}
diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/scheduler/BillingScheduler.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/scheduler/BillingScheduler.java
index 9724d43..ceaf8ce 100644
--- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/scheduler/BillingScheduler.java
+++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/scheduler/BillingScheduler.java
@@ -37,6 +37,6 @@ public class BillingScheduler {
@Scheduled(cron = "${dlab.cron}")
public void getBillingReport() {
- billingService.updateBillingData();
+// billingService.updateBillingData();
}
}
diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/service/BillingService.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/service/BillingService.java
index 71015aa..dd69f7a 100644
--- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/service/BillingService.java
+++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/service/BillingService.java
@@ -22,4 +22,6 @@ package com.epam.dlab.billing.gcp.service;
public interface BillingService {
void updateBillingData();
+
+ void updateBillingData2();
}
diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/service/BillingServiceImpl.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/service/BillingServiceImpl.java
index 2a20206..3ec4eac 100644
--- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/service/BillingServiceImpl.java
+++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/service/BillingServiceImpl.java
@@ -97,22 +97,39 @@ public class BillingServiceImpl implements BillingService {
.filter(bd -> bd.getDlabId() != null)
.collect(Collectors.toMap(BillingData::getDlabId, b -> b));
log.info("Billable resources are: {}", billableResources);
- final Map<String, List<BillingData>> billingDataMap = billingDAO.getBillingData()
+// final Map<String, List<BillingData>> billingDataMap = billingDAO.getBillingData()
+// .stream()
+// .map(bd -> toBillingData(bd, getOrDefault(billableResources, bd.getTag())))
+// .collect(Collectors.groupingBy(bd -> bd.getUsageDate().substring(0,
+// USAGE_DATE_FORMAT.length())));
+
+// billingDataMap.forEach((usageDate, billingDataList) -> {
+// log.info("Updating billing information for month {}", usageDate);
+// billingRepository.deleteByUsageDateRegex("^" + usageDate);
+// billingRepository.insert(billingDataList);
+// updateExploratoryCost(billingDataList);
+// });
+
+ log.info("Finished updating billing data");
+
+
+ } catch (Exception e) {
+ log.error("Can not update billing due to: {}", e.getMessage(), e);
+ }
+ }
+
+ @Override
+ public void updateBillingData2() {
+ try {
+ Map<String, List<GcpBillingData>> collect = billingDAO.getBillingData()
.stream()
- .map(bd -> toBillingData(bd, getOrDefault(billableResources, bd.getTag())))
- .collect(Collectors.groupingBy(bd -> bd.getUsageDate().substring(0,
- USAGE_DATE_FORMAT.length())));
+ .collect(Collectors.groupingBy(bd -> bd.getUsageDate().substring(0, USAGE_DATE_FORMAT.length())));
- billingDataMap.forEach((usageDate, billingDataList) -> {
+ collect.forEach((usageDate, billingDataList) -> {
log.info("Updating billing information for month {}", usageDate);
billingRepository.deleteByUsageDateRegex("^" + usageDate);
billingRepository.insert(billingDataList);
- updateExploratoryCost(billingDataList);
});
-
- log.info("Finished updating billing data");
-
-
} catch (Exception e) {
log.error("Can not update billing due to: {}", e.getMessage(), e);
}
@@ -164,23 +181,17 @@ public class BillingServiceImpl implements BillingService {
}
- private BillingData toBillingData(GcpBillingData bd, BillingData billableResource) {
+ private BillingData toBillingData(GcpBillingData bd) {
return BillingData.builder()
- .displayName(billableResource.getDisplayName())
.cost(bd.getCost().setScale(2, BigDecimal.ROUND_HALF_UP).doubleValue())
.currency(bd.getCurrency())
.product(bd.getProduct())
- .project(billableResource.getProject())
.usageDateTo(bd.getUsageDateTo())
.usageDateFrom(bd.getUsageDateFrom())
.usageDate(bd.getUsageDateFrom().format((DateTimeFormatter.ofPattern(DATE_FORMAT))))
.usageType(bd.getUsageType())
- .user(billableResource.getUser())
- .exploratoryName(billableResource.getExploratoryName())
- .computationalName(billableResource.getComputationalName())
.dlabId(bd.getTag())
- .resourceType(billableResource.getResourceType())
.build();
}
}
diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/util/BillingUtils.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/util/BillingUtils.java
index 2f26f10..6a6341e 100644
--- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/util/BillingUtils.java
+++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/util/BillingUtils.java
@@ -21,6 +21,7 @@ package com.epam.dlab.billing.gcp.util;
import com.epam.dlab.billing.gcp.documents.UserInstance;
import com.epam.dlab.billing.gcp.model.BillingData;
+import com.epam.dlab.dto.billing.BillingResourceType;
import java.util.stream.Stream;
@@ -40,9 +41,9 @@ public class BillingUtils {
final String edgeVolumeId = String.format(EDGE_VOLUME_FORMAT, sbn, project.toLowerCase(), endpoint);
final String edgeBucketId = String.format(EDGE_BUCKET_FORMAT, sbn, project.toLowerCase());
return Stream.of(
- BillingData.builder().displayName("EDGE node").user(SHARED_RESOURCE).project(project).dlabId(userEdgeId).resourceType(BillingData.ResourceType.EDGE).build(),
- BillingData.builder().displayName("EDGE volume").user(SHARED_RESOURCE).project(project).dlabId(edgeVolumeId).resourceType(BillingData.ResourceType.VOLUME).build(),
- BillingData.builder().displayName("EDGE bucket").user(SHARED_RESOURCE).project(project).dlabId(edgeBucketId).resourceType(BillingData.ResourceType.EDGE_BUCKET).build()
+ BillingData.builder().resourceName("EDGE node").user(SHARED_RESOURCE).project(project).dlabId(userEdgeId).resourceType(BillingResourceType.EDGE).build(),
+ BillingData.builder().resourceName("EDGE volume").user(SHARED_RESOURCE).project(project).dlabId(edgeVolumeId).resourceType(BillingResourceType.VOLUME).build(),
+ BillingData.builder().resourceName("EDGE bucket").user(SHARED_RESOURCE).project(project).dlabId(edgeBucketId).resourceType(BillingResourceType.EDGE_BUCKET).build()
);
}
@@ -50,12 +51,12 @@ public class BillingUtils {
final String ssnId = sbn + "-ssn";
final String bucketName = sbn.replaceAll("_", "-");
return Stream.of(
- BillingData.builder().user(SHARED_RESOURCE).displayName("SSN").dlabId(ssnId).resourceType(BillingData.ResourceType.SSN).build(),
- BillingData.builder().user(SHARED_RESOURCE).displayName("SSN Volume").dlabId(String.format(VOLUME_PRIMARY_FORMAT, ssnId)).resourceType(BillingData.ResourceType.VOLUME).build(),
- BillingData.builder().user(SHARED_RESOURCE).displayName("SSN bucket").dlabId(bucketName + "-ssn" +
- "-bucket").resourceType(BillingData.ResourceType.SSN_BUCKET).build(),
- BillingData.builder().user(SHARED_RESOURCE).displayName("Collaboration bucket").dlabId(bucketName +
- "-shared-bucket").resourceType(BillingData.ResourceType.SHARED_BUCKET).build()
+ BillingData.builder().user(SHARED_RESOURCE).resourceName("SSN").dlabId(ssnId).resourceType(BillingResourceType.SSN).build(),
+ BillingData.builder().user(SHARED_RESOURCE).resourceName("SSN Volume").dlabId(String.format(VOLUME_PRIMARY_FORMAT, ssnId)).resourceType(BillingResourceType.VOLUME).build(),
+ BillingData.builder().user(SHARED_RESOURCE).resourceName("SSN bucket").dlabId(bucketName + "-ssn" +
+ "-bucket").resourceType(BillingResourceType.SSN_BUCKET).build(),
+ BillingData.builder().user(SHARED_RESOURCE).resourceName("Collaboration bucket").dlabId(bucketName +
+ "-shared-bucket").resourceType(BillingResourceType.SHARED_BUCKET).build()
);
}
@@ -64,15 +65,15 @@ public class BillingUtils {
.stream()
.filter(cr -> cr.getComputationalId() != null)
.flatMap(cr -> Stream.of(computationalBillableResource(userInstance, cr),
- withExploratoryName(userInstance).displayName(cr.getComputationalName() + ":" + VOLUME_PRIMARY).dlabId(String.format(VOLUME_PRIMARY_FORMAT, cr.getComputationalId()))
- .resourceType(BillingData.ResourceType.VOLUME).computationalName(cr.getComputationalName()).build()));
+ withExploratoryName(userInstance).resourceName(cr.getComputationalName() + ":" + VOLUME_PRIMARY).dlabId(String.format(VOLUME_PRIMARY_FORMAT, cr.getComputationalId()))
+ .resourceType(BillingResourceType.VOLUME).computationalName(cr.getComputationalName()).build()));
final String exploratoryId = userInstance.getExploratoryId();
final String primaryVolumeId = String.format(VOLUME_PRIMARY_FORMAT, exploratoryId);
final String secondaryVolumeId = String.format(VOLUME_SECONDARY_FORMAT, exploratoryId);
final Stream<BillingData> exploratoryStream = Stream.of(
- withExploratoryName(userInstance).displayName(userInstance.getExploratoryName()).dlabId(exploratoryId).resourceType(BillingData.ResourceType.EXPLORATORY).build(),
- withExploratoryName(userInstance).displayName(VOLUME_PRIMARY).dlabId(primaryVolumeId).resourceType(BillingData.ResourceType.VOLUME).build(),
- withExploratoryName(userInstance).displayName(VOLUME_SECONDARY).dlabId(secondaryVolumeId).resourceType(BillingData.ResourceType.VOLUME).build());
+ withExploratoryName(userInstance).resourceName(userInstance.getExploratoryName()).dlabId(exploratoryId).resourceType(BillingResourceType.EXPLORATORY).build(),
+ withExploratoryName(userInstance).resourceName(VOLUME_PRIMARY).dlabId(primaryVolumeId).resourceType(BillingResourceType.VOLUME).build(),
+ withExploratoryName(userInstance).resourceName(VOLUME_SECONDARY).dlabId(secondaryVolumeId).resourceType(BillingResourceType.VOLUME).build());
return Stream.concat(computationalStream, exploratoryStream);
}
@@ -80,8 +81,8 @@ public class BillingUtils {
UserInstance.ComputationalResource cr) {
return withExploratoryName(userInstance)
.dlabId(cr.getComputationalId())
- .displayName(cr.getComputationalName())
- .resourceType(BillingData.ResourceType.COMPUTATIONAL)
+ .resourceName(cr.getComputationalName())
+ .resourceType(BillingResourceType.COMPUTATIONAL)
.computationalName(cr.getComputationalName())
.project(userInstance.getProject())
.build();
diff --git a/services/billing-gcp/src/main/resources/application.yml b/services/billing-gcp/src/main/resources/application.yml
index f1f3ce6..aae4af9 100644
--- a/services/billing-gcp/src/main/resources/application.yml
+++ b/services/billing-gcp/src/main/resources/application.yml
@@ -3,7 +3,7 @@ spring:
mongodb:
username: admin
password: admin
- database: <MONGO_PASSWORD>
+ database: dlabdb
port: 27017
host: localhost
dlab:
@@ -11,6 +11,16 @@ dlab:
bigQueryDataset: <DATASET_NAME>
cron: 0 * * * * *
+server:
+ port: 8081
+ servlet:
+ contextPath: /api/billing
+
+server.ssl.key-store-type: JKS
+server.ssl.key-store: /Users/ofuks/keys/dlabcert/billing.jks
+server.ssl.key-store-password: KEYSTORE_PASSWORD
+server.ssl.key-alias: billing
+
logging:
file: /var/opt/dlab/log/ssn/billing.log
level:
diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/model/GcpBillingData.java b/services/dlab-model/src/main/java/com/epam/dlab/dto/billing/BillingData.java
similarity index 67%
copy from services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/model/GcpBillingData.java
copy to services/dlab-model/src/main/java/com/epam/dlab/dto/billing/BillingData.java
index a11dcce..27db613 100644
--- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/model/GcpBillingData.java
+++ b/services/dlab-model/src/main/java/com/epam/dlab/dto/billing/BillingData.java
@@ -17,22 +17,26 @@
* under the License.
*/
-package com.epam.dlab.billing.gcp.model;
+package com.epam.dlab.dto.billing;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Builder;
import lombok.Data;
-import java.math.BigDecimal;
import java.time.LocalDate;
@Data
@Builder
-public class GcpBillingData {
- private final LocalDate usageDateFrom;
- private final LocalDate usageDateTo;
- private final String product;
- private final String usageType;
- private final BigDecimal cost;
- private final String currency;
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class BillingData {
private final String tag;
+ @JsonProperty("from")
+ private LocalDate usageDateFrom;
+ @JsonProperty("to")
+ private LocalDate usageDateTo;
+ private String product;
+ private String usageType;
+ private Double cost;
+ private String currency;
}
diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/service/BillingService.java b/services/dlab-model/src/main/java/com/epam/dlab/dto/billing/BillingResourceType.java
similarity index 81%
copy from services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/service/BillingService.java
copy to services/dlab-model/src/main/java/com/epam/dlab/dto/billing/BillingResourceType.java
index 71015aa..692ad84 100644
--- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/service/BillingService.java
+++ b/services/dlab-model/src/main/java/com/epam/dlab/dto/billing/BillingResourceType.java
@@ -17,9 +17,15 @@
* under the License.
*/
-package com.epam.dlab.billing.gcp.service;
+package com.epam.dlab.dto.billing;
-public interface BillingService {
-
- void updateBillingData();
+public enum BillingResourceType {
+ EDGE,
+ SSN,
+ SHARED_BUCKET,
+ SSN_BUCKET,
+ EDGE_BUCKET,
+ VOLUME,
+ EXPLORATORY,
+ COMPUTATIONAL
}
diff --git a/services/dlab-webapp-common/src/main/java/com/epam/dlab/rest/client/RESTService.java b/services/dlab-webapp-common/src/main/java/com/epam/dlab/rest/client/RESTService.java
index ab1d29e..d14ce42 100644
--- a/services/dlab-webapp-common/src/main/java/com/epam/dlab/rest/client/RESTService.java
+++ b/services/dlab-webapp-common/src/main/java/com/epam/dlab/rest/client/RESTService.java
@@ -19,17 +19,15 @@
package com.epam.dlab.rest.client;
-import com.epam.dlab.exceptions.DlabException;
import lombok.extern.slf4j.Slf4j;
-import javax.ws.rs.ProcessingException;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.Entity;
import javax.ws.rs.client.Invocation;
import javax.ws.rs.client.WebTarget;
+import javax.ws.rs.core.GenericType;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MediaType;
-import java.net.ConnectException;
import java.net.URI;
import java.util.Collections;
import java.util.Map;
@@ -71,6 +69,16 @@ public class RESTService {
return builder.get(clazz);
}
+ public <T> T get(String path, GenericType<T> genericType) {
+ return get(path, null, genericType);
+ }
+
+ public <T> T get(String path, String accessToken, GenericType<T> genericType) {
+ Invocation.Builder builder = getBuilder(path, accessToken, Collections.emptyMap());
+ log.debug("REST get secured {} {}", path, accessToken);
+ return builder.get(genericType);
+ }
+
public <T> T post(String path, Object parameter, Class<T> clazz) {
return post(path, null, parameter, clazz);
}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/BaseBillingDAO.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/BaseBillingDAO.java
index 9edfaa8..71223db 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/BaseBillingDAO.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/BaseBillingDAO.java
@@ -122,8 +122,8 @@ public abstract class BaseBillingDAO extends BaseDAO implements BillingDAO {
}
pipeline.add(groupCriteria());
pipeline.add(sortCriteria());
- final Map<String, BaseShape> shapes = getShapes(filter.getShape());
- return prepareReport(filter.getStatuses(), !filter.getShape().isEmpty(),
+ final Map<String, BaseShape> shapes = getShapes(filter.getShapes());
+ return prepareReport(filter.getStatuses(), !filter.getShapes().isEmpty(),
getCollection(BILLING).aggregate(pipeline), shapes, isFullReport);
}
@@ -335,13 +335,13 @@ public abstract class BaseBillingDAO extends BaseDAO implements BillingDAO {
List<Bson> searchCriteria = new ArrayList<>();
- if (filter.getUser() != null && !filter.getUser().isEmpty()) {
- searchCriteria.add(Filters.in(MongoKeyWords.DLAB_USER, filter.getUser()));
+ if (filter.getUsers() != null && !filter.getUsers().isEmpty()) {
+ searchCriteria.add(Filters.in(MongoKeyWords.DLAB_USER, filter.getUsers()));
}
- if (filter.getResourceType() != null && !filter.getResourceType().isEmpty()) {
+ if (filter.getResourceTypes() != null && !filter.getResourceTypes().isEmpty()) {
searchCriteria.add(Filters.in("dlab_resource_type",
- DlabResourceType.getResourceTypeIds(filter.getResourceType())));
+ DlabResourceType.getResourceTypeIds(filter.getResourceTypes())));
}
if (filter.getDlabId() != null && !filter.getDlabId().isEmpty()) {
@@ -474,9 +474,9 @@ public abstract class BaseBillingDAO extends BaseDAO implements BillingDAO {
protected void setUserFilter(UserInfo userInfo, BillingFilter filter, boolean isFullReport) {
if (isFullReport) {
- usersToLowerCase(filter.getUser());
+ usersToLowerCase(filter.getUsers());
} else {
- filter.setUser(Lists.newArrayList(userInfo.getName().toLowerCase()));
+ filter.setUsers(Lists.newArrayList(userInfo.getName().toLowerCase()));
}
}
}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/azure/AzureBillingDAO.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/azure/AzureBillingDAO.java
index 04c5e6d..8eeb52c 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/azure/AzureBillingDAO.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/azure/AzureBillingDAO.java
@@ -42,8 +42,8 @@ public class AzureBillingDAO extends BaseBillingDAO {
@Override
protected List<Bson> cloudMatchCriteria(BillingFilter filter) {
- if (!filter.getService().isEmpty()) {
- return Collections.singletonList(Filters.in(MongoKeyWords.METER_CATEGORY, filter.getService()));
+ if (!filter.getProducts().isEmpty()) {
+ return Collections.singletonList(Filters.in(MongoKeyWords.METER_CATEGORY, filter.getProducts()));
} else {
return Collections.emptyList();
}
diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/model/BillingData.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/BillingReportDTO.java
similarity index 58%
copy from services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/model/BillingData.java
copy to services/self-service/src/main/java/com/epam/dlab/backendapi/domain/BillingReportDTO.java
index 32a98ed..25f1831 100644
--- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/model/BillingData.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/BillingReportDTO.java
@@ -17,54 +17,31 @@
* under the License.
*/
-package com.epam.dlab.billing.gcp.model;
+package com.epam.dlab.backendapi.domain;
+import com.epam.dlab.dto.billing.BillingResourceType;
+import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Builder;
import lombok.Data;
-import org.springframework.data.annotation.Id;
-import org.springframework.data.mongodb.core.mapping.Document;
-import org.springframework.data.mongodb.core.mapping.Field;
import java.time.LocalDate;
@Data
@Builder
-@Document(collection = "billing")
-public class BillingData {
- @Id
- private String id;
- private String user;
- @Field("resource_name")
- private String displayName;
+public class BillingReportDTO {
+ private String dlabId;
+ @JsonProperty("resource_name")
private String resourceName;
- @Field("from")
+ private String project;
+ private String user;
+ @JsonProperty("from")
private LocalDate usageDateFrom;
- @Field("to")
+ @JsonProperty("to")
private LocalDate usageDateTo;
- @Field("usage_date")
- private String usageDate;
private String product;
private String usageType;
private Double cost;
- @Field("currency_code")
private String currency;
- private String project;
- private String exploratoryName;
- private String computationalName;
- @Field("dlab_id")
- private String dlabId;
- @Field("dlab_resource_type")
- private ResourceType resourceType;
-
-
- public enum ResourceType {
- EDGE,
- SSN,
- SHARED_BUCKET,
- SSN_BUCKET,
- EDGE_BUCKET,
- VOLUME,
- EXPLORATORY,
- COMPUTATIONAL
- }
+ @JsonProperty("resource_type")
+ private BillingResourceType resourceType;
}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/modules/CloudProviderModule.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/modules/CloudProviderModule.java
index 7ea2739..0ee0d10 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/modules/CloudProviderModule.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/modules/CloudProviderModule.java
@@ -33,11 +33,13 @@ import com.epam.dlab.backendapi.resources.azure.ComputationalResourceAzure;
import com.epam.dlab.backendapi.resources.gcp.ComputationalResourceGcp;
import com.epam.dlab.backendapi.resources.gcp.GcpOauthResource;
import com.epam.dlab.backendapi.service.BillingService;
+import com.epam.dlab.backendapi.service.BillingServiceNew;
import com.epam.dlab.backendapi.service.InfrastructureInfoService;
import com.epam.dlab.backendapi.service.InfrastructureTemplateService;
import com.epam.dlab.backendapi.service.aws.AwsBillingService;
import com.epam.dlab.backendapi.service.azure.AzureBillingService;
import com.epam.dlab.backendapi.service.gcp.GcpBillingService;
+import com.epam.dlab.backendapi.service.impl.BillingServiceImplNew;
import com.epam.dlab.backendapi.service.impl.InfrastructureInfoServiceImpl;
import com.epam.dlab.backendapi.service.impl.InfrastructureTemplateServiceImpl;
import com.epam.dlab.cloud.CloudModule;
@@ -69,6 +71,7 @@ public class CloudProviderModule extends CloudModule {
@Override
protected void configure() {
bindBilling();
+ bind(BillingServiceNew.class).to(BillingServiceImplNew.class);
bind(InfrastructureInfoService.class).to(InfrastructureInfoServiceImpl.class);
bind(InfrastructureTemplateService.class).to(InfrastructureTemplateServiceImpl.class);
bind(SchedulerConfiguration.class).toInstance(
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/BillingResource.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/BillingResource.java
index 0d27fad..26cc9a9 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/BillingResource.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/BillingResource.java
@@ -22,6 +22,7 @@ package com.epam.dlab.backendapi.resources;
import com.epam.dlab.auth.UserInfo;
import com.epam.dlab.backendapi.resources.dto.BillingFilter;
import com.epam.dlab.backendapi.service.BillingService;
+import com.epam.dlab.backendapi.service.BillingServiceNew;
import com.google.inject.Inject;
import io.dropwizard.auth.Auth;
import org.bson.Document;
@@ -29,6 +30,7 @@ import org.bson.Document;
import javax.validation.Valid;
import javax.validation.constraints.NotNull;
import javax.ws.rs.Consumes;
+import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
@@ -41,10 +43,12 @@ import javax.ws.rs.core.Response;
public class BillingResource {
private final BillingService billingService;
+ private final BillingServiceNew billingServiceNew;
@Inject
- public BillingResource(BillingService billingService) {
+ public BillingResource(BillingService billingService, BillingServiceNew billingServiceNew) {
this.billingService = billingService;
+ this.billingServiceNew = billingServiceNew;
}
@POST
@@ -54,6 +58,16 @@ public class BillingResource {
return billingService.getBillingReport(userInfo, formDTO);
}
+ @GET
+ @Path("/report2")
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response getBillingReport2(
+// @Auth UserInfo userInfo,
+// @Valid @NotNull BillingFilter formDTO
+ ) {
+ return Response.ok(billingServiceNew.getBillingReport(null, null)).build();
+ }
+
@POST
@Path("/report/download")
@Produces(MediaType.APPLICATION_OCTET_STREAM)
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/BillingFilter.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/BillingFilter.java
index f820169..52363a8 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/BillingFilter.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/BillingFilter.java
@@ -23,29 +23,36 @@ import com.epam.dlab.dto.UserInstanceStatus;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.NonNull;
import java.util.Collections;
import java.util.List;
@Data
+@NoArgsConstructor
@JsonIgnoreProperties(ignoreUnknown = true)
public class BillingFilter {
- @JsonProperty
- protected List<String> user;
+ @NonNull
+ private List<String> users;
+ @NonNull
@JsonProperty("dlab_id")
- protected String dlabId;
- @JsonProperty("resource_type")
- protected List<String> resourceType;
+ private String dlabId;
+ @NonNull
@JsonProperty("date_start")
- protected String dateStart;
+ private String dateStart;
+ @NonNull
@JsonProperty("date_end")
- protected String dateEnd;
- @JsonProperty("status")
- protected List<UserInstanceStatus> statuses = Collections.emptyList();
- @JsonProperty("project")
- protected List<String> projects;
- @JsonProperty
- private List<String> service;
- @JsonProperty
- private List<String> shape;
+ private String dateEnd;
+ @NonNull
+ @JsonProperty("resource_type")
+ private List<String> resourceTypes;
+ @NonNull
+ private List<UserInstanceStatus> statuses = Collections.emptyList();
+ @NonNull
+ private List<String> projects;
+ @NonNull
+ private List<String> products;
+ @NonNull
+ private List<String> shapes;
}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/BillingService.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/BillingService.java
index 5f79280..c16bd10 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/BillingService.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/BillingService.java
@@ -59,7 +59,7 @@ public abstract class BillingService {
}
public Document getBillingReport(UserInfo userInfo, BillingFilter filter) {
- filter.getUser().replaceAll(s -> s.equalsIgnoreCase(BaseBillingDAO.SHARED_RESOURCE_NAME) ? null : s);
+ filter.getUsers().replaceAll(s -> s.equalsIgnoreCase(BaseBillingDAO.SHARED_RESOURCE_NAME) ? null : s);
return getReport(userInfo, filter);
}
diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/BillingDAO.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/BillingServiceNew.java
similarity index 72%
copy from services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/BillingDAO.java
copy to services/self-service/src/main/java/com/epam/dlab/backendapi/service/BillingServiceNew.java
index 7e6b0b7..b486a23 100644
--- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/BillingDAO.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/BillingServiceNew.java
@@ -17,13 +17,15 @@
* under the License.
*/
-package com.epam.dlab.billing.gcp.dao;
+package com.epam.dlab.backendapi.service;
-import com.epam.dlab.billing.gcp.model.GcpBillingData;
+import com.epam.dlab.auth.UserInfo;
+import com.epam.dlab.backendapi.domain.BillingReportDTO;
+import com.epam.dlab.backendapi.resources.dto.BillingFilter;
import java.util.List;
-public interface BillingDAO {
+public interface BillingServiceNew {
- List<GcpBillingData> getBillingData() throws InterruptedException;
+ List<BillingReportDTO> getBillingReport(UserInfo userInfo, BillingFilter filter);
}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/ExploratoryService.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/ExploratoryService.java
index 4348819..f744bc5 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/ExploratoryService.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/ExploratoryService.java
@@ -50,6 +50,8 @@ public interface ExploratoryService {
List<UserInstanceDTO> getInstancesWithStatuses(String user, UserInstanceStatus exploratoryStatus,
UserInstanceStatus computationalStatus);
+ List<UserInstanceDTO> findAll();
+
void updateClusterConfig(UserInfo userInfo, String exploratoryName, List<ClusterConfig> config);
Optional<UserInstanceDTO> getUserInstance(String user, String exploratoryName);
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BillingServiceImplNew.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BillingServiceImplNew.java
new file mode 100644
index 0000000..733f169
--- /dev/null
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BillingServiceImplNew.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.backendapi.service.impl;
+
+import com.epam.dlab.auth.UserInfo;
+import com.epam.dlab.backendapi.conf.SelfServiceApplicationConfiguration;
+import com.epam.dlab.backendapi.domain.BillingReportDTO;
+import com.epam.dlab.backendapi.domain.EndpointDTO;
+import com.epam.dlab.backendapi.domain.ProjectDTO;
+import com.epam.dlab.backendapi.domain.ProjectEndpointDTO;
+import com.epam.dlab.backendapi.resources.dto.BillingFilter;
+import com.epam.dlab.backendapi.service.BillingServiceNew;
+import com.epam.dlab.backendapi.service.EndpointService;
+import com.epam.dlab.backendapi.service.ExploratoryService;
+import com.epam.dlab.backendapi.service.ProjectService;
+import com.epam.dlab.backendapi.util.BillingUtils;
+import com.epam.dlab.constants.ServiceConsts;
+import com.epam.dlab.dto.billing.BillingData;
+import com.epam.dlab.exceptions.DlabException;
+import com.epam.dlab.rest.client.RESTService;
+import com.google.inject.Inject;
+import com.google.inject.name.Named;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.http.client.utils.URIBuilder;
+
+import javax.ws.rs.core.GenericType;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+@Slf4j
+public class BillingServiceImplNew implements BillingServiceNew {
+ private final ProjectService projectService;
+ private final EndpointService endpointService;
+ private final ExploratoryService exploratoryService;
+ private final SelfServiceApplicationConfiguration configuration;
+ private final RESTService provisioningService;
+
+ @Inject
+ public BillingServiceImplNew(ProjectService projectService, EndpointService endpointService,
+ ExploratoryService exploratoryService, SelfServiceApplicationConfiguration configuration,
+ @Named(ServiceConsts.PROVISIONING_SERVICE_NAME) RESTService provisioningService) {
+ this.projectService = projectService;
+ this.endpointService = endpointService;
+ this.exploratoryService = exploratoryService;
+ this.configuration = configuration;
+ this.provisioningService = provisioningService;
+ }
+
+ @Override
+ public List<BillingReportDTO> getBillingReport(UserInfo userInfo, BillingFilter filter) {
+// filter.getUser().replaceAll(s -> s.equalsIgnoreCase(BaseBillingDAO.SHARED_RESOURCE_NAME) ? null : s); // tell front end not to pass SHARED_RESOURCE_NAME and remove this line
+ final String serviceBaseName = configuration.getServiceBaseName();
+ final Stream<BillingReportDTO> ssnBillingDataStream = BillingUtils.ssnBillingDataStream(serviceBaseName);
+ final Stream<BillingReportDTO> billableUserInstances = exploratoryService.findAll()
+ .stream()
+ .filter(userInstance -> Objects.nonNull(userInstance.getExploratoryId()))
+ .flatMap(BillingUtils::exploratoryBillingDataStream);
+
+ final Stream<BillingReportDTO> billableEdges = projectService.getProjects()
+ .stream()
+ .collect(Collectors.toMap(ProjectDTO::getName, ProjectDTO::getEndpoints))
+ .entrySet()
+ .stream()
+ .flatMap(e -> projectEdges(serviceBaseName, e.getKey(), e.getValue()));
+
+ final Map<String, BillingReportDTO> billableResources = Stream.of(billableUserInstances, billableEdges, ssnBillingDataStream)
+ .flatMap(s -> s)
+ .filter(bd -> Objects.nonNull(bd.getDlabId()))
+ .collect(Collectors.toMap(BillingReportDTO::getDlabId, b -> b));
+ log.debug("Billable resources are: {}", billableResources);
+
+ List<BillingReportDTO> billingReport = getRemoteBillingData()
+ .stream()
+ .map(bd -> toBillingData(bd, getOrDefault(billableResources, bd.getTag())))
+ .collect(Collectors.toList());
+ log.debug("Billing report: {}", billingReport);
+
+ return billingReport;
+ }
+
+ private Stream<BillingReportDTO> projectEdges(String serviceBaseName, String projectName, List<ProjectEndpointDTO> endpoints) {
+ return endpoints
+ .stream()
+ .flatMap(endpoint -> BillingUtils.edgeBillingDataStream(projectName, serviceBaseName, endpoint.getName()));
+ }
+
+ private BillingReportDTO getOrDefault(Map<String, BillingReportDTO> billableResources, String tag) {
+ return billableResources.getOrDefault(tag, BillingReportDTO.builder().dlabId(tag).build());
+ }
+
+ private List<BillingData> getRemoteBillingData() {
+ List<EndpointDTO> endpoints = endpointService.getEndpoints();
+ ExecutorService executor = Executors.newFixedThreadPool(endpoints.size());
+ List<Callable<List<BillingData>>> callableTasks = new ArrayList<>();
+ endpoints.forEach(e ->
+ callableTasks.add(getTask(getBillingUrl(e.getUrl()))));
+
+ List<BillingData> billingData;
+ try {
+ log.debug("Trying to retrieve billing info for {}", endpoints);
+ billingData = executor.invokeAll(callableTasks)
+ .stream()
+ .map(this::getBillingReportDTOS)
+ .flatMap(Collection::stream)
+ .collect(Collectors.toList());
+ } catch (Exception e) {
+ executor.shutdown();
+ log.error("Cannot retrieve billing information {}", e.getMessage(), e);
+ throw new DlabException("Cannot retrieve billing information");
+ }
+ executor.shutdown();
+ return billingData;
+ }
+
+ private List<BillingData> getBillingReportDTOS(Future<List<BillingData>> s) {
+ try {
+ return s.get();
+ } catch (InterruptedException | ExecutionException e) {
+ log.error("Cannot retrieve billing information {}", e.getMessage(), e);
+ throw new DlabException("Cannot retrieve billing information");
+ }
+ }
+
+ private String getBillingUrl(String endpointUrl) {
+ URI uri;
+ try {
+ uri = new URI(endpointUrl);
+ } catch (URISyntaxException e) {
+ log.error("Wrong URI syntax {}", e.getMessage(), e);
+ throw new DlabException("Wrong URI syntax");
+ }
+ return new URIBuilder()
+ .setScheme(uri.getScheme())
+ .setHost(uri.getHost())
+ .setPort(8081)
+ .setPath("/api/billing/report")
+ .toString();
+ }
+
+ private Callable<List<BillingData>> getTask(String url) {
+ return () -> provisioningService.get(url, new GenericType<List<BillingData>>() {
+ });
+ }
+
+ private BillingReportDTO toBillingData(BillingData billingData, BillingReportDTO billingReportDTO) {
+ return BillingReportDTO.builder()
+ .currency(billingData.getCurrency())
+ .product(billingData.getProduct())
+ .project(billingReportDTO.getProject())
+ .usageDateTo(billingData.getUsageDateTo())
+ .usageDateFrom(billingData.getUsageDateFrom())
+ .usageType(billingData.getUsageType())
+ .user(billingReportDTO.getUser())
+ .dlabId(billingData.getTag())
+ .resourceType(billingReportDTO.getResourceType())
+ .build();
+ }
+}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/ExploratoryServiceImpl.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/ExploratoryServiceImpl.java
index 17b8967..59f9adc 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/ExploratoryServiceImpl.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/ExploratoryServiceImpl.java
@@ -192,6 +192,11 @@ public class ExploratoryServiceImpl implements ExploratoryService {
}
@Override
+ public List<UserInstanceDTO> findAll() {
+ return exploratoryDAO.getInstances();
+ }
+
+ @Override
public void updateClusterConfig(UserInfo userInfo, String exploratoryName, List<ClusterConfig> config) {
final String userName = userInfo.getName();
final String token = userInfo.getAccessToken();
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java
new file mode 100644
index 0000000..6fbb94c
--- /dev/null
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.backendapi.util;
+
+import com.epam.dlab.backendapi.domain.BillingReportDTO;
+import com.epam.dlab.dto.UserInstanceDTO;
+import com.epam.dlab.dto.billing.BillingResourceType;
+import com.epam.dlab.dto.computational.UserComputationalResource;
+
+import java.util.stream.Stream;
+
+public class BillingUtils {
+
+ private static final String EDGE_FORMAT = "%s-%s-%s-edge";
+ private static final String EDGE_VOLUME_FORMAT = "%s-%s-%s-edge-volume-primary";
+ private static final String EDGE_BUCKET_FORMAT = "%s-%s-bucket";
+ private static final String VOLUME_PRIMARY_FORMAT = "%s-volume-primary";
+ private static final String VOLUME_SECONDARY_FORMAT = "%s-volume-secondary";
+ private static final String VOLUME_PRIMARY = "Volume primary";
+ private static final String VOLUME_SECONDARY = "Volume secondary";
+ private static final String SHARED_RESOURCE = "Shared resource";
+
+ public static Stream<BillingReportDTO> edgeBillingDataStream(String project, String sbn, String endpoint) {
+ final String userEdgeId = String.format(EDGE_FORMAT, sbn, project.toLowerCase(), endpoint);
+ final String edgeVolumeId = String.format(EDGE_VOLUME_FORMAT, sbn, project.toLowerCase(), endpoint);
+ final String edgeBucketId = String.format(EDGE_BUCKET_FORMAT, sbn, project.toLowerCase());
+ return Stream.of(
+ BillingReportDTO.builder().resourceName("EDGE node").user(SHARED_RESOURCE).project(project).dlabId(userEdgeId).resourceType(BillingResourceType.EDGE).build(),
+ BillingReportDTO.builder().resourceName("EDGE volume").user(SHARED_RESOURCE).project(project).dlabId(edgeVolumeId).resourceType(BillingResourceType.VOLUME).build(),
+ BillingReportDTO.builder().resourceName("EDGE bucket").user(SHARED_RESOURCE).project(project).dlabId(edgeBucketId).resourceType(BillingResourceType.EDGE_BUCKET).build()
+ );
+ }
+
+ public static Stream<BillingReportDTO> ssnBillingDataStream(String sbn) {
+ final String ssnId = sbn + "-ssn";
+ final String bucketName = sbn.replaceAll("_", "-");
+ return Stream.of(
+ BillingReportDTO.builder().user(SHARED_RESOURCE).resourceName("SSN").dlabId(ssnId).resourceType(BillingResourceType.SSN).build(),
+ BillingReportDTO.builder().user(SHARED_RESOURCE).resourceName("SSN Volume").dlabId(String.format(VOLUME_PRIMARY_FORMAT, ssnId)).resourceType(BillingResourceType.VOLUME).build(),
+ BillingReportDTO.builder().user(SHARED_RESOURCE).resourceName("SSN bucket").dlabId(bucketName + "-ssn" +
+ "-bucket").resourceType(BillingResourceType.SSN_BUCKET).build(),
+ BillingReportDTO.builder().user(SHARED_RESOURCE).resourceName("Collaboration bucket").dlabId(bucketName +
+ "-shared-bucket").resourceType(BillingResourceType.SHARED_BUCKET).build()
+ );
+ }
+
+ public static Stream<BillingReportDTO> exploratoryBillingDataStream(UserInstanceDTO userInstance) {
+ final Stream<BillingReportDTO> computationalStream = userInstance.getResources()
+ .stream()
+ .filter(cr -> cr.getComputationalId() != null)
+ .flatMap(cr -> Stream.of(computationalBillableResource(userInstance, cr),
+ withExploratoryName(userInstance).resourceName(cr.getComputationalName() + ":" + VOLUME_PRIMARY).dlabId(String.format(VOLUME_PRIMARY_FORMAT, cr.getComputationalId()))
+ .resourceType(BillingResourceType.VOLUME).build()));
+ final String exploratoryId = userInstance.getExploratoryId();
+ final String primaryVolumeId = String.format(VOLUME_PRIMARY_FORMAT, exploratoryId);
+ final String secondaryVolumeId = String.format(VOLUME_SECONDARY_FORMAT, exploratoryId);
+ final Stream<BillingReportDTO> exploratoryStream = Stream.of(
+ withExploratoryName(userInstance).resourceName(userInstance.getExploratoryName()).dlabId(exploratoryId).resourceType(BillingResourceType.EXPLORATORY).build(),
+ withExploratoryName(userInstance).resourceName(VOLUME_PRIMARY).dlabId(primaryVolumeId).resourceType(BillingResourceType.VOLUME).build(),
+ withExploratoryName(userInstance).resourceName(VOLUME_SECONDARY).dlabId(secondaryVolumeId).resourceType(BillingResourceType.VOLUME).build());
+ return Stream.concat(computationalStream, exploratoryStream);
+ }
+
+ private static BillingReportDTO computationalBillableResource(UserInstanceDTO userInstance,
+ UserComputationalResource cr) {
+ return withExploratoryName(userInstance)
+ .dlabId(cr.getComputationalId())
+ .resourceName(cr.getComputationalName())
+ .resourceType(BillingResourceType.COMPUTATIONAL)
+ .project(userInstance.getProject())
+ .build();
+ }
+
+ private static BillingReportDTO.BillingReportDTOBuilder withExploratoryName(UserInstanceDTO userInstance) {
+ return BillingReportDTO.builder().user(userInstance.getUser())
+ .project(userInstance.getProject());
+ }
+}
diff --git a/services/self-service/src/test/java/com/epam/dlab/backendapi/service/impl/InfrastructureTemplateServiceBaseTest.java b/services/self-service/src/test/java/com/epam/dlab/backendapi/service/impl/InfrastructureTemplateServiceBaseTest.java
index e53b78c..56c838a 100644
--- a/services/self-service/src/test/java/com/epam/dlab/backendapi/service/impl/InfrastructureTemplateServiceBaseTest.java
+++ b/services/self-service/src/test/java/com/epam/dlab/backendapi/service/impl/InfrastructureTemplateServiceBaseTest.java
@@ -89,7 +89,7 @@ public class InfrastructureTemplateServiceBaseTest {
emDto2.setExploratoryEnvironmentShapes(shapes2);
List<ExploratoryMetadataDTO> expectedEmdDtoList = Arrays.asList(emDto1, emDto2);
when(userGroupDao.getUserGroups(anyString())).thenReturn(Collections.emptySet());
- when(provisioningService.get(anyString(), anyString(), any())).thenReturn(expectedEmdDtoList.toArray());
+ when(provisioningService.get(anyString(), anyString(), any(Class.class))).thenReturn(expectedEmdDtoList.toArray());
when(settingsDAO.getConfOsFamily()).thenReturn("someConfOsFamily");
UserInfo userInfo = new UserInfo("test", "token");
@@ -108,7 +108,7 @@ public class InfrastructureTemplateServiceBaseTest {
public void getExploratoryTemplatesWithException() {
when(endpointService.get(anyString())).thenReturn(endpointDTO());
doThrow(new DlabException("Could not load list of exploratory templates for user"))
- .when(provisioningService).get(anyString(), anyString(), any());
+ .when(provisioningService).get(anyString(), anyString(), any(Class.class));
UserInfo userInfo = new UserInfo("test", "token");
try {
@@ -131,7 +131,7 @@ public class InfrastructureTemplateServiceBaseTest {
);
when(projectDAO.get(anyString())).thenReturn(Optional.of(new ProjectDTO("project", Collections.emptySet(),
null, null, null, null, true)));
- when(provisioningService.get(anyString(), anyString(), any())).thenReturn(expectedCmdDtoList.toArray(new ComputationalMetadataDTO[]{}));
+ when(provisioningService.get(anyString(), anyString(), any(Class.class))).thenReturn(expectedCmdDtoList.toArray(new ComputationalMetadataDTO[]{}));
List<FullComputationalTemplate> expectedFullCmdDtoList = expectedCmdDtoList.stream()
.map(e -> infrastructureTemplateServiceBaseChild.getCloudFullComputationalTemplate(e))
@@ -154,7 +154,7 @@ public class InfrastructureTemplateServiceBaseTest {
public void getComputationalTemplatesWhenMethodThrowsException() {
when(endpointService.get(anyString())).thenReturn(endpointDTO());
doThrow(new DlabException("Could not load list of computational templates for user"))
- .when(provisioningService).get(anyString(), anyString(), any());
+ .when(provisioningService).get(anyString(), anyString(), any(Class.class));
UserInfo userInfo = new UserInfo("test", "token");
try {
@@ -173,7 +173,7 @@ public class InfrastructureTemplateServiceBaseTest {
final ComputationalMetadataDTO computationalMetadataDTO = new ComputationalMetadataDTO("dataengine-service");
computationalMetadataDTO.setComputationResourceShapes(Collections.emptyMap());
List<ComputationalMetadataDTO> expectedCmdDtoList = Collections.singletonList(computationalMetadataDTO);
- when(provisioningService.get(anyString(), anyString(), any())).thenReturn(expectedCmdDtoList.toArray(new ComputationalMetadataDTO[]{}));
+ when(provisioningService.get(anyString(), anyString(), any(Class.class))).thenReturn(expectedCmdDtoList.toArray(new ComputationalMetadataDTO[]{}));
when(projectDAO.get(anyString())).thenReturn(Optional.of(new ProjectDTO("project", Collections.emptySet(),
null, null, null, null, true)));
when(configuration.getMinEmrInstanceCount()).thenReturn(1);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org