You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by bh...@apache.org on 2019/07/26 10:38:16 UTC
[incubator-dlab] branch bugfix-DLAB-948 created (now eef786e)
This is an automated email from the ASF dual-hosted git repository.
bhliva pushed a change to branch bugfix-DLAB-948
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git.
at eef786e DLAB-948 fixed bug connected with billing
This branch includes the following new commits:
new eef786e DLAB-948 fixed bug connected with billing
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org
[incubator-dlab] 01/01: DLAB-948 fixed bug connected with billing
Posted by bh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bhliva pushed a commit to branch bugfix-DLAB-948
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit eef786e4a1511d088a3d90d4b025ead2932d1f84
Author: bhliva <bo...@epam.com>
AuthorDate: Fri Jul 26 13:38:01 2019 +0300
DLAB-948 fixed bug connected with billing
---
.../com/epam/dlab/mongo/DlabResourceTypeDAO.java | 24 +-
.../dlab/backendapi/dao/aws/AwsBillingDAO.java | 303 +++++++++++----------
2 files changed, 162 insertions(+), 165 deletions(-)
diff --git a/services/billing-aws/src/main/java/com/epam/dlab/mongo/DlabResourceTypeDAO.java b/services/billing-aws/src/main/java/com/epam/dlab/mongo/DlabResourceTypeDAO.java
index 9738de6..f03e42f 100644
--- a/services/billing-aws/src/main/java/com/epam/dlab/mongo/DlabResourceTypeDAO.java
+++ b/services/billing-aws/src/main/java/com/epam/dlab/mongo/DlabResourceTypeDAO.java
@@ -21,7 +21,6 @@ package com.epam.dlab.mongo;
import com.epam.dlab.billing.BillingCalculationUtils;
import com.epam.dlab.billing.DlabResourceType;
-import com.epam.dlab.core.BillingUtils;
import com.epam.dlab.dto.base.DataEngineType;
import com.epam.dlab.exceptions.InitializationException;
import com.epam.dlab.exceptions.ParseException;
@@ -147,20 +146,17 @@ public class DlabResourceTypeDAO implements MongoConstants {
resourceList.append(getBucketName(sbName) + "-shared-bucket", "Collaboration bucket", DlabResourceType
.COLLABORATION_BUCKET, null, null);
- // Add EDGE
- Bson projection = fields(include(FIELD_ID, FIELD_EDGE_BUCKET));
- Iterable<Document> docs = connection.getCollection(COLLECTION_USER_EDGE).find().projection(projection);
+ // Add PROJECTS
+ Bson projection = fields(include("name"));
+ Iterable<Document> docs = connection.getCollection("Projects").find().projection(projection);
for (Document d : docs) {
- String username = d.getString(FIELD_ID);
- final String simpleUserName = BillingUtils.getSimpleUserName(username);
- resourceList.append(sbName + "-" + simpleUserName + "-edge", "EDGE Node",
- DlabResourceType.EDGE, username, null);
- resourceList.append(sbName + "-" + simpleUserName + "-bucket", "Personal bucket",
- DlabResourceType.COLLABORATION_BUCKET, username, null);
- resourceList.append(sbName + "-" + simpleUserName + "-edge-volume-primary",
- "EDGE Volume", DlabResourceType.VOLUME, username, null);
- resourceList.append(getBucketName(d.getString(FIELD_EDGE_BUCKET)), "EDGE bucket", DlabResourceType
- .EDGE_BUCKET, username, null);
+ String projectName = d.getString("name");
+ resourceList.append(sbName + "-" + projectName + "-edge", "EDGE Node",
+ DlabResourceType.EDGE, null, null);
+ resourceList.append(sbName + "-" + projectName + "-bucket", "Project bucket",
+ DlabResourceType.COLLABORATION_BUCKET, null, null);
+ resourceList.append(sbName + "-" + projectName + "-edge-volume-primary",
+ "EDGE Volume", DlabResourceType.VOLUME, null, null);
}
// Add exploratory
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/aws/AwsBillingDAO.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/aws/AwsBillingDAO.java
index c5e0c92..adcef5e 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/aws/AwsBillingDAO.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/aws/AwsBillingDAO.java
@@ -21,13 +21,13 @@ package com.epam.dlab.backendapi.dao.aws;
import com.epam.dlab.auth.UserInfo;
import com.epam.dlab.backendapi.dao.BaseBillingDAO;
+import com.epam.dlab.backendapi.domain.ProjectDTO;
import com.epam.dlab.backendapi.resources.dto.aws.AwsBillingFilter;
import com.epam.dlab.backendapi.roles.RoleType;
import com.epam.dlab.backendapi.roles.UserRoles;
import com.epam.dlab.billing.BillingCalculationUtils;
import com.epam.dlab.billing.DlabResourceType;
import com.epam.dlab.dto.UserInstanceStatus;
-import com.epam.dlab.util.UsernameUtils;
import com.mongodb.client.AggregateIterable;
import com.mongodb.client.FindIterable;
import org.apache.commons.lang3.StringUtils;
@@ -42,7 +42,6 @@ import java.util.Map;
import java.util.Optional;
import static com.epam.dlab.backendapi.dao.MongoCollections.BILLING;
-import static com.epam.dlab.backendapi.dao.MongoCollections.USER_EDGE;
import static com.epam.dlab.model.aws.ReportLine.*;
import static com.mongodb.client.model.Accumulators.*;
import static com.mongodb.client.model.Aggregates.*;
@@ -54,153 +53,155 @@ import static com.mongodb.client.model.Projections.include;
* DAO for user billing.
*/
public class AwsBillingDAO extends BaseBillingDAO<AwsBillingFilter> {
- private static final Logger LOGGER = LoggerFactory.getLogger(AwsBillingDAO.class);
-
- public static final String DLAB_RESOURCE_TYPE = "dlab_resource_type";
- public static final String USAGE_DATE_START = "usage_date_start";
- public static final String USAGE_DATE_END = "usage_date_end";
- public static final String TAG_RESOURCE_ID = "tag_resource_id";
-
- /**
- * Add the conditions to the list.
- *
- * @param conditions the list of conditions.
- * @param fieldName the name of field.
- * @param values the values.
- */
- private void addCondition(List<Bson> conditions, String fieldName, List<String> values) {
- if (values != null && !values.isEmpty()) {
- conditions.add(in(fieldName, values));
- }
- }
-
- /**
- * Build and returns the billing report.
- *
- * @param userInfo user info
- * @param filter the filter for report data.
- * @return billing report
- */
- public Document getReport(UserInfo userInfo, AwsBillingFilter filter) {
- // Create filter
- List<Bson> conditions = new ArrayList<>();
- boolean isFullReport = UserRoles.checkAccess(userInfo, RoleType.PAGE, "/api/infrastructure_provision/billing", userInfo.getRoles());
- setUserFilter(userInfo, filter, isFullReport);
- addCondition(conditions, USER, filter.getUser());
- addCondition(conditions, FIELD_PRODUCT, filter.getProduct());
- addCondition(conditions, DLAB_RESOURCE_TYPE, DlabResourceType.getResourceTypeIds(filter.getResourceType()));
-
- addAnotherConditionsIfNecessary(conditions, filter);
-
- // Create aggregation conditions
-
- List<Bson> pipeline = new ArrayList<>();
- if (!conditions.isEmpty()) {
- LOGGER.trace("Filter conditions is {}", conditions);
- pipeline.add(match(and(conditions)));
- }
- pipeline.add(
- group(getGroupingFields(USER, FIELD_DLAB_ID, DLAB_RESOURCE_TYPE, FIELD_PRODUCT, FIELD_RESOURCE_TYPE,
- FIELD_CURRENCY_CODE),
- sum(FIELD_COST, "$" + FIELD_COST),
- min(USAGE_DATE_START, "$" + FIELD_USAGE_DATE),
- max(USAGE_DATE_END, "$" + FIELD_USAGE_DATE)
- ));
- pipeline.add(
- sort(new Document(ID + "." + USER, 1)
- .append(ID + "." + FIELD_DLAB_ID, 1)
- .append(ID + "." + DLAB_RESOURCE_TYPE, 1)
- .append(ID + "." + FIELD_PRODUCT, 1))
- );
-
- // Get billing report and the list of shape info
- AggregateIterable<Document> agg = getCollection(BILLING).aggregate(pipeline);
- Map<String, ShapeInfo> shapes = getShapes(filter.getShape());
-
- // Build billing report lines
- List<Document> reportItems = new ArrayList<>();
- boolean filterByShape = !(filter.getShape() == null || filter.getShape().isEmpty());
- String usageDateStart = null;
- String usageDateEnd = null;
- double costTotal = 0;
-
- for (Document d : agg) {
- Document id = (Document) d.get(ID);
- String resourceId = id.getString(FIELD_DLAB_ID);
- ShapeInfo shape = shapes.get(resourceId);
- final UserInstanceStatus status = Optional.ofNullable(shape).map(ShapeInfo::getStatus).orElse(null);
- if ((filterByShape && shape == null) || (!filter.getStatuses().isEmpty() && filter.getStatuses().stream()
- .noneMatch(s -> s.equals(status)))) {
- continue;
- }
-
- String resourceTypeId = DlabResourceType.getResourceTypeName(id.getString(DLAB_RESOURCE_TYPE));
- String shapeName = generateShapeName(shape);
- String dateStart = d.getString(USAGE_DATE_START);
- if (StringUtils.compare(usageDateStart, dateStart, false) > 0) {
- usageDateStart = dateStart;
- }
- String dateEnd = d.getString(USAGE_DATE_END);
- if (StringUtils.compare(usageDateEnd, dateEnd) < 0) {
- usageDateEnd = dateEnd;
- }
- double cost = BillingCalculationUtils.round(d.getDouble(FIELD_COST), 2);
- costTotal += cost;
-
- Document item = new Document()
- .append(FIELD_USER_ID, getUserOrDefault(id.getString(USER)))
- .append(FIELD_DLAB_ID, resourceId)
- .append(DLAB_RESOURCE_TYPE, resourceTypeId)
- .append(SHAPE, shapeName)
- .append(STATUS,
- Optional.ofNullable(status).map(UserInstanceStatus::toString).orElse(StringUtils.EMPTY))
- .append(FIELD_PRODUCT, id.getString(FIELD_PRODUCT))
- .append(FIELD_RESOURCE_TYPE, id.getString(FIELD_RESOURCE_TYPE))
- .append(FIELD_COST, BillingCalculationUtils.formatDouble(cost))
- .append(FIELD_CURRENCY_CODE, id.getString(FIELD_CURRENCY_CODE))
- .append(USAGE_DATE_START, dateStart)
- .append(USAGE_DATE_END, dateEnd);
- reportItems.add(item);
- }
-
- return new Document()
- .append(SERVICE_BASE_NAME, settings.getServiceBaseName())
- .append(TAG_RESOURCE_ID, settings.getConfTagResourceId())
- .append(USAGE_DATE_START, usageDateStart)
- .append(USAGE_DATE_END, usageDateEnd)
- .append(ITEMS, reportItems)
- .append(COST_TOTAL, BillingCalculationUtils.formatDouble(BillingCalculationUtils.round(costTotal, 2)))
- .append(FIELD_CURRENCY_CODE, (reportItems.isEmpty() ? null :
- reportItems.get(0).getString(FIELD_CURRENCY_CODE)))
- .append(FULL_REPORT, isFullReport);
- }
-
- private void addAnotherConditionsIfNecessary(List<Bson> conditions, AwsBillingFilter filter) {
- if (filter.getDlabId() != null && !filter.getDlabId().isEmpty()) {
- conditions.add(regex(FIELD_DLAB_ID, filter.getDlabId(), "i"));
- }
-
- if (filter.getDateStart() != null && !filter.getDateStart().isEmpty()) {
- conditions.add(gte(FIELD_USAGE_DATE, filter.getDateStart()));
- }
- if (filter.getDateEnd() != null && !filter.getDateEnd().isEmpty()) {
- conditions.add(lte(FIELD_USAGE_DATE, filter.getDateEnd()));
- }
- }
-
- protected void appendSsnAndEdgeNodeType(List<String> shapeNames, Map<String, ShapeInfo> shapes) {
- // Add SSN and EDGE nodes
- final String ssnShape = "t2.medium";
- if (shapeNames == null || shapeNames.isEmpty() || shapeNames.contains(ssnShape)) {
- String serviceBaseName = settings.getServiceBaseName();
- shapes.put(serviceBaseName + "-ssn", new ShapeInfo(ssnShape, UserInstanceStatus.RUNNING));
- FindIterable<Document> docs = getCollection(USER_EDGE)
- .find()
- .projection(fields(include(ID, EDGE_STATUS)));
- for (Document d : docs) {
- shapes.put(String.join("-", serviceBaseName, UsernameUtils.removeDomain(d.getString(ID)), "edge"),
- new ShapeInfo(ssnShape, UserInstanceStatus.of(d.getString(EDGE_STATUS))));
- }
- }
- }
+ private static final Logger LOGGER = LoggerFactory.getLogger(AwsBillingDAO.class);
+
+ public static final String DLAB_RESOURCE_TYPE = "dlab_resource_type";
+ public static final String USAGE_DATE_START = "usage_date_start";
+ public static final String USAGE_DATE_END = "usage_date_end";
+ public static final String TAG_RESOURCE_ID = "tag_resource_id";
+
+ /**
+ * Add the conditions to the list.
+ *
+ * @param conditions the list of conditions.
+ * @param fieldName the name of field.
+ * @param values the values.
+ */
+ private void addCondition(List<Bson> conditions, String fieldName, List<String> values) {
+ if (values != null && !values.isEmpty()) {
+ conditions.add(in(fieldName, values));
+ }
+ }
+
+ /**
+ * Build and returns the billing report.
+ *
+ * @param userInfo user info
+ * @param filter the filter for report data.
+ * @return billing report
+ */
+ public Document getReport(UserInfo userInfo, AwsBillingFilter filter) {
+ // Create filter
+ List<Bson> conditions = new ArrayList<>();
+ boolean isFullReport = UserRoles.checkAccess(userInfo, RoleType.PAGE, "/api/infrastructure_provision/billing",
+ userInfo.getRoles());
+ setUserFilter(userInfo, filter, isFullReport);
+ addCondition(conditions, USER, filter.getUser());
+ addCondition(conditions, FIELD_PRODUCT, filter.getProduct());
+ addCondition(conditions, DLAB_RESOURCE_TYPE, DlabResourceType.getResourceTypeIds(filter.getResourceType()));
+
+ addAnotherConditionsIfNecessary(conditions, filter);
+
+ // Create aggregation conditions
+
+ List<Bson> pipeline = new ArrayList<>();
+ if (!conditions.isEmpty()) {
+ LOGGER.trace("Filter conditions is {}", conditions);
+ pipeline.add(match(and(conditions)));
+ }
+ pipeline.add(
+ group(getGroupingFields(USER, FIELD_DLAB_ID, DLAB_RESOURCE_TYPE, FIELD_PRODUCT, FIELD_RESOURCE_TYPE,
+ FIELD_CURRENCY_CODE),
+ sum(FIELD_COST, "$" + FIELD_COST),
+ min(USAGE_DATE_START, "$" + FIELD_USAGE_DATE),
+ max(USAGE_DATE_END, "$" + FIELD_USAGE_DATE)
+ ));
+ pipeline.add(
+ sort(new Document(ID + "." + USER, 1)
+ .append(ID + "." + FIELD_DLAB_ID, 1)
+ .append(ID + "." + DLAB_RESOURCE_TYPE, 1)
+ .append(ID + "." + FIELD_PRODUCT, 1))
+ );
+
+ // Get billing report and the list of shape info
+ AggregateIterable<Document> agg = getCollection(BILLING).aggregate(pipeline);
+ Map<String, ShapeInfo> shapes = getShapes(filter.getShape());
+
+ // Build billing report lines
+ List<Document> reportItems = new ArrayList<>();
+ boolean filterByShape = !(filter.getShape() == null || filter.getShape().isEmpty());
+ String usageDateStart = null;
+ String usageDateEnd = null;
+ double costTotal = 0;
+
+ for (Document d : agg) {
+ Document id = (Document) d.get(ID);
+ String resourceId = id.getString(FIELD_DLAB_ID);
+ ShapeInfo shape = shapes.get(resourceId);
+ final UserInstanceStatus status = Optional.ofNullable(shape).map(ShapeInfo::getStatus).orElse(null);
+ if ((filterByShape && shape == null) || (!filter.getStatuses().isEmpty() && filter.getStatuses().stream()
+ .noneMatch(s -> s.equals(status)))) {
+ continue;
+ }
+
+ String resourceTypeId = DlabResourceType.getResourceTypeName(id.getString(DLAB_RESOURCE_TYPE));
+ String shapeName = generateShapeName(shape);
+ String dateStart = d.getString(USAGE_DATE_START);
+ if (StringUtils.compare(usageDateStart, dateStart, false) > 0) {
+ usageDateStart = dateStart;
+ }
+ String dateEnd = d.getString(USAGE_DATE_END);
+ if (StringUtils.compare(usageDateEnd, dateEnd) < 0) {
+ usageDateEnd = dateEnd;
+ }
+ double cost = BillingCalculationUtils.round(d.getDouble(FIELD_COST), 2);
+ costTotal += cost;
+
+ Document item = new Document()
+ .append(FIELD_USER_ID, getUserOrDefault(id.getString(USER)))
+ .append(FIELD_DLAB_ID, resourceId)
+ .append(DLAB_RESOURCE_TYPE, resourceTypeId)
+ .append(SHAPE, shapeName)
+ .append(STATUS,
+ Optional.ofNullable(status).map(UserInstanceStatus::toString).orElse(StringUtils.EMPTY))
+ .append(FIELD_PRODUCT, id.getString(FIELD_PRODUCT))
+ .append(FIELD_RESOURCE_TYPE, id.getString(FIELD_RESOURCE_TYPE))
+ .append(FIELD_COST, BillingCalculationUtils.formatDouble(cost))
+ .append(FIELD_CURRENCY_CODE, id.getString(FIELD_CURRENCY_CODE))
+ .append(USAGE_DATE_START, dateStart)
+ .append(USAGE_DATE_END, dateEnd);
+ reportItems.add(item);
+ }
+
+ return new Document()
+ .append(SERVICE_BASE_NAME, settings.getServiceBaseName())
+ .append(TAG_RESOURCE_ID, settings.getConfTagResourceId())
+ .append(USAGE_DATE_START, usageDateStart)
+ .append(USAGE_DATE_END, usageDateEnd)
+ .append(ITEMS, reportItems)
+ .append(COST_TOTAL, BillingCalculationUtils.formatDouble(BillingCalculationUtils.round(costTotal, 2)))
+ .append(FIELD_CURRENCY_CODE, (reportItems.isEmpty() ? null :
+ reportItems.get(0).getString(FIELD_CURRENCY_CODE)))
+ .append(FULL_REPORT, isFullReport);
+ }
+
+ private void addAnotherConditionsIfNecessary(List<Bson> conditions, AwsBillingFilter filter) {
+ if (filter.getDlabId() != null && !filter.getDlabId().isEmpty()) {
+ conditions.add(regex(FIELD_DLAB_ID, filter.getDlabId(), "i"));
+ }
+
+ if (filter.getDateStart() != null && !filter.getDateStart().isEmpty()) {
+ conditions.add(gte(FIELD_USAGE_DATE, filter.getDateStart()));
+ }
+ if (filter.getDateEnd() != null && !filter.getDateEnd().isEmpty()) {
+ conditions.add(lte(FIELD_USAGE_DATE, filter.getDateEnd()));
+ }
+ }
+
+ protected void appendSsnAndEdgeNodeType(List<String> shapeNames, Map<String, ShapeInfo> shapes) {
+ // Add SSN and EDGE nodes
+ final String ssnShape = "t2.medium";
+ if (shapeNames == null || shapeNames.isEmpty() || shapeNames.contains(ssnShape)) {
+ String serviceBaseName = settings.getServiceBaseName();
+ shapes.put(serviceBaseName + "-ssn", new ShapeInfo(ssnShape, UserInstanceStatus.RUNNING));
+ FindIterable<Document> docs = getCollection("Projects")
+ .find()
+ .projection(fields(include("name", "status")));
+ for (Document d : docs) {
+ shapes.put(String.join("-", serviceBaseName, d.getString("name"), "edge"),
+ new ShapeInfo(ssnShape,
+ ProjectDTO.Status.from(ProjectDTO.Status.valueOf(d.getString("status")))));
+ }
+ }
+ }
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org