You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by of...@apache.org on 2020/04/14 14:40:10 UTC
[incubator-dlab] branch DLAB-1571 updated (0852d2c -> 623a31a)
This is an automated email from the ASF dual-hosted git repository.
ofuks pushed a change to branch DLAB-1571
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git.
from 0852d2c Merge remote-tracking branch 'origin/DLAB-1571' into DLAB-1571
new fc9e515 fixed billing issues
new 2ef1b7d changed billing scheduler
new 623a31a fixed billing issues
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../java/com/epam/dlab/BillingServiceImpl.java | 2 +-
.../billing/azure/CalculateBillingServiceImpl.java | 2 +-
.../billing/gcp/dao/impl/BigQueryBillingDAO.java | 2 +-
services/self-service/self-service.yml | 2 +-
.../service/impl/BillingServiceImpl.java | 22 ++++----
.../epam/dlab/backendapi/util/BillingUtils.java | 64 ++++++++++++++--------
6 files changed, 54 insertions(+), 40 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org
[incubator-dlab] 03/03: fixed billing issues
Posted by of...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ofuks pushed a commit to branch DLAB-1571
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 623a31a7dda51e14daf28b4074ae70f293bd942e
Author: Oleh Fuks <ol...@gmail.com>
AuthorDate: Tue Apr 14 17:39:56 2020 +0300
fixed billing issues
---
.../epam/dlab/backendapi/util/BillingUtils.java | 50 ++++++++++++++--------
1 file changed, 33 insertions(+), 17 deletions(-)
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java
index 5ac725c..49ad999 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java
@@ -61,6 +61,7 @@ public class BillingUtils {
private static final String VOLUME_SECONDARY_FORMAT = "%s-volume-secondary";
private static final String VOLUME_SECONDARY_COMPUTATIONAL_FORMAT = "%s-%s-volume-secondary";
private static final String IMAGE_STANDARD_FORMAT1 = "%s-%s-%s-%s-notebook-image";
+ private static final String IMAGE_STANDARD_FORMAT2 = "%s-%s-%s-notebook-image";
private static final String IMAGE_CUSTOM_FORMAT = "%s-%s-%s-%s-%s";
private static final String SHARED_RESOURCE = "Shared resource";
@@ -94,29 +95,34 @@ public class BillingUtils {
public static Stream<BillingReportLine> sharedEndpointBillingDataStream(String endpoint, String sbn) {
final String projectEndpointBucketId = String.format(ENDPOINT_SHARED_BUCKET_FORMAT, sbn, endpoint).toLowerCase();
final String endpointId = String.format(ENDPOINT_FORMAT, sbn, endpoint).toLowerCase();
- return Stream.of(
+ return Stream.concat(Stream.of(
BillingReportLine.builder().resourceName("Endpoint shared bucket").user(SHARED_RESOURCE).project(SHARED_RESOURCE).dlabId(projectEndpointBucketId).resourceType(BUCKET).build(),
BillingReportLine.builder().resourceName("Endpoint").user(SHARED_RESOURCE).project(SHARED_RESOURCE).dlabId(endpointId).resourceType(ENDPOINT).build()
- );
+ ),
+ standardImageBillingDataStream(sbn, endpoint));
}
public static Stream<BillingReportLine> exploratoryBillingDataStream(UserInstanceDTO userInstance, Integer maxSparkInstanceCount) {
final Stream<BillingReportLine> computationalStream = userInstance.getResources()
.stream()
.filter(cr -> cr.getComputationalId() != null)
- .flatMap(cr -> Stream.concat(Stream.of(
- withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).dlabId(cr.getComputationalId()).resourceType(COMPUTATIONAL).shape(getComputationalShape(cr))
- .exploratoryName(userInstance.getExploratoryName()).build(),
- withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).dlabId(String.format(VOLUME_PRIMARY_FORMAT, cr.getComputationalId())).resourceType(VOLUME).build(),
- withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).dlabId(String.format(VOLUME_PRIMARY_COMPUTATIONAL_FORMAT, cr.getComputationalId(), "m"))
- .resourceType(VOLUME).build(),
- withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).dlabId(String.format(VOLUME_SECONDARY_COMPUTATIONAL_FORMAT, cr.getComputationalId(), "m"))
- .resourceType(VOLUME).build()
- ),
- getSlaveVolumes(userInstance, cr, maxSparkInstanceCount)
- ));
+ .flatMap(cr -> {
+ final String computationalId = cr.getComputationalId().toLowerCase();
+ return Stream.concat(Stream.of(
+ withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).dlabId(computationalId).resourceType(COMPUTATIONAL).shape(getComputationalShape(cr))
+ .exploratoryName(userInstance.getExploratoryName()).build(),
+ withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).dlabId(String.format(VOLUME_PRIMARY_FORMAT, computationalId)).resourceType(VOLUME).build(),
+ withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).dlabId(String.format(VOLUME_SECONDARY_FORMAT, computationalId)).resourceType(VOLUME).build(),
+ withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).dlabId(String.format(VOLUME_PRIMARY_COMPUTATIONAL_FORMAT, computationalId, "m"))
+ .resourceType(VOLUME).build(),
+ withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).dlabId(String.format(VOLUME_SECONDARY_COMPUTATIONAL_FORMAT, computationalId, "m"))
+ .resourceType(VOLUME).build()
+ ),
+ getSlaveVolumes(userInstance, cr, maxSparkInstanceCount)
+ );
+ });
final String exploratoryName = userInstance.getExploratoryName();
- final String exploratoryId = userInstance.getExploratoryId();
+ final String exploratoryId = userInstance.getExploratoryId().toLowerCase();
final String primaryVolumeId = String.format(VOLUME_PRIMARY_FORMAT, exploratoryId);
final String secondaryVolumeId = String.format(VOLUME_SECONDARY_FORMAT, exploratoryId);
final Stream<BillingReportLine> exploratoryStream = Stream.of(
@@ -137,9 +143,9 @@ public class BillingUtils {
private static Stream<BillingReportLine> getSlaveVolumes(UserInstanceDTO userInstance, UserComputationalResource cr, Integer maxSparkInstanceCount) {
List<BillingReportLine> list = new ArrayList<>();
for (int i = 1; i <= maxSparkInstanceCount; i++) {
- list.add(withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).dlabId(String.format(VOLUME_PRIMARY_COMPUTATIONAL_FORMAT, cr.getComputationalId(), "s" + i))
+ list.add(withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).dlabId(String.format(VOLUME_PRIMARY_COMPUTATIONAL_FORMAT, cr.getComputationalId().toLowerCase(), "s" + i))
.resourceType(VOLUME).build());
- list.add(withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).dlabId(String.format(VOLUME_SECONDARY_COMPUTATIONAL_FORMAT, cr.getComputationalId(), "s" + i))
+ list.add(withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).dlabId(String.format(VOLUME_SECONDARY_COMPUTATIONAL_FORMAT, cr.getComputationalId().toLowerCase(), "s" + i))
.resourceType(VOLUME).build());
}
return list.stream();
@@ -155,7 +161,17 @@ public class BillingUtils {
String.format(DATAENGINE_SERVICE_NAME_FORMAT, resource.getMasterNodeShape(), System.lineSeparator(), null, null);
}
- public static Stream<BillingReportLine> standardImageBillingDataStream(String sbn, String project, String endpoint) {
+ private static Stream<BillingReportLine> standardImageBillingDataStream(String sbn, String endpoint) {
+ List<BillingReportLine> list = new ArrayList<>();
+ for (String notebook : AVAILABLE_NOTEBOOKS) {
+ list.add(BillingReportLine.builder().resourceName(IMAGE_NAME).dlabId(String.format(IMAGE_STANDARD_FORMAT2, sbn, endpoint, notebook).toLowerCase())
+ .project(SHARED_RESOURCE).resourceType(IMAGE).build());
+ }
+
+ return list.stream();
+ }
+
+ private static Stream<BillingReportLine> standardImageBillingDataStream(String sbn, String project, String endpoint) {
List<BillingReportLine> list = new ArrayList<>();
for (String notebook : AVAILABLE_NOTEBOOKS) {
list.add(BillingReportLine.builder().resourceName(IMAGE_NAME).dlabId(String.format(IMAGE_STANDARD_FORMAT1, sbn, project, endpoint, notebook).toLowerCase())
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org
[incubator-dlab] 02/03: changed billing scheduler
Posted by of...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ofuks pushed a commit to branch DLAB-1571
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 2ef1b7d626843890df325b0f0b93bed0c3bbd830
Author: Oleh Fuks <ol...@gmail.com>
AuthorDate: Tue Apr 14 16:28:29 2020 +0300
changed billing scheduler
---
services/self-service/self-service.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/services/self-service/self-service.yml b/services/self-service/self-service.yml
index 4e71316..4d6cad5 100644
--- a/services/self-service/self-service.yml
+++ b/services/self-service/self-service.yml
@@ -152,7 +152,7 @@ schedulers:
cron: "0 */15 * ? * *"
billingScheduler:
enabled: true
- cron: "*/50 * * ? * * *"
+ cron: "0 */15 * ? * *"
guacamole:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org
[incubator-dlab] 01/03: fixed billing issues
Posted by of...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ofuks pushed a commit to branch DLAB-1571
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit fc9e515e7cf67ec0a9a1a4ec702cf76a3425f81a
Author: Oleh Fuks <ol...@gmail.com>
AuthorDate: Tue Apr 14 15:02:17 2020 +0300
fixed billing issues
---
.../java/com/epam/dlab/BillingServiceImpl.java | 2 +-
.../billing/azure/CalculateBillingServiceImpl.java | 2 +-
.../billing/gcp/dao/impl/BigQueryBillingDAO.java | 2 +-
.../service/impl/BillingServiceImpl.java | 22 ++++++++++------------
.../epam/dlab/backendapi/util/BillingUtils.java | 14 +++++++-------
5 files changed, 20 insertions(+), 22 deletions(-)
diff --git a/services/billing-aws/src/main/java/com/epam/dlab/BillingServiceImpl.java b/services/billing-aws/src/main/java/com/epam/dlab/BillingServiceImpl.java
index d474790..8ac6c48 100644
--- a/services/billing-aws/src/main/java/com/epam/dlab/BillingServiceImpl.java
+++ b/services/billing-aws/src/main/java/com/epam/dlab/BillingServiceImpl.java
@@ -76,7 +76,7 @@ public class BillingServiceImpl implements BillingService {
private BillingData toBillingData(Document billingData) {
return BillingData.builder()
- .tag(billingData.getString(FIELD_DLAB_ID))
+ .tag(billingData.getString(FIELD_DLAB_ID).toLowerCase())
.usageDateFrom(Optional.ofNullable(billingData.getString(FIELD_USAGE_DATE)).map(LocalDate::parse).orElse(null))
.usageDateTo(Optional.ofNullable(billingData.getString(FIELD_USAGE_DATE)).map(LocalDate::parse).orElse(null))
.usageDate(billingData.getString(FIELD_USAGE_DATE))
diff --git a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/CalculateBillingServiceImpl.java b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/CalculateBillingServiceImpl.java
index de38aff..3b3d60b 100644
--- a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/CalculateBillingServiceImpl.java
+++ b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/CalculateBillingServiceImpl.java
@@ -233,7 +233,7 @@ public class CalculateBillingServiceImpl implements CalculateBillingService {
private BillingData toBillingData(AzureDailyResourceInvoice billingData) {
return BillingData.builder()
- .tag(billingData.getDlabId())
+ .tag(billingData.getDlabId().toLowerCase())
.usageDateFrom(Optional.ofNullable(billingData.getUsageStartDate()).map(LocalDate::parse).orElse(null))
.usageDateTo(Optional.ofNullable(billingData.getUsageEndDate()).map(LocalDate::parse).orElse(null))
.usageDate(billingData.getDay())
diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/impl/BigQueryBillingDAO.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/impl/BigQueryBillingDAO.java
index 6949d95..061283d 100644
--- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/impl/BigQueryBillingDAO.java
+++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/impl/BigQueryBillingDAO.java
@@ -120,7 +120,7 @@ public class BigQueryBillingDAO implements BillingDAO {
.product(fields.get("product").getStringValue())
.usageType(fields.get("usageType").getStringValue())
.currency(fields.get("currency").getStringValue())
- .tag(fields.get("value").getStringValue())
+ .tag(fields.get("value").getStringValue().toLowerCase())
.usageDate(toLocalDate(fields, "usage_date_from").format((DateTimeFormatter.ofPattern(DATE_FORMAT))))
.build();
}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BillingServiceImpl.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BillingServiceImpl.java
index 3d0ecea..ce88cb9 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BillingServiceImpl.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BillingServiceImpl.java
@@ -164,19 +164,16 @@ public class BillingServiceImpl implements BillingService {
billingDataMap.forEach((endpointDTO, billingData) -> {
log.info("Updating billing information for endpoint {}. Billing data {}", endpointDTO.getName(), billingData);
- updateBillingData(userInfo, endpointDTO, billingData);
+ try {
+ updateBillingData(endpointDTO, billingData);
+ } catch (Exception e) {
+ log.error("Something went wrong while trying to update billing for {}. {}", endpointDTO.getName(), e.getMessage());
+ }
});
}
- private Map<String, BillingReportLine> getBillableResources(UserInfo userInfo) {
- Set<ProjectDTO> projects;
- if (isFullReport(userInfo)) {
- projects = new HashSet<>(projectService.getProjects());
- } else {
- projects = new HashSet<>(projectService.getProjects(userInfo));
- projects.addAll(projectService.getUserProjects(userInfo, false));
- }
-
+ private Map<String, BillingReportLine> getBillableResources() {
+ Set<ProjectDTO> projects = new HashSet<>(projectService.getProjects());
final Stream<BillingReportLine> ssnBillingDataStream = BillingUtils.ssnBillingDataStream(sbn);
final Stream<BillingReportLine> billableEdges = projects
.stream()
@@ -211,10 +208,10 @@ public class BillingServiceImpl implements BillingService {
.flatMap(endpoint -> BillingUtils.edgeBillingDataStream(projectName, serviceBaseName, endpoint.getName()));
}
- private void updateBillingData(UserInfo userInfo, EndpointDTO endpointDTO, List<BillingData> billingData) {
+ private void updateBillingData(EndpointDTO endpointDTO, List<BillingData> billingData) {
final String endpointName = endpointDTO.getName();
final CloudProvider cloudProvider = endpointDTO.getCloudProvider();
- final Map<String, BillingReportLine> billableResources = getBillableResources(userInfo);
+ final Map<String, BillingReportLine> billableResources = getBillableResources();
final Stream<BillingReportLine> billingReportLineStream = billingData
.stream()
.peek(bd -> bd.setApplication(endpointName))
@@ -333,6 +330,7 @@ public class BillingServiceImpl implements BillingService {
.resourceType(billingReportLine.getResourceType())
.resourceName(billingReportLine.getResourceName())
.shape(billingReportLine.getShape())
+ .exploratoryName(billingReportLine.getExploratoryName())
.build();
}
}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java
index da59725..5ac725c 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java
@@ -70,9 +70,9 @@ public class BillingUtils {
private static final String DATAENGINE_SERVICE_NAME_FORMAT = "Master: %s%sSlave: %d x %s";
public static Stream<BillingReportLine> edgeBillingDataStream(String project, String sbn, String endpoint) {
- final String userEdgeId = String.format(EDGE_FORMAT, sbn, project.toLowerCase(), endpoint);
- final String edgeVolumeId = String.format(EDGE_VOLUME_FORMAT, sbn, project.toLowerCase(), endpoint);
- final String endpointBucketId = String.format(PROJECT_ENDPOINT_BUCKET_FORMAT, sbn, project.toLowerCase(), endpoint);
+ final String userEdgeId = String.format(EDGE_FORMAT, sbn, project, endpoint).toLowerCase();
+ final String edgeVolumeId = String.format(EDGE_VOLUME_FORMAT, sbn, project, endpoint).toLowerCase();
+ final String endpointBucketId = String.format(PROJECT_ENDPOINT_BUCKET_FORMAT, sbn, project, endpoint).toLowerCase();
return Stream.concat(Stream.of(
BillingReportLine.builder().resourceName(endpoint).user(SHARED_RESOURCE).project(project).dlabId(userEdgeId).resourceType(EDGE).build(),
@@ -92,8 +92,8 @@ public class BillingUtils {
}
public static Stream<BillingReportLine> sharedEndpointBillingDataStream(String endpoint, String sbn) {
- final String projectEndpointBucketId = String.format(ENDPOINT_SHARED_BUCKET_FORMAT, sbn, endpoint.toLowerCase());
- final String endpointId = String.format(ENDPOINT_FORMAT, sbn, endpoint.toLowerCase());
+ final String projectEndpointBucketId = String.format(ENDPOINT_SHARED_BUCKET_FORMAT, sbn, endpoint).toLowerCase();
+ final String endpointId = String.format(ENDPOINT_FORMAT, sbn, endpoint).toLowerCase();
return Stream.of(
BillingReportLine.builder().resourceName("Endpoint shared bucket").user(SHARED_RESOURCE).project(SHARED_RESOURCE).dlabId(projectEndpointBucketId).resourceType(BUCKET).build(),
BillingReportLine.builder().resourceName("Endpoint").user(SHARED_RESOURCE).project(SHARED_RESOURCE).dlabId(endpointId).resourceType(ENDPOINT).build()
@@ -128,7 +128,7 @@ public class BillingUtils {
}
public static Stream<BillingReportLine> customImageBillingDataStream(ImageInfoRecord image, String sbn) {
- String imageId = String.format(IMAGE_CUSTOM_FORMAT, sbn, image.getProject(), image.getEndpoint(), image.getApplication(), image.getName());
+ String imageId = String.format(IMAGE_CUSTOM_FORMAT, sbn, image.getProject(), image.getEndpoint(), image.getApplication(), image.getName()).toLowerCase();
return Stream.of(
BillingReportLine.builder().resourceName(IMAGE_NAME).project(image.getProject()).dlabId(imageId).resourceType(IMAGE).build()
);
@@ -158,7 +158,7 @@ public class BillingUtils {
public static Stream<BillingReportLine> standardImageBillingDataStream(String sbn, String project, String endpoint) {
List<BillingReportLine> list = new ArrayList<>();
for (String notebook : AVAILABLE_NOTEBOOKS) {
- list.add(BillingReportLine.builder().resourceName(IMAGE_NAME).dlabId(String.format(IMAGE_STANDARD_FORMAT1, sbn, project, endpoint, notebook))
+ list.add(BillingReportLine.builder().resourceName(IMAGE_NAME).dlabId(String.format(IMAGE_STANDARD_FORMAT1, sbn, project, endpoint, notebook).toLowerCase())
.project(SHARED_RESOURCE).resourceType(IMAGE).build());
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org