You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by bh...@apache.org on 2019/11/12 10:16:48 UTC
[incubator-dlab] branch develop updated (4dd9fae -> 91332dd)
This is an automated email from the ASF dual-hosted git repository.
bhliva pushed a change to branch develop
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git.
from 4dd9fae Merge pull request #394 from apache/DLAB-1242
new 4e96d9c [DLAB-1230] Fixed billing issues [Azure]
new b945660 [DLAB-1230] Fixed billing issues with edges [Azure]
new adcaa4b Code review
new 91332dd Code review
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../azure/AzureBillableResourcesService.java | 88 ++++++++++++----------
.../azure/model/AzureDailyResourceInvoice.java | 3 +
.../azure/model/AzureDlabBillableResource.java | 1 +
3 files changed, 53 insertions(+), 39 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org
[incubator-dlab] 01/04: [DLAB-1230] Fixed billing issues [Azure]
Posted by bh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bhliva pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 4e96d9c2ed168ca80373cce0fdc32865bb9d0c21
Author: Oleh Fuks <ol...@gmail.com>
AuthorDate: Fri Nov 8 17:01:25 2019 +0200
[DLAB-1230] Fixed billing issues [Azure]
---
.../azure/AzureBillableResourcesService.java | 39 ++++++++++++----------
.../azure/model/AzureDailyResourceInvoice.java | 3 ++
.../azure/model/AzureDlabBillableResource.java | 1 +
3 files changed, 26 insertions(+), 17 deletions(-)
diff --git a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureBillableResourcesService.java b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureBillableResourcesService.java
index 24cb3be..7b5fd61 100644
--- a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureBillableResourcesService.java
+++ b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureBillableResourcesService.java
@@ -48,6 +48,7 @@ import static com.mongodb.client.model.Projections.fields;
*/
@Slf4j
public class AzureBillableResourcesService {
+ private static final String SHARED_RESOURCE = "Shared resource";
private static final String[] USER_INSTANCES_EXCLUDED_FIELDS = {"scheduler_data", "last_activity",
"computational_resources.scheduler_data", "computational_resources.last_activity"};
private final ObjectMapper objectMapper = new ObjectMapper();
@@ -173,11 +174,8 @@ public class AzureBillableResourcesService {
new com.fasterxml.jackson.core.type.TypeReference<List<EdgeInfoAzure>>() {
});
- if (edgeInfoList != null && !edgeInfoList.isEmpty()) {
- for (EdgeInfoAzure edgeInfoAzure : edgeInfoList) {
- billableResources.addAll(getEdgeAndStorageAccount(edgeInfoAzure));
- }
- }
+ Optional.ofNullable(edgeInfoList).ifPresent(e -> e.stream().filter(Objects::nonNull).forEach(
+ edgeInfoAzure -> billableResources.addAll(getEdgeAndStorageAccount(edgeInfoAzure))));
return billableResources;
} catch (IOException e) {
@@ -194,28 +192,27 @@ public class AzureBillableResourcesService {
billableResources.add(AzureDlabBillableResource.builder()
.id(edgeInfoAzure.getUserStorageAccountTagName())
.type(DlabResourceType.EDGE_STORAGE_ACCOUNT)
- .user(edgeInfoAzure.getId()).build());
+ .user(SHARED_RESOURCE)
+ .build());
}
if (StringUtils.isNotEmpty(edgeInfoAzure.getInstanceId())) {
billableResources.add(AzureDlabBillableResource.builder()
.id(edgeInfoAzure.getInstanceId())
.type(DlabResourceType.EDGE)
- .user(edgeInfoAzure.getId()).build());
+ .user(SHARED_RESOURCE)
+ .build());
billableResources.add(AzureDlabBillableResource.builder()
- .id(serviceBaseName + "-" + edgeUserSimpleName(edgeInfoAzure) + "-edge-volume-primary")
+ .id(edgeInfoAzure.getInstanceId() + "-volume-primary")
.type(DlabResourceType.VOLUME)
- .user(edgeInfoAzure.getId()).build());
+ .user(SHARED_RESOURCE)
+ .build());
}
return billableResources;
}
- private String edgeUserSimpleName(EdgeInfoAzure edgeInfoAzure) {
- return edgeInfoAzure.getId().replaceAll("@.*", "");
- }
-
private Set<AzureDlabBillableResource> getNotebooksAndClusters() {
Set<AzureDlabBillableResource> billableResources = new HashSet<>();
@@ -249,8 +246,10 @@ public class AzureBillableResourcesService {
.id(userInstanceDTO.getExploratoryId())
.type(DlabResourceType.EXPLORATORY)
.user(userInstanceDTO.getUser())
+ .project(userInstanceDTO.getProject())
.notebookId(userInstanceDTO.getExploratoryId())
- .resourceName(userInstanceDTO.getExploratoryName()).build());
+ .resourceName(userInstanceDTO.getExploratoryName())
+ .build());
notebookResources.addAll(getVolumes(userInstanceDTO, userInstanceDTO.getExploratoryId(), "Volume primary",
"Volume secondary"));
@@ -262,8 +261,10 @@ public class AzureBillableResourcesService {
.id(userComputationalResource.getComputationalId())
.type(DlabResourceType.COMPUTATIONAL)
.user(userInstanceDTO.getUser())
+ .project(userInstanceDTO.getProject())
.notebookId(userInstanceDTO.getExploratoryId())
- .resourceName(userComputationalResource.getComputationalName()).build());
+ .resourceName(userComputationalResource.getComputationalName())
+ .build());
final List<AzureDlabBillableResource> volumes = getVolumes(userInstanceDTO,
userComputationalResource.getComputationalId(),
userComputationalResource.getComputationalName() + " volume primary",
@@ -292,14 +293,18 @@ public class AzureBillableResourcesService {
.id(exploratoryId + "-volume-primary")
.type(DlabResourceType.VOLUME)
.user(userInstanceDTO.getUser())
+ .project(userInstanceDTO.getProject())
.notebookId(userInstanceDTO.getExploratoryId())
- .resourceName(primaryVolumeName).build(),
+ .resourceName(primaryVolumeName)
+ .build(),
AzureDlabBillableResource.builder()
.id(exploratoryId + "-volume-secondary")
.type(DlabResourceType.VOLUME)
.user(userInstanceDTO.getUser())
+ .project(userInstanceDTO.getProject())
.notebookId(userInstanceDTO.getExploratoryId())
- .resourceName(secondaryVolumeName).build()
+ .resourceName(secondaryVolumeName)
+ .build()
);
}
}
diff --git a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/model/AzureDailyResourceInvoice.java b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/model/AzureDailyResourceInvoice.java
index 8d45587..ff132a2 100644
--- a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/model/AzureDailyResourceInvoice.java
+++ b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/model/AzureDailyResourceInvoice.java
@@ -38,6 +38,8 @@ public class AzureDailyResourceInvoice extends MongoDocument<AzureDailyResourceI
@JsonProperty
private String user;
@JsonProperty
+ private String project;
+ @JsonProperty
private String exploratoryId;
@JsonProperty
private String computationalId;
@@ -62,6 +64,7 @@ public class AzureDailyResourceInvoice extends MongoDocument<AzureDailyResourceI
public AzureDailyResourceInvoice(AzureDlabBillableResource azureDlabBillableResource) {
this.dlabId = azureDlabBillableResource.getId();
this.user = azureDlabBillableResource.getUser();
+ this.project = azureDlabBillableResource.getProject();
this.resourceType = azureDlabBillableResource.getType();
this.resourceName = azureDlabBillableResource.getResourceName();
diff --git a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/model/AzureDlabBillableResource.java b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/model/AzureDlabBillableResource.java
index 9c04d40..639788b 100644
--- a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/model/AzureDlabBillableResource.java
+++ b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/model/AzureDlabBillableResource.java
@@ -33,6 +33,7 @@ public class AzureDlabBillableResource {
private String id;
private DlabResourceType type;
private String user;
+ private String project;
private String notebookId;
private String resourceName;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org
[incubator-dlab] 02/04: [DLAB-1230] Fixed billing issues with edges
[Azure]
Posted by bh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bhliva pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit b94566073a4ffe95657b4983327aea2e72bc6d8e
Author: Oleh Fuks <ol...@gmail.com>
AuthorDate: Mon Nov 11 13:30:35 2019 +0200
[DLAB-1230] Fixed billing issues with edges [Azure]
---
.../azure/AzureBillableResourcesService.java | 40 ++++++++++------------
1 file changed, 19 insertions(+), 21 deletions(-)
diff --git a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureBillableResourcesService.java b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureBillableResourcesService.java
index 7b5fd61..fa3a518 100644
--- a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureBillableResourcesService.java
+++ b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureBillableResourcesService.java
@@ -161,31 +161,26 @@ public class AzureBillableResourcesService {
private Set<AzureDlabBillableResource> getEdgeAndStorageAccount() {
Set<AzureDlabBillableResource> billableResources = new HashSet<>();
- try {
-
- final FindIterable<Document> prjDocuments = mongoDbBillingClient.getDatabase()
- .getCollection("Projects").find();
- final List<Document> edges = StreamSupport.stream(prjDocuments.spliterator(), false)
- .flatMap(d -> ((List<Document>) d.get("endpoints")).stream())
- .map(d -> (Document) d.get("edgeInfo"))
- .collect(Collectors.toList());
- List<EdgeInfoAzure> edgeInfoList = objectMapper.readValue(
- objectMapper.writeValueAsString(edges),
- new com.fasterxml.jackson.core.type.TypeReference<List<EdgeInfoAzure>>() {
- });
-
- Optional.ofNullable(edgeInfoList).ifPresent(e -> e.stream().filter(Objects::nonNull).forEach(
- edgeInfoAzure -> billableResources.addAll(getEdgeAndStorageAccount(edgeInfoAzure))));
+ Map<String, List<Document>> projectEndpoints = StreamSupport.stream(mongoDbBillingClient.getDatabase()
+ .getCollection("Projects").find().spliterator(), false)
+ .collect(Collectors.toMap(key -> key.getString("name").toLowerCase(),
+ value -> (List<Document>) value.get("endpoints")));
+
+ projectEndpoints.forEach((key, value) -> value.forEach(endpoint -> {
+ try {
+ billableResources.addAll(getEdgeAndStorageAccount(key, objectMapper.readValue(
+ objectMapper.writeValueAsString(endpoint.get("edgeInfo")),
+ new com.fasterxml.jackson.core.type.TypeReference<EdgeInfoAzure>() {
+ })));
+ } catch (IOException ex) {
+ log.error("Error during preparation of billable resources", ex);
+ }
+ }));
- return billableResources;
- } catch (IOException e) {
- log.error("Error during preparation of billable resources", e);
- }
return billableResources;
}
- private Set<AzureDlabBillableResource> getEdgeAndStorageAccount(EdgeInfoAzure edgeInfoAzure) {
-
+ private Set<AzureDlabBillableResource> getEdgeAndStorageAccount(String projectName, EdgeInfoAzure edgeInfoAzure) {
Set<AzureDlabBillableResource> billableResources = new HashSet<>();
if (StringUtils.isNotEmpty(edgeInfoAzure.getUserContainerName())) {
@@ -193,6 +188,7 @@ public class AzureBillableResourcesService {
.id(edgeInfoAzure.getUserStorageAccountTagName())
.type(DlabResourceType.EDGE_STORAGE_ACCOUNT)
.user(SHARED_RESOURCE)
+ .project(projectName)
.build());
}
@@ -201,12 +197,14 @@ public class AzureBillableResourcesService {
.id(edgeInfoAzure.getInstanceId())
.type(DlabResourceType.EDGE)
.user(SHARED_RESOURCE)
+ .project(projectName)
.build());
billableResources.add(AzureDlabBillableResource.builder()
.id(edgeInfoAzure.getInstanceId() + "-volume-primary")
.type(DlabResourceType.VOLUME)
.user(SHARED_RESOURCE)
+ .project(projectName)
.build());
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org
[incubator-dlab] 03/04: Code review
Posted by bh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bhliva pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit adcaa4b3b29913114f598361f0190f6726534c26
Author: Oleh Fuks <ol...@gmail.com>
AuthorDate: Mon Nov 11 17:09:41 2019 +0200
Code review
---
.../azure/AzureBillableResourcesService.java | 29 ++++++++++++----------
1 file changed, 16 insertions(+), 13 deletions(-)
diff --git a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureBillableResourcesService.java b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureBillableResourcesService.java
index fa3a518..ae9ff1c 100644
--- a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureBillableResourcesService.java
+++ b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureBillableResourcesService.java
@@ -159,25 +159,28 @@ public class AzureBillableResourcesService {
}
private Set<AzureDlabBillableResource> getEdgeAndStorageAccount() {
- Set<AzureDlabBillableResource> billableResources = new HashSet<>();
-
Map<String, List<Document>> projectEndpoints = StreamSupport.stream(mongoDbBillingClient.getDatabase()
.getCollection("Projects").find().spliterator(), false)
.collect(Collectors.toMap(key -> key.getString("name").toLowerCase(),
value -> (List<Document>) value.get("endpoints")));
- projectEndpoints.forEach((key, value) -> value.forEach(endpoint -> {
- try {
- billableResources.addAll(getEdgeAndStorageAccount(key, objectMapper.readValue(
- objectMapper.writeValueAsString(endpoint.get("edgeInfo")),
- new com.fasterxml.jackson.core.type.TypeReference<EdgeInfoAzure>() {
- })));
- } catch (IOException ex) {
- log.error("Error during preparation of billable resources", ex);
- }
- }));
+ Set<AzureDlabBillableResource> edgeInfo = projectEndpoints.entrySet()
+ .stream()
+ .flatMap(e -> e.getValue().stream().map(endpoint -> {
+ try {
+ return getEdgeAndStorageAccount(e.getKey(), objectMapper.readValue(objectMapper.writeValueAsString(endpoint.get("edgeInfo")),
+ new com.fasterxml.jackson.core.type.TypeReference<EdgeInfoAzure>() {
+ }));
+ } catch (IOException ex) {
+ ex.printStackTrace();
+ }
+ return null;
+ }))
+ .filter(Objects::nonNull)
+ .flatMap(Set::stream)
+ .collect(Collectors.toSet());
- return billableResources;
+ return new HashSet<>(edgeInfo);
}
private Set<AzureDlabBillableResource> getEdgeAndStorageAccount(String projectName, EdgeInfoAzure edgeInfoAzure) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org
[incubator-dlab] 04/04: Code review
Posted by bh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bhliva pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 91332dd9d50b5dbf56c25d3d9ff7da0dc18f7d6e
Author: Oleh Fuks <ol...@gmail.com>
AuthorDate: Mon Nov 11 17:58:24 2019 +0200
Code review
---
.../azure/AzureBillableResourcesService.java | 30 ++++++++++++----------
1 file changed, 17 insertions(+), 13 deletions(-)
diff --git a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureBillableResourcesService.java b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureBillableResourcesService.java
index ae9ff1c..c4ee5b4 100644
--- a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureBillableResourcesService.java
+++ b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureBillableResourcesService.java
@@ -37,6 +37,7 @@ import org.bson.Document;
import java.io.IOException;
import java.util.*;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import static com.mongodb.client.model.Projections.exclude;
@@ -164,23 +165,26 @@ public class AzureBillableResourcesService {
.collect(Collectors.toMap(key -> key.getString("name").toLowerCase(),
value -> (List<Document>) value.get("endpoints")));
- Set<AzureDlabBillableResource> edgeInfo = projectEndpoints.entrySet()
+ return projectEndpoints.entrySet()
.stream()
- .flatMap(e -> e.getValue().stream().map(endpoint -> {
+ .flatMap(projectEndpoint -> getEdgeAndStoragePerProject(projectEndpoint.getKey(), projectEndpoint.getValue()))
+ .collect(Collectors.toSet());
+ }
+
+ private Stream<AzureDlabBillableResource> getEdgeAndStoragePerProject(String projectName, List<Document> endpoints) {
+ return endpoints
+ .stream()
+ .flatMap(endpoint -> {
try {
- return getEdgeAndStorageAccount(e.getKey(), objectMapper.readValue(objectMapper.writeValueAsString(endpoint.get("edgeInfo")),
+ return getEdgeAndStorageAccount(projectName, objectMapper.readValue(
+ objectMapper.writeValueAsString(endpoint.get("edgeInfo")),
new com.fasterxml.jackson.core.type.TypeReference<EdgeInfoAzure>() {
- }));
- } catch (IOException ex) {
- ex.printStackTrace();
+ })).stream();
+ } catch (IOException e) {
+ log.error("Error during preparation of billable resources", e);
}
- return null;
- }))
- .filter(Objects::nonNull)
- .flatMap(Set::stream)
- .collect(Collectors.toSet());
-
- return new HashSet<>(edgeInfo);
+ return Stream.empty();
+ });
}
private Set<AzureDlabBillableResource> getEdgeAndStorageAccount(String projectName, EdgeInfoAzure edgeInfoAzure) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org