You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by of...@apache.org on 2020/03/17 16:38:22 UTC
[incubator-dlab] 01/02: Added more changes to Azure remote billing
This is an automated email from the ASF dual-hosted git repository.
ofuks pushed a commit to branch DLAB-1571
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 3ba048723faf6bb0f3529ddda3dfc0526a54c98b
Author: Oleh Fuks <ol...@gmail.com>
AuthorDate: Tue Mar 17 15:07:09 2020 +0200
Added more changes to Azure remote billing
---
.../azure/AzureBillableResourcesService.java | 315 ---------------------
.../billing/azure/AzureBillingDetailsService.java | 194 -------------
.../azure/AzureInvoiceCalculationService.java | 60 ++--
.../dlab/billing/azure/BillingSchedulerAzure.java | 26 +-
.../azure/model/AzureDailyResourceInvoice.java | 34 +--
5 files changed, 23 insertions(+), 606 deletions(-)
diff --git a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureBillableResourcesService.java b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureBillableResourcesService.java
deleted file mode 100644
index c4ee5b4..0000000
--- a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureBillableResourcesService.java
+++ /dev/null
@@ -1,315 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.epam.dlab.billing.azure;
-
-import com.epam.dlab.MongoKeyWords;
-import com.epam.dlab.billing.DlabResourceType;
-import com.epam.dlab.billing.azure.model.AzureDlabBillableResource;
-import com.epam.dlab.dto.UserInstanceDTO;
-import com.epam.dlab.dto.azure.edge.EdgeInfoAzure;
-import com.epam.dlab.dto.computational.UserComputationalResource;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.collect.Sets;
-import com.mongodb.client.FindIterable;
-import com.mongodb.client.model.Filters;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
-import org.bson.Document;
-
-import java.io.IOException;
-import java.util.*;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-import java.util.stream.StreamSupport;
-
-import static com.mongodb.client.model.Projections.exclude;
-import static com.mongodb.client.model.Projections.fields;
-
-/**
- * Helps to retrieve billable resources that are created in scope of DLab usage. Uses MongoDB as data source
- * for created resources
- */
-@Slf4j
-public class AzureBillableResourcesService {
- private static final String SHARED_RESOURCE = "Shared resource";
- private static final String[] USER_INSTANCES_EXCLUDED_FIELDS = {"scheduler_data", "last_activity",
- "computational_resources.scheduler_data", "computational_resources.last_activity"};
- private final ObjectMapper objectMapper = new ObjectMapper();
-
- private MongoDbBillingClient mongoDbBillingClient;
- private String serviceBaseName;
- private String sharedStorageAccountTagName;
- private String ssnStorageAccountTagName;
- private String azureDataLakeTagName;
-
- /**
- * Constructs the service class
- *
- * @param mongoDbBillingClient mongodb client to retrieve all billable resources
- * @param sharedStorageAccountTagName shared storage account tag name
- * @param ssnStorageAccountTagName ssn storage account tag name
- * @param azureDataLakeTagName azure DataLake tag name
- */
- public AzureBillableResourcesService(MongoDbBillingClient mongoDbBillingClient, String sharedStorageAccountTagName,
- String ssnStorageAccountTagName, String azureDataLakeTagName) {
- this.mongoDbBillingClient = mongoDbBillingClient;
-
- this.serviceBaseName = getConfigurationSettingValue(MongoKeyWords.SERVICE_BASE_NAME_KEY)
- .replace('_', '-').toLowerCase();
-
- this.sharedStorageAccountTagName = sharedStorageAccountTagName;
- this.ssnStorageAccountTagName = ssnStorageAccountTagName;
- this.azureDataLakeTagName = azureDataLakeTagName;
- }
-
-
- /**
- * Collects billable resources
- *
- * @return set of all billable resources that were created in scope by DLab from its installation to current time
- */
- public Set<AzureDlabBillableResource> getBillableResources() {
-
- Set<AzureDlabBillableResource> billableResources = new HashSet<>();
-
- billableResources.addAll(getSsn());
- billableResources.addAll(getDataLake());
- billableResources.addAll(getEdgeAndStorageAccount());
- billableResources.addAll(getNotebooksAndClusters());
-
- List<AzureDlabBillableResource> list = new ArrayList<>(billableResources);
- list.sort(Comparator.comparing(AzureDlabBillableResource::getId));
-
- try {
- log.debug("Billable resources is \n {}", objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString
- (list));
- } catch (JsonProcessingException e) {
- log.debug("Error during pretty printing. Show simple list", e);
- log.debug("Billable resources is {}", list);
- }
-
- return billableResources;
- }
-
- private String getConfigurationSettingValue(String key) {
-
- Document document = mongoDbBillingClient.getDatabase().getCollection(MongoKeyWords.SETTINGS_COLLECTION)
- .find(Filters.eq(MongoKeyWords.MONGO_ID, key)).first();
-
- if (document != null) {
- String value = document.getString("value");
- if (StringUtils.isEmpty(value)) {
- throw new IllegalStateException("Configuration " + key + " does not have value in settings");
- }
- log.info("Key {} has value {}", key, value);
- return value;
- } else {
- throw new IllegalStateException("Configuration " + key + " is not present in settings");
- }
-
- }
-
- private String getConfigurationSettingValueOrEmpty(String key) {
- try {
- return getConfigurationSettingValue(key);
- } catch (IllegalStateException e) {
- log.warn("key {} is not found", key, e);
- return null;
- }
- }
-
- private Set<AzureDlabBillableResource> getSsn() {
-
- return Sets.newHashSet(
- AzureDlabBillableResource.builder().id(serviceBaseName + "-ssn").type(DlabResourceType.SSN).build(),
- AzureDlabBillableResource.builder().id(ssnStorageAccountTagName).type(DlabResourceType
- .SSN_STORAGE_ACCOUNT).build(),
- AzureDlabBillableResource.builder().id(sharedStorageAccountTagName).type(DlabResourceType
- .COLLABORATION_STORAGE_ACCOUNT).build(),
- AzureDlabBillableResource.builder().id(serviceBaseName + "-ssn-volume-primary")
- .type(DlabResourceType.VOLUME).build()
- );
- }
-
- private Set<AzureDlabBillableResource> getDataLake() {
-
- if (azureDataLakeTagName != null) {
- return Sets.newHashSet(AzureDlabBillableResource.builder().id(azureDataLakeTagName)
- .type(DlabResourceType.DATA_LAKE_STORE).build());
- }
-
- return Sets.newHashSet();
- }
-
- private Set<AzureDlabBillableResource> getEdgeAndStorageAccount() {
- Map<String, List<Document>> projectEndpoints = StreamSupport.stream(mongoDbBillingClient.getDatabase()
- .getCollection("Projects").find().spliterator(), false)
- .collect(Collectors.toMap(key -> key.getString("name").toLowerCase(),
- value -> (List<Document>) value.get("endpoints")));
-
- return projectEndpoints.entrySet()
- .stream()
- .flatMap(projectEndpoint -> getEdgeAndStoragePerProject(projectEndpoint.getKey(), projectEndpoint.getValue()))
- .collect(Collectors.toSet());
- }
-
- private Stream<AzureDlabBillableResource> getEdgeAndStoragePerProject(String projectName, List<Document> endpoints) {
- return endpoints
- .stream()
- .flatMap(endpoint -> {
- try {
- return getEdgeAndStorageAccount(projectName, objectMapper.readValue(
- objectMapper.writeValueAsString(endpoint.get("edgeInfo")),
- new com.fasterxml.jackson.core.type.TypeReference<EdgeInfoAzure>() {
- })).stream();
- } catch (IOException e) {
- log.error("Error during preparation of billable resources", e);
- }
- return Stream.empty();
- });
- }
-
- private Set<AzureDlabBillableResource> getEdgeAndStorageAccount(String projectName, EdgeInfoAzure edgeInfoAzure) {
- Set<AzureDlabBillableResource> billableResources = new HashSet<>();
-
- if (StringUtils.isNotEmpty(edgeInfoAzure.getUserContainerName())) {
- billableResources.add(AzureDlabBillableResource.builder()
- .id(edgeInfoAzure.getUserStorageAccountTagName())
- .type(DlabResourceType.EDGE_STORAGE_ACCOUNT)
- .user(SHARED_RESOURCE)
- .project(projectName)
- .build());
- }
-
- if (StringUtils.isNotEmpty(edgeInfoAzure.getInstanceId())) {
- billableResources.add(AzureDlabBillableResource.builder()
- .id(edgeInfoAzure.getInstanceId())
- .type(DlabResourceType.EDGE)
- .user(SHARED_RESOURCE)
- .project(projectName)
- .build());
-
- billableResources.add(AzureDlabBillableResource.builder()
- .id(edgeInfoAzure.getInstanceId() + "-volume-primary")
- .type(DlabResourceType.VOLUME)
- .user(SHARED_RESOURCE)
- .project(projectName)
- .build());
- }
-
- return billableResources;
- }
-
- private Set<AzureDlabBillableResource> getNotebooksAndClusters() {
-
- Set<AzureDlabBillableResource> billableResources = new HashSet<>();
-
- try {
- final FindIterable<Document> userInstanceDocuments = mongoDbBillingClient.getDatabase()
- .getCollection(MongoKeyWords.NOTEBOOK_COLLECTION)
- .find()
- .projection(fields(exclude(USER_INSTANCES_EXCLUDED_FIELDS)));
- List<UserInstanceDTO> userInstanceDTOS = objectMapper.readValue(
- objectMapper.writeValueAsString(userInstanceDocuments),
- new com.fasterxml.jackson.core.type.TypeReference<List<UserInstanceDTO>>() {
- });
-
- if (userInstanceDTOS != null && !userInstanceDTOS.isEmpty()) {
- userInstanceDTOS.forEach(e -> billableResources.addAll(getNotebookAndClusters(e)));
- }
-
- } catch (IOException e) {
- log.error("Error during preparation of billable resources", e);
- }
-
- return billableResources;
- }
-
- private Set<AzureDlabBillableResource> getNotebookAndClusters(UserInstanceDTO userInstanceDTO) {
- Set<AzureDlabBillableResource> notebookResources = new HashSet<>();
-
- if (StringUtils.isNotEmpty(userInstanceDTO.getExploratoryId())) {
- notebookResources.add(AzureDlabBillableResource.builder()
- .id(userInstanceDTO.getExploratoryId())
- .type(DlabResourceType.EXPLORATORY)
- .user(userInstanceDTO.getUser())
- .project(userInstanceDTO.getProject())
- .notebookId(userInstanceDTO.getExploratoryId())
- .resourceName(userInstanceDTO.getExploratoryName())
- .build());
- notebookResources.addAll(getVolumes(userInstanceDTO, userInstanceDTO.getExploratoryId(), "Volume primary",
- "Volume secondary"));
-
- if (userInstanceDTO.getResources() != null && !userInstanceDTO.getResources().isEmpty()) {
- for (UserComputationalResource userComputationalResource : userInstanceDTO.getResources()) {
- if (StringUtils.isNotEmpty(userComputationalResource.getComputationalId())) {
-
- notebookResources.add(AzureDlabBillableResource.builder()
- .id(userComputationalResource.getComputationalId())
- .type(DlabResourceType.COMPUTATIONAL)
- .user(userInstanceDTO.getUser())
- .project(userInstanceDTO.getProject())
- .notebookId(userInstanceDTO.getExploratoryId())
- .resourceName(userComputationalResource.getComputationalName())
- .build());
- final List<AzureDlabBillableResource> volumes = getVolumes(userInstanceDTO,
- userComputationalResource.getComputationalId(),
- userComputationalResource.getComputationalName() + " volume primary",
- userComputationalResource.getComputationalName() + " volume secondary");
- notebookResources.addAll(volumes);
-
- } else {
- log.error("Computational with empty id {} is found in notebook {}. Skip it.",
- userComputationalResource, userInstanceDTO);
- }
- }
- }
-
- } else {
- log.error("Notebook {} with empty id id found. Skip it.", userInstanceDTO);
- }
-
- return notebookResources;
- }
-
- private List<AzureDlabBillableResource> getVolumes(UserInstanceDTO userInstanceDTO, String exploratoryId, String
- primaryVolumeName, String secondaryVolumeName) {
-
- return Arrays.asList(
- AzureDlabBillableResource.builder()
- .id(exploratoryId + "-volume-primary")
- .type(DlabResourceType.VOLUME)
- .user(userInstanceDTO.getUser())
- .project(userInstanceDTO.getProject())
- .notebookId(userInstanceDTO.getExploratoryId())
- .resourceName(primaryVolumeName)
- .build(),
- AzureDlabBillableResource.builder()
- .id(exploratoryId + "-volume-secondary")
- .type(DlabResourceType.VOLUME)
- .user(userInstanceDTO.getUser())
- .project(userInstanceDTO.getProject())
- .notebookId(userInstanceDTO.getExploratoryId())
- .resourceName(secondaryVolumeName)
- .build()
- );
- }
-}
diff --git a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureBillingDetailsService.java b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureBillingDetailsService.java
deleted file mode 100644
index 6d84eb8..0000000
--- a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureBillingDetailsService.java
+++ /dev/null
@@ -1,194 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.epam.dlab.billing.azure;
-
-import com.epam.dlab.MongoKeyWords;
-import com.epam.dlab.billing.BillingCalculationUtils;
-import com.epam.dlab.billing.DlabResourceType;
-import com.google.common.collect.Lists;
-import com.mongodb.client.AggregateIterable;
-import com.mongodb.client.model.*;
-import com.mongodb.client.result.UpdateResult;
-import lombok.extern.slf4j.Slf4j;
-import org.bson.Document;
-import org.bson.conversions.Bson;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.function.Consumer;
-import java.util.stream.StreamSupport;
-
-@Slf4j
-public class AzureBillingDetailsService {
- private MongoDbBillingClient mongoDbBillingClient;
- private String currencyCode;
-
- public AzureBillingDetailsService(MongoDbBillingClient mongoDbBillingClient, String currencyCode) {
- this.mongoDbBillingClient = mongoDbBillingClient;
- this.currencyCode = currencyCode;
- }
-
- public void updateBillingDetails() {
- StreamSupport.stream(mongoDbBillingClient.getDatabase()
- .getCollection(MongoKeyWords.NOTEBOOK_COLLECTION).find().spliterator(), false)
- .map(a -> a.getString(MongoKeyWords.DLAB_USER))
- .forEach(this::updateBillingDetails);
- }
-
- public void updateBillingDetails(String user) {
- log.debug("Updating billing details for user {}", user);
-
- try {
- AggregateIterable<Document> aggregateIterable = mongoDbBillingClient.getDatabase()
- .getCollection(MongoKeyWords.BILLING_DETAILS)
- .aggregate(Lists.newArrayList(
- Aggregates.match(
- Filters.and(
- Filters.eq(MongoKeyWords.DLAB_USER, user),
- Filters.in(MongoKeyWords.RESOURCE_TYPE,
- DlabResourceType.EXPLORATORY.toString(),
- DlabResourceType.COMPUTATIONAL.toString(),
- DlabResourceType.VOLUME.toString())
- )
- ),
-
- Aggregates.group(getGroupingFields(
- MongoKeyWords.DLAB_ID,
- MongoKeyWords.DLAB_USER,
- MongoKeyWords.EXPLORATORY_ID,
- MongoKeyWords.RESOURCE_TYPE,
- MongoKeyWords.RESOURCE_NAME,
- MongoKeyWords.COMPUTATIONAL_ID,
- MongoKeyWords.METER_CATEGORY),
- Accumulators.sum(MongoKeyWords.COST, MongoKeyWords.prepend$(MongoKeyWords.COST)),
- Accumulators.min(MongoKeyWords.USAGE_FROM, MongoKeyWords.prepend$(MongoKeyWords
- .USAGE_DAY)),
- Accumulators.max(MongoKeyWords.USAGE_TO, MongoKeyWords.prepend$(MongoKeyWords
- .USAGE_DAY))
- ),
-
- Aggregates.sort(Sorts.ascending(
- MongoKeyWords.prependId(MongoKeyWords.RESOURCE_NAME),
- MongoKeyWords.prependId(MongoKeyWords.METER_CATEGORY)))
- )
- );
-
- updateBillingDetails(user, mapToDetails(aggregateIterable));
- } catch (RuntimeException e) {
- log.error("Updating billing details for user {} is failed", user, e);
- }
- }
-
- private List<Document> mapToDetails(AggregateIterable<Document> aggregateIterable) {
- List<Document> billingDetails = new ArrayList<>();
- for (Document document : aggregateIterable) {
- Document oldRef = (Document) document.get(MongoKeyWords.MONGO_ID);
- Document newDocument = new Document();
-
- newDocument.append(MongoKeyWords.USAGE_FROM, document.getString(MongoKeyWords.USAGE_FROM));
- newDocument.append(MongoKeyWords.USAGE_TO, document.getString(MongoKeyWords.USAGE_TO));
- newDocument.append(MongoKeyWords.COST, document.getDouble(MongoKeyWords.COST));
-
- newDocument.append(MongoKeyWords.METER_CATEGORY, oldRef.getString(MongoKeyWords.METER_CATEGORY));
- newDocument.append(MongoKeyWords.RESOURCE_NAME, oldRef.getString(MongoKeyWords.RESOURCE_NAME));
- newDocument.append(MongoKeyWords.EXPLORATORY_ID, oldRef.getString(MongoKeyWords.EXPLORATORY_ID));
- newDocument.append(MongoKeyWords.RESOURCE_TYPE, oldRef.getString(MongoKeyWords.RESOURCE_TYPE));
- newDocument.append(MongoKeyWords.CURRENCY_CODE, currencyCode);
-
- billingDetails.add(newDocument);
- }
-
- return billingDetails;
- }
-
-
- private void updateBillingDetails(String user, List<Document> billingDetails) {
- if (!billingDetails.isEmpty()) {
- Map<String, List<Document>> info = new HashMap<>();
-
- Consumer<Document> aggregator = e -> {
-
- String notebookId = e.getString(MongoKeyWords.EXPLORATORY_ID);
- List<Document> documents = info.get(notebookId);
- if (documents == null) {
- documents = new ArrayList<>();
- }
-
- documents.add(e);
- info.put(notebookId, documents);
- };
-
- billingDetails.stream()
- .filter(e -> DlabResourceType.EXPLORATORY.toString().equals(e.getString(MongoKeyWords
- .RESOURCE_TYPE)))
- .forEach(aggregator);
-
- billingDetails.stream()
- .filter(e -> DlabResourceType.COMPUTATIONAL.toString().equals(e.getString(MongoKeyWords
- .RESOURCE_TYPE)))
- .forEach(aggregator);
-
- billingDetails.stream()
- .filter(e -> DlabResourceType.VOLUME.toString().equals(e.getString(MongoKeyWords.RESOURCE_TYPE)))
- .forEach(aggregator);
-
-
- for (Map.Entry<String, List<Document>> entry : info.entrySet()) {
- double sum = entry.getValue().stream().mapToDouble(e -> e.getDouble(MongoKeyWords.COST)).sum();
-
- entry.getValue().forEach(e -> e.put(MongoKeyWords.COST_STRING,
- BillingCalculationUtils.formatDouble(e.getDouble(MongoKeyWords.COST))));
-
- log.debug("Update billing for notebook {}, cost is {} {}", entry.getKey(), sum, currencyCode);
-
- Bson updates = Updates.combine(
- Updates.set(MongoKeyWords.COST_STRING, BillingCalculationUtils.formatDouble(sum)),
- Updates.set(MongoKeyWords.COST, sum),
- Updates.set(MongoKeyWords.CURRENCY_CODE, currencyCode),
- Updates.set(MongoKeyWords.BILLING_DETAILS, entry.getValue()));
-
- UpdateResult updateResult = mongoDbBillingClient.getDatabase()
- .getCollection(MongoKeyWords.NOTEBOOK_COLLECTION)
- .updateOne(
- Filters.and(
- Filters.eq(MongoKeyWords.DLAB_USER, user),
- Filters.eq(MongoKeyWords.EXPLORATORY_ID_OLD, entry.getKey())
- ),
- updates
- );
-
- log.debug("Update result for {}/{} is {}", user, entry.getKey(), updateResult);
- }
- } else {
- log.warn("No billing details found for notebooks for user {}", user);
- }
- }
-
-
- private Document getGroupingFields(String... fieldNames) {
- Document d = new Document();
- for (String name : fieldNames) {
- d.put(name, "$" + name);
- }
- return d;
- }
-}
diff --git a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureInvoiceCalculationService.java b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureInvoiceCalculationService.java
index 0fd6098..02ec569 100644
--- a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureInvoiceCalculationService.java
+++ b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureInvoiceCalculationService.java
@@ -37,7 +37,10 @@ import org.apache.commons.lang3.StringUtils;
import java.io.IOException;
import java.net.URISyntaxException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import java.util.stream.Collectors;
/**
@@ -61,13 +64,9 @@ public class AzureInvoiceCalculationService {
* Constructs service class
*
* @param billingConfigurationAzure contains <code>billing-azure</code> module configuration
- * @param billableResources resources that invoices should be calculated for
*/
- public AzureInvoiceCalculationService(BillingConfigurationAzure billingConfigurationAzure,
- Set<AzureDlabBillableResource> billableResources) {
+ public AzureInvoiceCalculationService(BillingConfigurationAzure billingConfigurationAzure) {
this.billingConfigurationAzure = billingConfigurationAzure;
- this.billableResources = billableResources.stream().collect(Collectors.toMap(AzureDlabBillableResource::getId,
- e -> e));
}
/**
@@ -135,8 +134,6 @@ public class AzureInvoiceCalculationService {
if (usageAggregateRecordList != null && !usageAggregateRecordList.isEmpty()) {
log.info("Processing {} usage records", usageAggregateRecordList.size());
-
-
usageAggregateRecordList = usageAggregateRecordList.stream().filter(e ->
matchProperStructure(e) && isBillableDlabResource(e))
.collect(Collectors.toList());
@@ -164,7 +161,6 @@ public class AzureInvoiceCalculationService {
}
private boolean matchProperStructure(UsageAggregateRecord record) {
-
if (record.getProperties() == null) {
return false;
}
@@ -181,13 +177,10 @@ public class AzureInvoiceCalculationService {
private boolean isBillableDlabResource(UsageAggregateRecord record) {
String dlabId = record.getProperties().getParsedInstanceData().getMicrosoftResources().getTags().get("Name");
- return dlabId != null && !dlabId.isEmpty() && billableResources.containsKey(dlabId);
+ return dlabId != null && !dlabId.isEmpty();
}
- private AzureDailyResourceInvoice calculateInvoice(Map<String, Meter> rates, UsageAggregateRecord record,
- String dlabId) {
-
- AzureDlabBillableResource azureDlabBillableResource = billableResources.get(dlabId);
+ private AzureDailyResourceInvoice calculateInvoice(Map<String, Meter> rates, UsageAggregateRecord record, String dlabId) {
String meterId = record.getProperties().getMeterId();
Meter rateCard = rates.get(meterId);
@@ -196,34 +189,25 @@ public class AzureInvoiceCalculationService {
if (meterRates != null) {
Double rate = meterRates.get(AzureRateCardClient.MAIN_RATE_KEY);
if (rate != null) {
-
- AzureDailyResourceInvoice azureDailyResourceInvoice = new AzureDailyResourceInvoice
- (azureDlabBillableResource);
- azureDailyResourceInvoice.setUsageStartDate(record.getProperties().getUsageStartTime());
- azureDailyResourceInvoice.setUsageEndDate(record.getProperties().getUsageEndTime());
- azureDailyResourceInvoice.setMeterCategory(record.getProperties().getMeterCategory());
- azureDailyResourceInvoice.setCost(
- BillingCalculationUtils.round(rate * record.getProperties().getQuantity(), 2));
- azureDailyResourceInvoice.setDay(getDay(record.getProperties().getUsageStartTime()));
- azureDailyResourceInvoice.setCurrencyCode(billingConfigurationAzure.getCurrency());
-
- log.trace("Generated invoice for azure resource {}", azureDailyResourceInvoice);
-
- return azureDailyResourceInvoice;
-
+ return AzureDailyResourceInvoice.builder()
+ .dlabId(dlabId)
+ .usageStartDate(record.getProperties().getUsageStartTime())
+ .usageEndDate(record.getProperties().getUsageEndTime())
+ .meterCategory(record.getProperties().getMeterCategory())
+ .cost(BillingCalculationUtils.round(rate * record.getProperties().getQuantity(), 3))
+ .day(getDay(record.getProperties().getUsageStartTime()))
+ .currencyCode(billingConfigurationAzure.getCurrency())
+ .build();
} else {
- log.error("Rate Card {} has no rate for meter id {} and rate id {}. Skip record {}. Azure resource" +
- " {}",
- rateCard, meterId, AzureRateCardClient.MAIN_RATE_KEY, record, azureDlabBillableResource);
+ log.error("Rate Card {} has no rate for meter id {} and rate id {}. Skip record {}.",
+ rateCard, meterId, AzureRateCardClient.MAIN_RATE_KEY, record);
}
} else {
- log.error("Rate Card {} has no meter rates fro meter id {}. Skip record {}. Azure resource {}",
- rateCard, meterId, record, azureDlabBillableResource);
+ log.error("Rate Card {} has no meter rates fro meter id {}. Skip record {}",
+ rateCard, meterId, record);
}
} else {
- log.error("Meter rate {} form usage aggregate is not found in rate card. Skip record {}. Azure resource " +
- "{}",
- meterId, record, azureDlabBillableResource);
+ log.error("Meter rate {} form usage aggregate is not found in rate card. Skip record {}.", meterId, record);
}
return null;
@@ -231,7 +215,6 @@ public class AzureInvoiceCalculationService {
private String getNewToken() {
try {
-
log.info("Requesting authentication token ... ");
ApplicationTokenCredentials applicationTokenCredentials = new ApplicationTokenCredentials(
billingConfigurationAzure.getClientId(),
@@ -247,7 +230,6 @@ public class AzureInvoiceCalculationService {
}
private String getDay(String dateTime) {
-
if (dateTime != null) {
String[] parts = dateTime.split("T");
if (parts.length == 2) {
diff --git a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/BillingSchedulerAzure.java b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/BillingSchedulerAzure.java
index 0e11a1f..f81fe2e 100644
--- a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/BillingSchedulerAzure.java
+++ b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/BillingSchedulerAzure.java
@@ -23,7 +23,6 @@ import com.epam.dlab.MongoKeyWords;
import com.epam.dlab.billing.azure.config.AzureAuthFile;
import com.epam.dlab.billing.azure.config.BillingConfigurationAzure;
import com.epam.dlab.billing.azure.model.AzureDailyResourceInvoice;
-import com.epam.dlab.billing.azure.model.AzureDlabBillableResource;
import com.epam.dlab.billing.azure.model.BillingPeriod;
import com.epam.dlab.exceptions.DlabException;
import com.epam.dlab.util.mongo.modules.IsoDateModule;
@@ -47,7 +46,6 @@ import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
-import java.util.Set;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@@ -122,7 +120,6 @@ public class BillingSchedulerAzure {
private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormat.forPattern
("yyyy-MM-dd'T'HH:mm:ss.SSS'Z");
private static final String SCHEDULER_ID = "azureBillingScheduler";
- private AzureBillingDetailsService azureBillingDetailsService;
private BillingConfigurationAzure billingConfigurationAzure;
private MongoDbBillingClient client;
private ObjectMapper objectMapper = new ObjectMapper().registerModule(new IsoDateModule());
@@ -131,8 +128,6 @@ public class BillingSchedulerAzure {
public CalculateBilling(BillingConfigurationAzure billingConfigurationAzure, MongoDbBillingClient client) {
this.billingConfigurationAzure = billingConfigurationAzure;
this.client = client;
- this.azureBillingDetailsService = new AzureBillingDetailsService(client,
- billingConfigurationAzure.getCurrency());
}
@Override
@@ -146,14 +141,7 @@ public class BillingSchedulerAzure {
log.info("Billing period from db is {}", billingPeriod);
if (shouldTriggerJobByTime(currentTime, billingPeriod)) {
-
boolean hasNew = run(billingPeriod);
- if (hasNew) {
- log.info("Updating billing details");
- azureBillingDetailsService.updateBillingDetails();
- }
-
-
updateBillingPeriod(billingPeriod, currentTime, hasNew);
}
}
@@ -193,32 +181,20 @@ public class BillingSchedulerAzure {
}
private boolean run(BillingPeriod billingPeriod) {
-
- AzureBillableResourcesService azureBillableResourcesService = new AzureBillableResourcesService(client,
- billingConfigurationAzure.getSharedStorageAccountTagName(),
- billingConfigurationAzure.getSsnStorageAccountTagName(),
- billingConfigurationAzure.getDatalakeTagName());
- Set<AzureDlabBillableResource> billableResources = azureBillableResourcesService.getBillableResources();
-
AzureInvoiceCalculationService azureInvoiceCalculationService
- = new AzureInvoiceCalculationService(billingConfigurationAzure, billableResources);
+ = new AzureInvoiceCalculationService(billingConfigurationAzure);
List<AzureDailyResourceInvoice> dailyInvoices = azureInvoiceCalculationService.generateInvoiceData(
DATE_TIME_FORMATTER.print(new DateTime(billingPeriod.getFrom()).withZone(DateTimeZone.UTC)),
DATE_TIME_FORMATTER.print(new DateTime(billingPeriod.getTo()).withZone(DateTimeZone.UTC)));
-
if (!dailyInvoices.isEmpty()) {
-
client.getDatabase().getCollection(MongoKeyWords.BILLING_DETAILS)
.insertMany(dailyInvoices.stream().map(AzureDailyResourceInvoice::to)
.collect(Collectors.toList()));
-
return true;
-
} else {
log.warn("Daily invoices is empty for period {}", billingPeriod);
-
return false;
}
}
diff --git a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/model/AzureDailyResourceInvoice.java b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/model/AzureDailyResourceInvoice.java
index ff132a2..486ddd5 100644
--- a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/model/AzureDailyResourceInvoice.java
+++ b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/model/AzureDailyResourceInvoice.java
@@ -19,35 +19,21 @@
package com.epam.dlab.billing.azure.model;
-import com.epam.dlab.billing.DlabResourceType;
import com.epam.dlab.billing.azure.MongoDocument;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Builder;
import lombok.Data;
import lombok.EqualsAndHashCode;
-import lombok.NoArgsConstructor;
@Data
-@NoArgsConstructor
+@Builder
@EqualsAndHashCode(callSuper = true)
@JsonIgnoreProperties(ignoreUnknown = true)
public class AzureDailyResourceInvoice extends MongoDocument<AzureDailyResourceInvoice> {
@JsonProperty
private String dlabId;
@JsonProperty
- private String user;
- @JsonProperty
- private String project;
- @JsonProperty
- private String exploratoryId;
- @JsonProperty
- private String computationalId;
- @JsonProperty
- private DlabResourceType resourceType;
- @JsonProperty
- private String resourceName;
- @JsonProperty
private String meterCategory;
@JsonProperty
private String usageStartDate;
@@ -59,22 +45,4 @@ public class AzureDailyResourceInvoice extends MongoDocument<AzureDailyResourceI
private double cost;
@JsonProperty
private String currencyCode;
-
- @Builder
- public AzureDailyResourceInvoice(AzureDlabBillableResource azureDlabBillableResource) {
- this.dlabId = azureDlabBillableResource.getId();
- this.user = azureDlabBillableResource.getUser();
- this.project = azureDlabBillableResource.getProject();
- this.resourceType = azureDlabBillableResource.getType();
- this.resourceName = azureDlabBillableResource.getResourceName();
-
- if (resourceType == DlabResourceType.EXPLORATORY) {
- this.exploratoryId = azureDlabBillableResource.getId();
- } else if (resourceType == DlabResourceType.COMPUTATIONAL) {
- this.computationalId = azureDlabBillableResource.getId();
- this.exploratoryId = azureDlabBillableResource.getNotebookId();
- } else if (resourceType == DlabResourceType.VOLUME) {
- this.exploratoryId = azureDlabBillableResource.getNotebookId();
- }
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org