You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by of...@apache.org on 2020/03/16 15:40:18 UTC
[incubator-dlab] 01/09: [DLAB-643] Conveyed billing for remote aws
endpoint
This is an automated email from the ASF dual-hosted git repository.
ofuks pushed a commit to branch DLAB-1571
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 4b791b9c16b71f15a4a70649e76fda82851e1ece
Author: Oleh Fuks <ol...@gmail.com>
AuthorDate: Fri Mar 13 12:02:08 2020 +0200
[DLAB-643] Conveyed billing for remote aws endpoint
---
pom.xml | 2 +-
services/billing-aws/billing.yml | 3 +-
services/billing-aws/pom.xml | 128 +++++----
.../java/com/epam/dlab/BillingAwsApplication.java | 37 +++
.../main/java/com/epam/dlab/BillingScheduler.java | 4 +-
.../java/com/epam/dlab/conf/SecurityConfig.java | 66 +++++
.../epam/dlab/controller/BillingController.java | 50 ++++
.../main/java/com/epam/dlab/dao/BillingDAO.java | 31 +++
.../com/epam/dlab/dao/impl/BillingDAOImpl.java | 109 ++++++++
.../java/com/epam/dlab/mongo/AdapterMongoDb.java | 9 +-
.../com/epam/dlab/mongo/DlabResourceTypeDAO.java | 297 +--------------------
.../billing-aws/src/main/resources/application.yml | 34 +++
12 files changed, 407 insertions(+), 363 deletions(-)
diff --git a/pom.xml b/pom.xml
index 853ef15..b6808a8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -74,7 +74,7 @@
<com.google.inject.version>4.2.0</com.google.inject.version>
<dropwizard-template-config.version>1.4.0</dropwizard-template-config.version>
<com.aegisql.conveyor.version>1.1.7</com.aegisql.conveyor.version>
- <org.mongodb.version>3.3.0</org.mongodb.version>
+ <org.mongodb.version>3.8.2</org.mongodb.version>
<junit.version>4.12</junit.version>
<org.mockito.version>1.10.19</org.mockito.version>
<java.version>1.8</java.version>
diff --git a/services/billing-aws/billing.yml b/services/billing-aws/billing.yml
index ee70487..510411f 100644
--- a/services/billing-aws/billing.yml
+++ b/services/billing-aws/billing.yml
@@ -34,7 +34,8 @@ database: dlabdb
scheduler:
# Schedule is comma separated values of time in format hh[:mm[:ss]]. hh - in the 24-hour clock, at 8:15PM is 20:15.
- schedule: 0:00, 3:00, 6:00, 9:00, 12:00, 15:00, 18:00, 21:00
+ schedule: 0:00, 1:00, 2:00, 3:00, 4:00, 5:00, 6:00, 7:00, 8:00, 9:00, 10:00, 11:00, 12:00, 13:00, 14:00, 15:00, 16:00,
+ 17:00, 18:00, 19:00, 20:00, 21:00, 22:00, 23:00
# Adapter for reading source data. Known types: file, s3file
adapterIn:
diff --git a/services/billing-aws/pom.xml b/services/billing-aws/pom.xml
index a411c20..ec4c830 100644
--- a/services/billing-aws/pom.xml
+++ b/services/billing-aws/pom.xml
@@ -28,7 +28,7 @@
<version>1.0</version>
<relativePath>../../pom.xml</relativePath>
</parent>
-
+
<artifactId>billing-aws</artifactId>
<properties>
@@ -37,6 +37,25 @@
<org.freemarker.version>2.3.22</org.freemarker.version>
</properties>
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-dependencies</artifactId>
+ <version>2.1.3.RELEASE</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.keycloak.bom</groupId>
+ <artifactId>keycloak-adapter-bom</artifactId>
+ <version>4.8.3.Final</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
<dependencies>
<dependency>
<groupId>com.epam.dlab</groupId>
@@ -64,11 +83,6 @@
<version>${org.freemarker.version}</version>
</dependency>
<dependency>
- <groupId>com.epam.dlab</groupId>
- <artifactId>dlab-model</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
<groupId>javax.validation</groupId>
<artifactId>validation-api</artifactId>
<version>2.0.0.Final</version>
@@ -119,80 +133,62 @@
<artifactId>guava</artifactId>
<version>24.1-jre</version>
</dependency>
+
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-configuration-processor</artifactId>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-data-mongodb</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-web</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-security</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.keycloak</groupId>
+ <artifactId>keycloak-spring-boot-starter</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.keycloak</groupId>
+ <artifactId>keycloak-spring-security-adapter</artifactId>
+ <version>4.8.3.Final</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>${org.mockito.version}</version>
<scope>test</scope>
</dependency>
-
+ <dependency>
+ <groupId>com.epam.dlab</groupId>
+ <artifactId>dlab-model</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
</dependencies>
<build>
<plugins>
- <plugin>
- <artifactId>maven-shade-plugin</artifactId>
- <version>${maven-shade-plugin.version}</version>
+ <plugin>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-maven-plugin</artifactId>
<executions>
<execution>
- <phase>package</phase>
<goals>
- <goal>shade</goal>
+ <goal>repackage</goal>
</goals>
- <configuration>
- <createDependencyReducedPom>false</createDependencyReducedPom>
- <minimizeJar>false</minimizeJar>
- <filters>
- <filter>
- <artifact>org.hibernate:hibernate-validator</artifact>
- <includes>**</includes>
- </filter>
- <filter>
- <artifact>org.glassfish.web:javax.el</artifact>
- <includes>**</includes>
- </filter>
- <filter>
- <artifact>org.jboss.logging:jboss-logging</artifact>
- <includes>**</includes>
- </filter>
- <filter>
- <artifact>com.fasterxml:classmate</artifact>
- <includes>**</includes>
- </filter>
-
- <filter>
- <artifact>javax.validation:validation-api</artifact>
- <includes>**</includes>
- </filter>
- <filter>
- <artifact>commons-logging:commons-logging</artifact>
- <includes>**</includes>
- </filter>
- <filter>
- <artifact>com.fasterxml.jackson.core:jackson-databind</artifact>
- <includes>**</includes>
- </filter>
- </filters>
- <transformers>
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass>com.epam.dlab.BillingScheduler</mainClass>
- <manifestEntries>
- <Created-By><EPAM> Systems</Created-By>
- <Name>com/epam/dlab</Name>
- <Implementation-Title>DLab Billing Tool</Implementation-Title>
- <Implementation-Version>${dlab.version}</Implementation-Version>
- <Implementation-Vendor><EPAM> Systems</Implementation-Vendor>
- <Build-Time>${maven.build.timestamp}</Build-Time>
- <Build-OS>${os.name}</Build-OS>
- <GIT-Branch>${scmBranch}</GIT-Branch>
- <GIT-Commit>${buildNumber}</GIT-Commit>
- </manifestEntries>
- </transformer>
- </transformers>
- </configuration>
</execution>
</executions>
</plugin>
diff --git a/services/billing-aws/src/main/java/com/epam/dlab/BillingAwsApplication.java b/services/billing-aws/src/main/java/com/epam/dlab/BillingAwsApplication.java
new file mode 100644
index 0000000..cede034
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/dlab/BillingAwsApplication.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab;
+
+import com.epam.dlab.exceptions.InitializationException;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.data.mongodb.repository.config.EnableMongoRepositories;
+
+@SpringBootApplication
+@EnableMongoRepositories
+@EnableConfigurationProperties
+public class BillingAwsApplication {
+
+ public static void main(String[] args) throws InitializationException {
+ SpringApplication.run(BillingAwsApplication.class, args);
+ BillingScheduler.startScheduler(args);
+ }
+}
diff --git a/services/billing-aws/src/main/java/com/epam/dlab/BillingScheduler.java b/services/billing-aws/src/main/java/com/epam/dlab/BillingScheduler.java
index 5db8269..7a7d5ff 100644
--- a/services/billing-aws/src/main/java/com/epam/dlab/BillingScheduler.java
+++ b/services/billing-aws/src/main/java/com/epam/dlab/BillingScheduler.java
@@ -229,7 +229,7 @@ public class BillingScheduler implements Runnable {
* @param args the arguments of command line.
* @throws InitializationException
*/
- public static void main(String[] args) throws InitializationException {
+ public static void startScheduler(String[] args) throws InitializationException {
if (ServiceUtils.printAppVersion(BillingTool.class, args)) {
return;
}
@@ -247,8 +247,6 @@ public class BillingScheduler implements Runnable {
} else {
throw new InitializationException("Missing the name of configuration file");
}
- } else {
- throw new InitializationException("Unknow argument: " + args[i]);
}
}
diff --git a/services/billing-aws/src/main/java/com/epam/dlab/conf/SecurityConfig.java b/services/billing-aws/src/main/java/com/epam/dlab/conf/SecurityConfig.java
new file mode 100644
index 0000000..dba4086
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/dlab/conf/SecurityConfig.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.conf;
+
+import org.keycloak.adapters.KeycloakConfigResolver;
+import org.keycloak.adapters.springboot.KeycloakSpringBootConfigResolver;
+import org.keycloak.adapters.springsecurity.KeycloakConfiguration;
+import org.keycloak.adapters.springsecurity.authentication.KeycloakAuthenticationProvider;
+import org.keycloak.adapters.springsecurity.config.KeycloakWebSecurityConfigurerAdapter;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.security.config.annotation.authentication.builders.AuthenticationManagerBuilder;
+import org.springframework.security.config.annotation.web.builders.HttpSecurity;
+import org.springframework.security.core.authority.mapping.SimpleAuthorityMapper;
+import org.springframework.security.core.session.SessionRegistryImpl;
+import org.springframework.security.web.authentication.session.RegisterSessionAuthenticationStrategy;
+import org.springframework.security.web.authentication.session.SessionAuthenticationStrategy;
+
+@KeycloakConfiguration
+class SecurityConfig extends KeycloakWebSecurityConfigurerAdapter {
+
+ @Autowired
+ public void configureGlobal(AuthenticationManagerBuilder auth) {
+ KeycloakAuthenticationProvider keycloakAuthenticationProvider = keycloakAuthenticationProvider();
+ keycloakAuthenticationProvider.setGrantedAuthoritiesMapper(new SimpleAuthorityMapper());
+ auth.authenticationProvider(keycloakAuthenticationProvider);
+ }
+
+ @Bean
+ public KeycloakConfigResolver KeycloakConfigResolver() {
+ return new KeycloakSpringBootConfigResolver();
+ }
+
+ @Bean
+ @Override
+ protected SessionAuthenticationStrategy sessionAuthenticationStrategy() {
+ return new RegisterSessionAuthenticationStrategy(new SessionRegistryImpl());
+ }
+
+ @Override
+ protected void configure(HttpSecurity http) throws Exception {
+ super.configure(http);
+ http
+ .anonymous().disable()
+ .authorizeRequests()
+ .anyRequest()
+ .authenticated();
+ }
+}
\ No newline at end of file
diff --git a/services/billing-aws/src/main/java/com/epam/dlab/controller/BillingController.java b/services/billing-aws/src/main/java/com/epam/dlab/controller/BillingController.java
new file mode 100644
index 0000000..82d6d16
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/dlab/controller/BillingController.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.controller;
+
+import com.epam.dlab.dao.BillingDAO;
+import com.epam.dlab.dto.billing.BillingData;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.List;
+
+@RestController
+public class BillingController {
+
+ private final BillingDAO billingDAO;
+
+ public BillingController(BillingDAO billingDAO) {
+ this.billingDAO = billingDAO;
+ }
+
+ @GetMapping
+ public ResponseEntity<List<BillingData>> getBilling(@RequestParam List<String> dlabIds) {
+ return new ResponseEntity<>(billingDAO.getBillingReport(dlabIds), HttpStatus.OK);
+ }
+
+ @GetMapping("/report")
+ public ResponseEntity<List<BillingData>> getBilling() {
+ return new ResponseEntity<>(billingDAO.getBillingReport(), HttpStatus.OK);
+ }
+}
diff --git a/services/billing-aws/src/main/java/com/epam/dlab/dao/BillingDAO.java b/services/billing-aws/src/main/java/com/epam/dlab/dao/BillingDAO.java
new file mode 100644
index 0000000..8ecaddf
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/dlab/dao/BillingDAO.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.dao;
+
+import com.epam.dlab.dto.billing.BillingData;
+
+import java.util.List;
+
+public interface BillingDAO {
+
+ List<BillingData> getBillingReport();
+
+ List<BillingData> getBillingReport(List<String> dlabIds);
+}
diff --git a/services/billing-aws/src/main/java/com/epam/dlab/dao/impl/BillingDAOImpl.java b/services/billing-aws/src/main/java/com/epam/dlab/dao/impl/BillingDAOImpl.java
new file mode 100644
index 0000000..4fe4606
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/dlab/dao/impl/BillingDAOImpl.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.dao.impl;
+
+import com.epam.dlab.dao.BillingDAO;
+import com.epam.dlab.dto.billing.BillingData;
+import com.epam.dlab.exceptions.DlabException;
+import lombok.extern.slf4j.Slf4j;
+import org.bson.Document;
+import org.springframework.data.mongodb.core.MongoTemplate;
+import org.springframework.data.mongodb.core.aggregation.Aggregation;
+import org.springframework.data.mongodb.core.aggregation.GroupOperation;
+import org.springframework.data.mongodb.core.aggregation.MatchOperation;
+import org.springframework.data.mongodb.core.query.Criteria;
+import org.springframework.stereotype.Component;
+
+import java.math.BigDecimal;
+import java.time.LocalDate;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import static com.epam.dlab.model.aws.ReportLine.FIELD_COST;
+import static com.epam.dlab.model.aws.ReportLine.FIELD_CURRENCY_CODE;
+import static com.epam.dlab.model.aws.ReportLine.FIELD_DLAB_ID;
+import static com.epam.dlab.model.aws.ReportLine.FIELD_PRODUCT;
+import static com.epam.dlab.model.aws.ReportLine.FIELD_RESOURCE_TYPE;
+import static com.epam.dlab.model.aws.ReportLine.FIELD_USAGE_DATE;
+import static org.springframework.data.mongodb.core.aggregation.Aggregation.group;
+import static org.springframework.data.mongodb.core.aggregation.Aggregation.newAggregation;
+
+@Component
+@Slf4j
+public class BillingDAOImpl implements BillingDAO {
+ private final MongoTemplate mongoTemplate;
+
+ public BillingDAOImpl(MongoTemplate mongoTemplate) {
+ this.mongoTemplate = mongoTemplate;
+ }
+
+ @Override
+ public List<BillingData> getBillingReport() {
+ try {
+ GroupOperation groupOperation = getGroupOperation();
+ Aggregation aggregation = newAggregation(groupOperation);
+
+ return mongoTemplate.aggregate(aggregation, "billing", Document.class).getMappedResults()
+ .stream()
+ .map(this::toBillingData)
+ .collect(Collectors.toList());
+ } catch (Exception e) {
+ log.error("Cannot retrieve billing information ", e);
+ throw new DlabException("Cannot retrieve billing information", e);
+ }
+ }
+
+ @Override
+ public List<BillingData> getBillingReport(List<String> dlabIds) {
+ try {
+ GroupOperation groupOperation = getGroupOperation();
+ MatchOperation matchOperation = Aggregation.match(Criteria.where("dlab_id").in(dlabIds));
+ Aggregation aggregation = newAggregation(matchOperation, groupOperation);
+
+ return mongoTemplate.aggregate(aggregation, "billing", Document.class).getMappedResults()
+ .stream()
+ .map(this::toBillingData)
+ .collect(Collectors.toList());
+ } catch (Exception e) {
+ log.error("Cannot retrieve billing information ", e);
+ throw new DlabException("Cannot retrieve billing information", e);
+ }
+ }
+
+ private GroupOperation getGroupOperation() {
+ return group(FIELD_PRODUCT, FIELD_CURRENCY_CODE, FIELD_RESOURCE_TYPE, FIELD_DLAB_ID)
+ .min(FIELD_USAGE_DATE).as("from")
+ .max(FIELD_USAGE_DATE).as("to")
+ .sum(FIELD_COST).as(FIELD_COST);
+ }
+
+ private BillingData toBillingData(Document billingData) {
+ return BillingData.builder()
+ .tag(billingData.getString(FIELD_DLAB_ID))
+ .usageDateFrom(Optional.ofNullable(billingData.getString("from")).map(LocalDate::parse).orElse(null))
+ .usageDateTo(Optional.ofNullable(billingData.getString("to")).map(LocalDate::parse).orElse(null))
+ .product(billingData.getString(FIELD_PRODUCT))
+ .usageType(billingData.getString(FIELD_RESOURCE_TYPE))
+ .cost(BigDecimal.valueOf(billingData.getDouble(FIELD_COST)).setScale(3, BigDecimal.ROUND_HALF_UP).doubleValue())
+ .currency(billingData.getString(FIELD_CURRENCY_CODE))
+ .build();
+ }
+}
diff --git a/services/billing-aws/src/main/java/com/epam/dlab/mongo/AdapterMongoDb.java b/services/billing-aws/src/main/java/com/epam/dlab/mongo/AdapterMongoDb.java
index a13ed05..e0af4e6 100644
--- a/services/billing-aws/src/main/java/com/epam/dlab/mongo/AdapterMongoDb.java
+++ b/services/billing-aws/src/main/java/com/epam/dlab/mongo/AdapterMongoDb.java
@@ -191,7 +191,7 @@ public class AdapterMongoDb extends DBAdapterBase {
}
@Override
- public void writeHeader(List<String> header) throws AdapterException {
+ public void writeHeader(List<String> header) {
// Nothing to do
}
@@ -235,13 +235,6 @@ public class AdapterMongoDb extends DBAdapterBase {
} catch (Exception e) {
throw new AdapterException("Cannot update total monthly cost. " + e.getLocalizedMessage(), e);
}
- try {
- if (months.size() > 0) {
- resourceTypeDAO.updateExploratoryCost();
- }
- } catch (Exception e) {
- throw new AdapterException("Cannot update total cost of exploratory. " + e.getLocalizedMessage(), e);
- }
}
@Override
diff --git a/services/billing-aws/src/main/java/com/epam/dlab/mongo/DlabResourceTypeDAO.java b/services/billing-aws/src/main/java/com/epam/dlab/mongo/DlabResourceTypeDAO.java
index dcfe369..0c95605 100644
--- a/services/billing-aws/src/main/java/com/epam/dlab/mongo/DlabResourceTypeDAO.java
+++ b/services/billing-aws/src/main/java/com/epam/dlab/mongo/DlabResourceTypeDAO.java
@@ -19,28 +19,28 @@
package com.epam.dlab.mongo;
-import com.epam.dlab.billing.BillingCalculationUtils;
-import com.epam.dlab.billing.DlabResourceType;
-import com.epam.dlab.dto.base.DataEngineType;
import com.epam.dlab.exceptions.InitializationException;
import com.epam.dlab.exceptions.ParseException;
import com.epam.dlab.model.aws.ReportLine;
import com.mongodb.client.AggregateIterable;
import com.mongodb.client.MongoCollection;
-import com.mongodb.client.model.Updates;
-import org.apache.commons.lang3.StringUtils;
import org.bson.Document;
import org.bson.conversions.Bson;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.text.SimpleDateFormat;
-import java.util.*;
-
-import static com.mongodb.client.model.Accumulators.*;
-import static com.mongodb.client.model.Aggregates.*;
-import static com.mongodb.client.model.Filters.*;
-import static com.mongodb.client.model.Projections.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static com.mongodb.client.model.Accumulators.sum;
+import static com.mongodb.client.model.Aggregates.group;
+import static com.mongodb.client.model.Aggregates.match;
+import static com.mongodb.client.model.Filters.and;
+import static com.mongodb.client.model.Filters.eq;
+import static com.mongodb.client.model.Filters.gte;
+import static com.mongodb.client.model.Filters.lte;
import static org.apache.commons.lang3.StringUtils.EMPTY;
/**
@@ -48,8 +48,6 @@ import static org.apache.commons.lang3.StringUtils.EMPTY;
*/
public class DlabResourceTypeDAO implements MongoConstants {
private static final Logger LOGGER = LoggerFactory.getLogger(DlabResourceTypeDAO.class);
- private static final String VOLUME_PRIMARY_SUFFIX = "-volume-primary";
- private static final String VOLUME_SECONDARY_SUFFIX = "-volume-secondary";
/**
* Mongo database connection.
@@ -63,11 +61,6 @@ public class DlabResourceTypeDAO implements MongoConstants {
private String serviceBaseNameId;
/**
- * Describe all DLab resources: SSN, EDGE, exploratory, computational and buckets.
- */
- private ResourceItemList resourceList;
-
- /**
* Instantiate DAO for billing resources.
*
* @param connection the connection to Mongo DB.
@@ -76,7 +69,6 @@ public class DlabResourceTypeDAO implements MongoConstants {
public DlabResourceTypeDAO(MongoDbConnection connection) throws InitializationException {
this.connection = connection;
setServiceBaseName();
- setResourceList();
}
/**
@@ -110,131 +102,6 @@ public class DlabResourceTypeDAO implements MongoConstants {
}
/**
- * Return DLab resources from Mongo DB.
- *
- * @throws InitializationException
- */
- public ResourceItemList getResourceList() {
- return resourceList;
- }
-
- /**
- * Load and return DLab resources from Mongo DB.
- *
- * @throws InitializationException
- */
- private void setResourceList() {
- resourceList = new ResourceItemList();
-
- // Add SSN
- String sbName = getServiceBaseName();
- resourceList.append(sbName + "-ssn", "SSN", DlabResourceType.SSN);
- resourceList.append(sbName + "-ssn-volume-primary", "SSN volume", DlabResourceType.VOLUME);
- resourceList.append(sbName + "-ssn-bucket", "SSN bucket", DlabResourceType.SSN_BUCKET);
-
- // collaboration bucket
- resourceList.append(sbName + "-shared-bucket", "Collaboration bucket", DlabResourceType
- .COLLABORATION_BUCKET);
-
- // Add PROJECTS
- Bson projection = fields(include("name", "endpoints"));
- Iterable<Document> docs = connection.getCollection("Projects").find().projection(projection);
- for (Document d : docs) {
- String projectName = d.getString("name");
- ((List<Document>) d.get("endpoints"))
- .stream()
- .map(endpoint -> endpoint.getString("name"))
- .forEach(endpoint -> {
- resourceList.append(sbName + "-" + endpoint + "-shared-bucket", "Shared endpoint bucket",
- DlabResourceType.COLLABORATION_BUCKET);
- resourceList.append(sbName + "-" + projectName + "-" + endpoint + "-bucket", "Project bucket",
- DlabResourceType.COLLABORATION_BUCKET, null, null, projectName);
- resourceList.append(sbName + "-" + projectName + "-" + endpoint + "-edge", "EDGE Node",
- DlabResourceType.EDGE, null, null, projectName);
- resourceList.append(sbName + "-" + projectName+ "-" + endpoint + "-edge-volume-primary",
- "EDGE Volume", DlabResourceType.VOLUME, null, null, projectName);
- });
- }
-
- // Add exploratory
- projection = fields(include(FIELD_USER,
- FIELD_EXPLORATORY_NAME,
- FIELD_EXPLORATORY_ID,
- FIELD_PROJECT,
- FIELD_COMPUTATIONAL_RESOURCES + "." + FIELD_COMPUTATIONAL_ID,
- FIELD_COMPUTATIONAL_RESOURCES + "." + FIELD_COMPUTATIONAL_NAME,
- FIELD_COMPUTATIONAL_RESOURCES + "." + FIELD_IMAGE,
- FIELD_COMPUTATIONAL_RESOURCES + "." + FIELD_DATAENGINE_INSTANCE_COUNT),
- excludeId());
- docs = connection.getCollection(COLLECTION_USER_INSTANCES).find().projection(projection);
- for (Document exp : docs) {
- String username = exp.getString(FIELD_USER);
- String exploratoryName = exp.getString(FIELD_EXPLORATORY_NAME);
- String exploratoryId = exp.getString(FIELD_EXPLORATORY_ID);
- String project = exp.getString(FIELD_PROJECT);
- resourceList.append(exploratoryId, exploratoryName, DlabResourceType.EXPLORATORY, username,
- exploratoryName, project);
- appendExploratoryVolumes(username, exploratoryName, exploratoryId, project);
-
- // Add computational
- @SuppressWarnings("unchecked")
- List<Document> compList = (List<Document>) exp.get(FIELD_COMPUTATIONAL_RESOURCES);
- if (compList == null) {
- continue;
- }
- for (Document comp : compList) {
- String computationalId = comp.getString(FIELD_COMPUTATIONAL_ID);
- String computationalName = comp.getString(FIELD_COMPUTATIONAL_NAME);
- final DataEngineType dataEngineType = DataEngineType.fromDockerImageName(comp.getString(FIELD_IMAGE));
- resourceList.append(computationalId, computationalName, DlabResourceType.COMPUTATIONAL, username,
- exploratoryName, project);
- if (DataEngineType.CLOUD_SERVICE == dataEngineType) {
- appendDataengineServiceVolumes(username, exploratoryName, computationalId, computationalName,
- project);
- } else {
- appendDataengineVolumes(username, exploratoryName, comp, computationalId, computationalName,
- project);
- }
- }
- }
- LOGGER.debug("resourceList is {}", resourceList);
- }
-
- private void appendExploratoryVolumes(String username, String exploratoryName, String exploratoryId,
- String project) {
- resourceList.append(exploratoryId + VOLUME_PRIMARY_SUFFIX, "Volume primary", DlabResourceType.VOLUME,
- username, exploratoryName, project);
- resourceList.append(exploratoryId + VOLUME_SECONDARY_SUFFIX, "Volume secondary", DlabResourceType.VOLUME,
- username, exploratoryName, project);
- }
-
- private void appendDataengineServiceVolumes(String username, String exploratoryName, String computationalId,
- String computationalName, String project) {
- resourceList.append(computationalId + VOLUME_PRIMARY_SUFFIX, computationalName + " volume primary",
- DlabResourceType.VOLUME, username, exploratoryName, project);
- resourceList.append(computationalId + VOLUME_SECONDARY_SUFFIX, computationalName + " volume secondary",
- DlabResourceType.VOLUME, username, exploratoryName, project);
- }
-
- private void appendDataengineVolumes(String username, String exploratoryName, Document comp, String
- computationalId, String computationalName, String project) {
- resourceList.append(computationalId + "-m-volume-primary", computationalName + " master volume primary",
- DlabResourceType.VOLUME, username, exploratoryName, project);
- resourceList.append(computationalId + "-m-volume-secondary", computationalName + " master volume secondary",
- DlabResourceType.VOLUME, username, exploratoryName, project);
- final Integer instanceCount = Integer.valueOf(comp.getString(FIELD_DATAENGINE_INSTANCE_COUNT));
- for (int i = instanceCount - 1; i > 0; i--) {
- final String slaveId = computationalId + "-s" + i;
- final String slaveName = computationalName + "-s" + i;
- resourceList.append(slaveId + VOLUME_PRIMARY_SUFFIX, slaveName + " volume primary", DlabResourceType
- .VOLUME, username, exploratoryName, project);
- resourceList.append(slaveId + VOLUME_SECONDARY_SUFFIX, slaveName + " volume secondary", DlabResourceType
- .VOLUME, username, exploratoryName, project);
- }
- }
-
-
- /**
* Convert and return the report line of billing to Mongo document.
*
* @param row report line.
@@ -248,21 +115,7 @@ public class DlabResourceTypeDAO implements MongoConstants {
resourceId);
}
resourceId = resourceId.substring(serviceBaseNameId.length());
-
- ResourceItem resource = resourceList.getById(resourceId);
Document d = new Document(ReportLine.FIELD_DLAB_ID, resourceId);
- if (resource == null) {
- d.put(FIELD_DLAB_RESOURCE_ID, null);
- d.put(FIELD_DLAB_RESOURCE_TYPE, null);
- d.put(ReportLine.FIELD_USER_ID, null);
- d.put(FIELD_EXPLORATORY_NAME, null);
- } else {
- d.put(FIELD_DLAB_RESOURCE_ID, resource.getResourceId());
- d.put(FIELD_DLAB_RESOURCE_TYPE, resource.getType().toString());
- d.put(ReportLine.FIELD_USER_ID, resource.getUser());
- d.put(FIELD_EXPLORATORY_NAME, resource.getExploratoryName());
- d.put(FIELD_PROJECT, resource.getProject());
- }
return d.append(ReportLine.FIELD_USAGE_DATE, row.getUsageDate())
.append(ReportLine.FIELD_PRODUCT, row.getProduct())
.append(ReportLine.FIELD_USAGE_TYPE, row.getUsageType())
@@ -280,7 +133,7 @@ public class DlabResourceTypeDAO implements MongoConstants {
*
* @param fieldNames the list of field names.
*/
- private Document getGrouppingFields(String... fieldNames) {
+ private Document getGroupingFields(String... fieldNames) {
Document d = new Document();
for (String name : fieldNames) {
d.put(name, "$" + name);
@@ -307,7 +160,7 @@ public class DlabResourceTypeDAO implements MongoConstants {
List<? extends Bson> pipeline = Arrays.asList(
match(and(gte(ReportLine.FIELD_USAGE_DATE, month + "-01"),
lte(ReportLine.FIELD_USAGE_DATE, month + "-31"))),
- group(getGrouppingFields(FIELD_DLAB_RESOURCE_ID,
+ group(getGroupingFields(FIELD_DLAB_RESOURCE_ID,
FIELD_DLAB_RESOURCE_TYPE,
FIELD_USER,
FIELD_EXPLORATORY_NAME,
@@ -333,128 +186,4 @@ public class DlabResourceTypeDAO implements MongoConstants {
collection.insertMany(totals);
}
}
-
- /**
- * Comparator to sort billing exploratory details.
- */
- class BillingComparator implements Comparator<Document> {
- @Override
- public int compare(Document d1, Document d2) {
- int result = StringUtils.compare(d1.getString(FIELD_RESOURCE_NAME), d2.getString(FIELD_RESOURCE_NAME));
- if (result == 0) {
- result = StringUtils.compare(d1.getString(ReportLine.FIELD_PRODUCT), d2.getString(ReportLine
- .FIELD_PRODUCT));
- if (result == 0) {
- return StringUtils.compare(d1.getString(ReportLine.FIELD_RESOURCE_TYPE), d2.getString(ReportLine
- .FIELD_RESOURCE_TYPE));
- }
- }
- return result;
- }
- }
-
- /**
- * Update exploratory cost in Mongo DB.
- *
- * @param user the name of user.
- * @param exploratoryName id of exploratory.
- */
- private void updateExploratoryCost(String user, String exploratoryName) {
- LOGGER.debug("Update explorartory {} cost for user {}", exploratoryName, user);
- List<? extends Bson> pipeline = Arrays.asList(
- match(and(eq(FIELD_USER, user),
- eq(FIELD_EXPLORATORY_NAME, exploratoryName))),
- group(getGrouppingFields(FIELD_DLAB_RESOURCE_ID,
- ReportLine.FIELD_PRODUCT,
- ReportLine.FIELD_RESOURCE_TYPE,
- ReportLine.FIELD_CURRENCY_CODE),
- sum(ReportLine.FIELD_COST, "$" + ReportLine.FIELD_COST),
- min(FIELD_USAGE_DATE_START, "$" + ReportLine.FIELD_USAGE_DATE),
- max(FIELD_USAGE_DATE_END, "$" + ReportLine.FIELD_USAGE_DATE)
- ),
- sort(new Document(FIELD_ID + "." + FIELD_DLAB_RESOURCE_ID, 1).append(FIELD_ID + "." + ReportLine
- .FIELD_PRODUCT, 1))
- );
- AggregateIterable<Document> docs = connection.getCollection(COLLECTION_BILLING)
- .aggregate(pipeline);
- LinkedList<Document> billing = new LinkedList<>();
- ResourceItemList resources = getResourceList();
- Double costTotal = null;
- String currencyCode = null;
- for (Document d : docs) {
- Document id = (Document) d.get(FIELD_ID);
- double cost = BillingCalculationUtils.round(d.getDouble(ReportLine.FIELD_COST), 2);
- costTotal = (costTotal == null ? cost : costTotal + cost);
- if (currencyCode == null) {
- currencyCode = id.getString(ReportLine.FIELD_CURRENCY_CODE);
- }
-
- Document total = new Document()
- .append(FIELD_RESOURCE_NAME, resources.getById(id.getString(FIELD_DLAB_RESOURCE_ID))
- .getResourceName())
- .append(ReportLine.FIELD_PRODUCT, id.getString(ReportLine.FIELD_PRODUCT))
- .append(ReportLine.FIELD_RESOURCE_TYPE, id.getString(ReportLine.FIELD_RESOURCE_TYPE))
- .append(ReportLine.FIELD_COST, BillingCalculationUtils.formatDouble(cost))
- .append(ReportLine.FIELD_CURRENCY_CODE, id.getString(ReportLine.FIELD_CURRENCY_CODE))
- .append(FIELD_USAGE_DATE_START, d.getString(FIELD_USAGE_DATE_START))
- .append(FIELD_USAGE_DATE_END, d.getString(FIELD_USAGE_DATE_END));
- billing.add(total);
- }
-
- LOGGER.debug("Total explorartory {} cost for user {} is {} {}, detail count is {}",
- exploratoryName, user, costTotal, currencyCode, billing.size());
- billing.sort(new BillingComparator());
-
- MongoCollection<Document> cExploratory = connection.getCollection(COLLECTION_USER_INSTANCES);
- Bson values = Updates.combine(
- Updates.set(ReportLine.FIELD_COST, BillingCalculationUtils.formatDouble(costTotal)),
- Updates.set(FIELD_CURRENCY_CODE, currencyCode),
- Updates.set(COLLECTION_BILLING, (!billing.isEmpty() ? billing : null)));
- cExploratory.updateOne(
- and(and(eq(FIELD_USER, user),
- eq(FIELD_EXPLORATORY_NAME, exploratoryName))),
- values);
- }
-
- /**
- * Update EDGE cost in Mongo DB.
- *
- * @param user the name of user.
- */
- private void updateEdgeCost(String user) {
- List<? extends Bson> pipeline = Arrays.asList(
- match(and(eq(FIELD_USER, user),
- eq(FIELD_EXPLORATORY_NAME, null))),
- group(getGrouppingFields(ReportLine.FIELD_CURRENCY_CODE),
- sum(ReportLine.FIELD_COST, "$" + ReportLine.FIELD_COST))
- );
- AggregateIterable<Document> docs = connection.getCollection(COLLECTION_BILLING_TOTAL)
- .aggregate(pipeline);
-
- MongoCollection<Document> cEdge = connection.getCollection(COLLECTION_USER_EDGE);
- for (Document d : docs) {
- Document id = (Document) d.get(FIELD_ID);
- Bson values = Updates.combine(
- Updates.set(ReportLine.FIELD_COST, BillingCalculationUtils.round(d.getDouble(ReportLine
- .FIELD_COST), 2)),
- Updates.set(FIELD_CURRENCY_CODE, id.get(ReportLine.FIELD_CURRENCY_CODE)));
- cEdge.updateOne(
- eq(FIELD_ID, user),
- values);
- }
- }
-
- /**
- * Update the cost of exploratory environment for all users in Mongo DB.
- */
- public void updateExploratoryCost() {
- for (int i = 0; i < resourceList.size(); i++) {
- ResourceItem item = resourceList.get(i);
- if (item.getType() == DlabResourceType.EXPLORATORY) {
- updateExploratoryCost(item.getUser(), item.getExploratoryName());
- } else if (item.getType() == DlabResourceType.EDGE) {
- updateEdgeCost(item.getUser());
- }
- }
- }
}
diff --git a/services/billing-aws/src/main/resources/application.yml b/services/billing-aws/src/main/resources/application.yml
new file mode 100644
index 0000000..b31f91a
--- /dev/null
+++ b/services/billing-aws/src/main/resources/application.yml
@@ -0,0 +1,34 @@
+spring:
+ main:
+ allow-bean-definition-overriding: true
+ data:
+ mongodb:
+ username: admin
+ password: admin
+ database: dlabdb
+ port: 27017
+ host: localhost
+
+server:
+ port: 8088
+ servlet:
+ contextPath: /api/billing
+
+server.ssl.key-store-type: JKS
+server.ssl.key-store: /Users/ofuks/keys/dlabcert/billing.jks
+server.ssl.key-store-password: KEYSTORE_PASSWORD
+server.ssl.key-alias: billing
+
+logging:
+ file: /var/opt/dlab/log/ssn/billing.log
+ level:
+ com:
+ epam: trace
+
+keycloak:
+ bearer-only: true
+ realm: DLAB_bhliva
+ resource: sss
+ credentials.secret: cf5a484b-039b-4161-8707-ad65c0f25962
+ ssl-required: none
+ auth-server-url: http://52.11.45.11:8080/auth
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org