You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by of...@apache.org on 2020/03/16 15:40:23 UTC
[incubator-dlab] 06/09: Conveyed billing for Azure remote endpoint
This is an automated email from the ASF dual-hosted git repository.
ofuks pushed a commit to branch DLAB-1571
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 667524ffdfb44984b441081bb6c40bec03408914
Author: Oleh Fuks <ol...@gmail.com>
AuthorDate: Mon Mar 16 13:33:15 2020 +0200
Conveyed billing for Azure remote endpoint
---
services/billing-azure/pom.xml | 109 ++++++++++++---------
.../billing/azure/BillingAzureApplication.java | 36 +++++++
.../billing/azure/config/DlabConfiguration.java | 47 +++++++++
.../dlab/billing/azure/config/SecurityConfig.java | 66 +++++++++++++
.../azure/controller/BillingController.java | 50 ++++++++++
.../epam/dlab/billing/azure/dao/BillingDAO.java | 31 ++++++
.../billing/azure/dao/impl/BillingDAOImpl.java | 109 +++++++++++++++++++++
.../src/main/resources/application.yml | 36 +++++++
8 files changed, 438 insertions(+), 46 deletions(-)
diff --git a/services/billing-azure/pom.xml b/services/billing-azure/pom.xml
index 41edc3c..75c8e35 100644
--- a/services/billing-azure/pom.xml
+++ b/services/billing-azure/pom.xml
@@ -28,16 +28,65 @@
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
-
<artifactId>billing-azure</artifactId>
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-dependencies</artifactId>
+ <version>2.1.3.RELEASE</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.keycloak.bom</groupId>
+ <artifactId>keycloak-adapter-bom</artifactId>
+ <version>4.8.3.Final</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
<dependencies>
<dependency>
- <groupId>com.microsoft.azure</groupId>
- <artifactId>azure-client-authentication</artifactId>
- <version>1.2.1</version>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-configuration-processor</artifactId>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-data-mongodb</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-web</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-security</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.keycloak</groupId>
+ <artifactId>keycloak-spring-boot-starter</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.keycloak</groupId>
+ <artifactId>keycloak-spring-security-adapter</artifactId>
+ <version>4.8.3.Final</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <version>${org.mockito.version}</version>
+ <scope>test</scope>
</dependency>
-
<dependency>
<groupId>com.epam.dlab</groupId>
<artifactId>dlab-model</artifactId>
@@ -45,6 +94,12 @@
</dependency>
<dependency>
+ <groupId>com.microsoft.azure</groupId>
+ <artifactId>azure-client-authentication</artifactId>
+ <version>1.2.1</version>
+ </dependency>
+
+ <dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongo-java-driver</artifactId>
<version>${org.mongodb.version}</version>
@@ -82,51 +137,13 @@
<build>
<plugins>
<plugin>
- <artifactId>maven-shade-plugin</artifactId>
- <version>${maven-shade-plugin.version}</version>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-maven-plugin</artifactId>
<executions>
<execution>
- <phase>package</phase>
<goals>
- <goal>shade</goal>
+ <goal>repackage</goal>
</goals>
- <configuration>
- <createDependencyReducedPom>false</createDependencyReducedPom>
- <minimizeJar>true</minimizeJar>
- <filters>
- <filter>
- <artifact>commons-logging:commons-logging</artifact>
- <includes>**</includes>
- </filter>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
- </filters>
- <transformers>
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass>com.epam.dlab.billing.azure.BillingSchedulerAzure</mainClass>
- <manifestEntries>
- <Created-By><EPAM> Systems</Created-By>
- <Name>com/epam/dlab</Name>
- <Implementation-Title>DLab Azure Billing Module</Implementation-Title>
- <Implementation-Version>${dlab.version}</Implementation-Version>
- <Implementation-Vendor><EPAM> Systems</Implementation-Vendor>
- <Build-Time>${maven.build.timestamp}</Build-Time>
- <Build-OS>${os.name}</Build-OS>
- <GIT-Branch>${scmBranch}</GIT-Branch>
- <GIT-Commit>${buildNumber}</GIT-Commit>
- </manifestEntries>
- </transformer>
- </transformers>
- </configuration>
</execution>
</executions>
</plugin>
diff --git a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/BillingAzureApplication.java b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/BillingAzureApplication.java
new file mode 100644
index 0000000..1a40767
--- /dev/null
+++ b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/BillingAzureApplication.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.billing.azure;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.data.mongodb.repository.config.EnableMongoRepositories;
+
+@SpringBootApplication
+@EnableMongoRepositories
+@EnableConfigurationProperties
+public class BillingAzureApplication {
+
+ public static void main(String[] args) {
+ SpringApplication.run(BillingAzureApplication.class, args);
+ }
+
+}
diff --git a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/config/DlabConfiguration.java b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/config/DlabConfiguration.java
new file mode 100644
index 0000000..c1163c0
--- /dev/null
+++ b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/config/DlabConfiguration.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.billing.azure.config;
+
+import lombok.Data;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+@ConfigurationProperties("dlab")
+@Data
+public class DlabConfiguration {
+ private long initialDelay;
+ private long period;
+ private String clientId;
+ private String clientSecret;
+ private String tenantId;
+ private String subscriptionId;
+ private String authenticationFile;
+ private String offerNumber;
+ private String currency;
+ private String locale;
+ private String regionInfo;
+ private boolean billingEnabled;
+ private String ssnStorageAccountTagName;
+ private String sharedStorageAccountTagName;
+ private String datalakeTagName;
+ private LoggingConfigurationFactory logging;
+ private AggregationOutputMongoDataSource aggregationOutputMongoDataSource;
+}
diff --git a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/config/SecurityConfig.java b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/config/SecurityConfig.java
new file mode 100644
index 0000000..35e341c
--- /dev/null
+++ b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/config/SecurityConfig.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.billing.azure.config;
+
+import org.keycloak.adapters.KeycloakConfigResolver;
+import org.keycloak.adapters.springboot.KeycloakSpringBootConfigResolver;
+import org.keycloak.adapters.springsecurity.KeycloakConfiguration;
+import org.keycloak.adapters.springsecurity.authentication.KeycloakAuthenticationProvider;
+import org.keycloak.adapters.springsecurity.config.KeycloakWebSecurityConfigurerAdapter;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.security.config.annotation.authentication.builders.AuthenticationManagerBuilder;
+import org.springframework.security.config.annotation.web.builders.HttpSecurity;
+import org.springframework.security.core.authority.mapping.SimpleAuthorityMapper;
+import org.springframework.security.core.session.SessionRegistryImpl;
+import org.springframework.security.web.authentication.session.RegisterSessionAuthenticationStrategy;
+import org.springframework.security.web.authentication.session.SessionAuthenticationStrategy;
+
+@KeycloakConfiguration
+class SecurityConfig extends KeycloakWebSecurityConfigurerAdapter {
+
+ @Autowired
+ public void configureGlobal(AuthenticationManagerBuilder auth) {
+ KeycloakAuthenticationProvider keycloakAuthenticationProvider = keycloakAuthenticationProvider();
+ keycloakAuthenticationProvider.setGrantedAuthoritiesMapper(new SimpleAuthorityMapper());
+ auth.authenticationProvider(keycloakAuthenticationProvider);
+ }
+
+ @Bean
+ public KeycloakConfigResolver KeycloakConfigResolver() {
+ return new KeycloakSpringBootConfigResolver();
+ }
+
+ @Bean
+ @Override
+ protected SessionAuthenticationStrategy sessionAuthenticationStrategy() {
+ return new RegisterSessionAuthenticationStrategy(new SessionRegistryImpl());
+ }
+
+ @Override
+ protected void configure(HttpSecurity http) throws Exception {
+ super.configure(http);
+ http
+ .anonymous().disable()
+ .authorizeRequests()
+ .anyRequest()
+ .authenticated();
+ }
+}
\ No newline at end of file
diff --git a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/controller/BillingController.java b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/controller/BillingController.java
new file mode 100644
index 0000000..53227a2
--- /dev/null
+++ b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/controller/BillingController.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.billing.azure.controller;
+
+import com.epam.dlab.billing.azure.dao.BillingDAO;
+import com.epam.dlab.dto.billing.BillingData;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.List;
+
+@RestController
+public class BillingController {
+
+ private final BillingDAO billingDAO;
+
+ public BillingController(BillingDAO billingDAO) {
+ this.billingDAO = billingDAO;
+ }
+
+ @GetMapping
+ public ResponseEntity<List<BillingData>> getBilling(@RequestParam List<String> dlabIds) {
+ return new ResponseEntity<>(billingDAO.getBillingReport(dlabIds), HttpStatus.OK);
+ }
+
+ @GetMapping("/report")
+ public ResponseEntity<List<BillingData>> getBilling() {
+ return new ResponseEntity<>(billingDAO.getBillingReport(), HttpStatus.OK);
+ }
+}
diff --git a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/dao/BillingDAO.java b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/dao/BillingDAO.java
new file mode 100644
index 0000000..1f578b1
--- /dev/null
+++ b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/dao/BillingDAO.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.billing.azure.dao;
+
+import com.epam.dlab.dto.billing.BillingData;
+
+import java.util.List;
+
+public interface BillingDAO {
+
+ List<BillingData> getBillingReport();
+
+ List<BillingData> getBillingReport(List<String> dlabIds);
+}
diff --git a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/dao/impl/BillingDAOImpl.java b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/dao/impl/BillingDAOImpl.java
new file mode 100644
index 0000000..34901d0
--- /dev/null
+++ b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/dao/impl/BillingDAOImpl.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.billing.azure.dao.impl;
+
+import com.epam.dlab.billing.azure.dao.BillingDAO;
+import com.epam.dlab.dto.billing.BillingData;
+import com.epam.dlab.exceptions.DlabException;
+import lombok.extern.slf4j.Slf4j;
+import org.bson.Document;
+import org.springframework.data.mongodb.core.MongoTemplate;
+import org.springframework.data.mongodb.core.aggregation.Aggregation;
+import org.springframework.data.mongodb.core.aggregation.GroupOperation;
+import org.springframework.data.mongodb.core.aggregation.MatchOperation;
+import org.springframework.data.mongodb.core.query.Criteria;
+import org.springframework.stereotype.Component;
+
+import java.math.BigDecimal;
+import java.time.LocalDate;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import static com.epam.dlab.model.aws.ReportLine.FIELD_COST;
+import static com.epam.dlab.model.aws.ReportLine.FIELD_CURRENCY_CODE;
+import static com.epam.dlab.model.aws.ReportLine.FIELD_DLAB_ID;
+import static com.epam.dlab.model.aws.ReportLine.FIELD_PRODUCT;
+import static com.epam.dlab.model.aws.ReportLine.FIELD_RESOURCE_TYPE;
+import static com.epam.dlab.model.aws.ReportLine.FIELD_USAGE_DATE;
+import static org.springframework.data.mongodb.core.aggregation.Aggregation.group;
+import static org.springframework.data.mongodb.core.aggregation.Aggregation.newAggregation;
+
+@Component
+@Slf4j
+public class BillingDAOImpl implements BillingDAO {
+ private final MongoTemplate mongoTemplate;
+
+ public BillingDAOImpl(MongoTemplate mongoTemplate) {
+ this.mongoTemplate = mongoTemplate;
+ }
+
+ @Override
+ public List<BillingData> getBillingReport() {
+ try {
+ GroupOperation groupOperation = getGroupOperation();
+ Aggregation aggregation = newAggregation(groupOperation);
+
+ return mongoTemplate.aggregate(aggregation, "billing", Document.class).getMappedResults()
+ .stream()
+ .map(this::toBillingData)
+ .collect(Collectors.toList());
+ } catch (Exception e) {
+ log.error("Cannot retrieve billing information ", e);
+ throw new DlabException("Cannot retrieve billing information", e);
+ }
+ }
+
+ @Override
+ public List<BillingData> getBillingReport(List<String> dlabIds) {
+ try {
+ GroupOperation groupOperation = getGroupOperation();
+ MatchOperation matchOperation = Aggregation.match(Criteria.where("dlab_id").in(dlabIds));
+ Aggregation aggregation = newAggregation(matchOperation, groupOperation);
+
+ return mongoTemplate.aggregate(aggregation, "billing", Document.class).getMappedResults()
+ .stream()
+ .map(this::toBillingData)
+ .collect(Collectors.toList());
+ } catch (Exception e) {
+ log.error("Cannot retrieve billing information ", e);
+ throw new DlabException("Cannot retrieve billing information", e);
+ }
+ }
+
+ private GroupOperation getGroupOperation() {
+ return group(FIELD_PRODUCT, FIELD_CURRENCY_CODE, FIELD_RESOURCE_TYPE, FIELD_DLAB_ID)
+ .min(FIELD_USAGE_DATE).as("from")
+ .max(FIELD_USAGE_DATE).as("to")
+ .sum(FIELD_COST).as(FIELD_COST);
+ }
+
+ private BillingData toBillingData(Document billingData) {
+ return BillingData.builder()
+ .tag(billingData.getString(FIELD_DLAB_ID))
+ .usageDateFrom(Optional.ofNullable(billingData.getString("from")).map(LocalDate::parse).orElse(null))
+ .usageDateTo(Optional.ofNullable(billingData.getString("to")).map(LocalDate::parse).orElse(null))
+ .product(billingData.getString(FIELD_PRODUCT))
+ .usageType(billingData.getString(FIELD_RESOURCE_TYPE))
+ .cost(BigDecimal.valueOf(billingData.getDouble(FIELD_COST)).setScale(3, BigDecimal.ROUND_HALF_UP).doubleValue())
+ .currency(billingData.getString(FIELD_CURRENCY_CODE))
+ .build();
+ }
+}
diff --git a/services/billing-azure/src/main/resources/application.yml b/services/billing-azure/src/main/resources/application.yml
new file mode 100644
index 0000000..52e6862
--- /dev/null
+++ b/services/billing-azure/src/main/resources/application.yml
@@ -0,0 +1,36 @@
+spring:
+ main:
+ allow-bean-definition-overriding: true
+ data:
+ mongodb:
+ username: admin
+ password: admin
+ database: dlabdb
+ port: 27017
+ host: localhost
+dlab:
+
+
+server:
+ port: 8088
+ servlet:
+ contextPath: /api/billing
+
+server.ssl.key-store-type: JKS
+server.ssl.key-store: /Users/ofuks/keys/dlabcert/billing.jks
+server.ssl.key-store-password: KEYSTORE_PASSWORD
+server.ssl.key-alias: billing
+
+logging:
+ file: /var/opt/dlab/log/ssn/billing.log
+ level:
+ com:
+ epam: trace
+
+keycloak:
+ bearer-only: true
+ realm: DLAB_bhliva
+ resource: sss
+ credentials.secret: cf5a484b-039b-4161-8707-ad65c0f25962
+ ssl-required: none
+ auth-server-url: http://52.11.45.11:8080/auth
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org