You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by bh...@apache.org on 2019/04/03 09:21:54 UTC
[incubator-dlab] 03/07: DLAB-23 added billing for gcp
This is an automated email from the ASF dual-hosted git repository.
bhliva pushed a commit to branch demo_gcp_billing
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 236e8b64def5a9909a7af1b999137b1bae638c6b
Author: bhliva <bo...@epam.com>
AuthorDate: Thu Mar 14 14:47:56 2019 +0200
DLAB-23 added billing for gcp
---
pom.xml | 18 ++-
services/billing-gcp/billing.yml | 15 --
services/billing-gcp/pom.xml | 109 ++++++--------
.../dlab/billing/gcp/BillingApplicationGcp.java | 24 ---
.../dlab/billing/gcp/BillingGcpApplication.java | 36 +++++
.../com/epam/dlab/billing/gcp/dao/BillingDAO.java | 29 ++++
.../billing/gcp/dao/impl/BigQueryBillingDAO.java | 78 ++++++++++
.../com/epam/dlab/billing/gcp/documents/Edge.java | 32 ++++
.../dlab/billing/gcp/documents/UserInstance.java | 54 +++++++
.../epam/dlab/billing/gcp/model/BillingData.java | 69 +++++++++
.../dlab/billing/gcp/model/GcpBillingData.java | 38 +++++
.../billing/gcp/repository/BillingRepository.java | 26 ++++
.../billing/gcp/repository/EdgeRepository.java | 26 ++++
.../gcp/repository/UserInstanceRepository.java | 30 ++++
.../billing/gcp/scheduler/BillingScheduler.java | 44 ++++++
.../dlab/billing/gcp/service/BillingService.java | 25 ++++
.../billing/gcp/service/BillingServiceImpl.java | 164 +++++++++++++++++++++
.../epam/dlab/billing/gcp/util/BillingUtils.java | 97 ++++++++++++
.../billing-gcp/src/main/resources/application.yml | 12 ++
19 files changed, 818 insertions(+), 108 deletions(-)
diff --git a/pom.xml b/pom.xml
index 537de40..6d20c4b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -35,25 +35,31 @@ limitations under the License.
<profiles>
<profile>
<id>aws</id>
+ <modules>
+ <module>services/billing-aws</module>
+ <module>services/security-aws</module>
+ </modules>
</profile>
<profile>
<id>azure</id>
+ <modules>
+ <module>services/billing-azure</module>
+ <module>services/security-azure</module>
+ </modules>
</profile>
<profile>
<id>gcp</id>
+ <modules>
+ <module>services/billing-gcp</module>
+ <module>services/security-gcp</module>
+ </modules>
</profile>
</profiles>
<modules>
<module>services/common</module>
- <module>services/billing-aws</module>
<module>services/provisioning-service</module>
<module>services/self-service</module>
- <module>services/billing-azure</module>
- <module>services/billing-gcp</module>
- <module>services/security-azure</module>
- <module>services/security-gcp</module>
- <module>services/security-aws</module>
<module>services/security-service</module>
<module>services/dlab-process</module>
<module>services/dlab-auth-common</module>
diff --git a/services/billing-gcp/billing.yml b/services/billing-gcp/billing.yml
deleted file mode 100644
index 633ecda..0000000
--- a/services/billing-gcp/billing.yml
+++ /dev/null
@@ -1,15 +0,0 @@
-#/*
-# * Copyright (c) 2017, EPAM SYSTEMS INC
-# *
-# * Licensed under the Apache License, Version 2.0 (the "License");
-# * you may not use this file except in compliance with the License.
-# * You may obtain a copy of the License at
-# *
-# * http://www.apache.org/licenses/LICENSE-2.0
-# *
-# * Unless required by applicable law or agreed to in writing, software
-# * distributed under the License is distributed on an "AS IS" BASIS,
-# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# * See the License for the specific language governing permissions and
-# * limitations under the License.
-# */
diff --git a/services/billing-gcp/pom.xml b/services/billing-gcp/pom.xml
index fb831cb..928b77f 100644
--- a/services/billing-gcp/pom.xml
+++ b/services/billing-gcp/pom.xml
@@ -1,88 +1,71 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
- ~ Copyright (c) 2017, EPAM SYSTEMS INC
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
~
- ~ Licensed under the Apache License, Version 2.0 (the "License");
- ~ you may not use this file except in compliance with the License.
- ~ You may obtain a copy of the License at
+ ~ http://www.apache.org/licenses/LICENSE-2.0
~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
<parent>
- <artifactId>dlab</artifactId>
<groupId>com.epam.dlab</groupId>
+ <artifactId>dlab</artifactId>
<version>1.0</version>
<relativePath>../../pom.xml</relativePath>
</parent>
- <modelVersion>4.0.0</modelVersion>
-
<artifactId>billing-gcp</artifactId>
+ <name>billing-gcp</name>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-dependencies</artifactId>
+ <version>2.1.3.RELEASE</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
<dependencies>
<dependency>
- <groupId>com.epam.dlab</groupId>
- <artifactId>common</artifactId>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-data-mongodb</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-web</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.cloud</groupId>
+ <artifactId>google-cloud-bigquery</artifactId>
+ <version>1.59.0</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
- <artifactId>maven-shade-plugin</artifactId>
- <version>${maven-shade-plugin.version}</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <createDependencyReducedPom>false</createDependencyReducedPom>
- <minimizeJar>true</minimizeJar>
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
- </filters>
- <transformers>
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass>com.epam.dlab.billing.gcp.BillingApplicationGcp</mainClass>
- <manifestEntries>
- <Created-By><EPAM> Systems</Created-By>
- <Name>com/epam/dlab</Name>
- <Implementation-Title>DLab GCP Billing Module</Implementation-Title>
- <Implementation-Version>${dlab.version}</Implementation-Version>
- <Implementation-Vendor><EPAM> Systems</Implementation-Vendor>
- <Build-Time>${maven.build.timestamp}</Build-Time>
- <Build-OS>${os.name}</Build-OS>
- <GIT-Branch>${scmBranch}</GIT-Branch>
- <GIT-Commit>${buildNumber}</GIT-Commit>
- </manifestEntries>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
-
-</project>
\ No newline at end of file
+
+</project>
diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/BillingApplicationGcp.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/BillingApplicationGcp.java
deleted file mode 100644
index f5c7921..0000000
--- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/BillingApplicationGcp.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright (c) 2017, EPAM SYSTEMS INC
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.epam.dlab.billing.gcp;
-
-public class BillingApplicationGcp {
-
- public static void main(String[] args) {
-
- }
-}
diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/BillingGcpApplication.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/BillingGcpApplication.java
new file mode 100644
index 0000000..74310b1
--- /dev/null
+++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/BillingGcpApplication.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.billing.gcp;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.data.mongodb.repository.config.EnableMongoRepositories;
+import org.springframework.scheduling.annotation.EnableScheduling;
+
+@SpringBootApplication
+@EnableScheduling
+@EnableMongoRepositories
+public class BillingGcpApplication {
+
+ public static void main(String[] args) {
+ SpringApplication.run(BillingGcpApplication.class, args);
+ }
+
+}
diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/BillingDAO.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/BillingDAO.java
new file mode 100644
index 0000000..99ed367
--- /dev/null
+++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/BillingDAO.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.billing.gcp.dao;
+
+import com.epam.dlab.billing.gcp.model.GcpBillingData;
+
+import java.util.List;
+
+public interface BillingDAO {
+
+ List<GcpBillingData> getBillingData(String ssnBaseName) throws InterruptedException;
+}
diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/impl/BigQueryBillingDAO.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/impl/BigQueryBillingDAO.java
new file mode 100644
index 0000000..357e5ac
--- /dev/null
+++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/impl/BigQueryBillingDAO.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.billing.gcp.dao.impl;
+
+import com.epam.dlab.billing.gcp.dao.BillingDAO;
+import com.epam.dlab.billing.gcp.model.GcpBillingData;
+import com.google.cloud.bigquery.*;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.ZoneId;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+
+@Component
+public class BigQueryBillingDAO implements BillingDAO {
+
+ public final String tableName;
+
+ private static final String QUERY = "SELECT b.sku.description usageType,TIMESTAMP_TRUNC(usage_start_time, DAY, 'UTC') usage_date_from, TIMESTAMP_TRUNC(usage_end_time, DAY, 'UTC') usage_date_to, sum(b.cost) cost, b.service.description product, label.value, currency\n" +
+ "FROM `" + "%s" + "` b\n" +
+ "CROSS JOIN UNNEST(b.labels) as label\n" +
+ "where label.key = 'name' and cost != 0 and label.value like @ssnBaseName\n" +
+ "group by usageType, usage_date_from, usage_date_to, product, value, currency";
+ private final BigQuery service = BigQueryOptions.getDefaultInstance().getService();
+
+ public BigQueryBillingDAO(@Value("${tableName}") String tableName) {
+ this.tableName = tableName;
+ }
+
+ @Override
+ public List<GcpBillingData> getBillingData(String ssnBaseName) throws InterruptedException {
+ QueryJobConfiguration queryConfig = QueryJobConfiguration.newBuilder(String.format(QUERY, tableName))
+ .addNamedParameter("ssnBaseName", QueryParameterValue.string(ssnBaseName + "%"))
+ .addNamedParameter("dataset", QueryParameterValue.string(tableName))
+ .build();
+ return StreamSupport.stream(service.query(queryConfig).getValues().spliterator(), false)
+ .map(this::toBillingData)
+ .collect(Collectors.toList());
+ }
+
+ private GcpBillingData toBillingData(FieldValueList fields) {
+
+ return GcpBillingData.builder()
+ .usageDateFrom(toLocalDate(fields, "usage_date_from"))
+ .usageDateTo(toLocalDate(fields, "usage_date_to"))
+ .cost(fields.get("cost").getNumericValue())
+ .product(fields.get("product").getStringValue())
+ .usageType(fields.get("usageType").getStringValue())
+ .currency(fields.get("currency").getStringValue())
+ .tag(fields.get("value").getStringValue()).build();
+ }
+
+ private LocalDate toLocalDate(FieldValueList fieldValues, String timestampFieldName) {
+ return LocalDate.from(Instant.ofEpochMilli(fieldValues.get(timestampFieldName).getTimestampValue() / 1000)
+ .atZone(ZoneId.systemDefault()));
+ }
+}
diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/documents/Edge.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/documents/Edge.java
new file mode 100644
index 0000000..26c0a58
--- /dev/null
+++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/documents/Edge.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.billing.gcp.documents;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Data;
+import org.springframework.data.mongodb.core.mapping.Document;
+
+@Document(collection = "userCloudCredentials")
+@Data
+public class Edge {
+
+ @JsonProperty("_id")
+ private String id;
+}
diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/documents/UserInstance.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/documents/UserInstance.java
new file mode 100644
index 0000000..b6680a0
--- /dev/null
+++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/documents/UserInstance.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.billing.gcp.documents;
+
+import com.epam.dlab.billing.gcp.model.BillingData;
+import lombok.Data;
+import org.springframework.data.annotation.Id;
+import org.springframework.data.mongodb.core.mapping.Document;
+import org.springframework.data.mongodb.core.mapping.Field;
+
+import java.util.List;
+
+@Document(collection = "userInstances")
+@Data
+public class UserInstance {
+
+ @Id
+ private String id;
+ @Field("user")
+ private String user;
+ @Field("exploratory_name")
+ private String exploratoryName;
+ @Field("exploratory_id")
+ private String exploratoryId;
+ private List<BillingData> billing;
+ private Double cost;
+ @Field("computational_resources")
+ private List<ComputationalResource> computationalResources;
+
+ @Data
+ public class ComputationalResource {
+ @Field("computational_name")
+ private String computationalName;
+ @Field("computational_id")
+ private String computationalId;
+ }
+}
diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/model/BillingData.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/model/BillingData.java
new file mode 100644
index 0000000..1039e8b
--- /dev/null
+++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/model/BillingData.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.billing.gcp.model;
+
+import lombok.Builder;
+import lombok.Data;
+import org.springframework.data.annotation.Id;
+import org.springframework.data.mongodb.core.mapping.Document;
+import org.springframework.data.mongodb.core.mapping.Field;
+
+import java.time.LocalDate;
+
+@Data
+@Builder
+@Document(collection = "billing")
+public class BillingData {
+ @Id
+ private String id;
+ private String user;
+ @Field("resource_name")
+ private String displayName;
+ private String resourceName;
+ @Field("usage_date_start")
+ private LocalDate usageDateFrom;
+ @Field("usage_date_end")
+ private LocalDate usageDateTo;
+ @Field("usage_date")
+ private String usageDate;
+ private String product;
+ private String usageType;
+ private Double cost;
+ @Field("currency_code")
+ private String currency;
+ private String exploratoryName;
+ private String computationalName;
+ @Field("dlab_id")
+ private String dlabId;
+ @Field("dlab_resource_type")
+ private ResourceType resourceType;
+
+
+ public enum ResourceType {
+ EDGE,
+ SSN,
+ SHARED_BUCKET,
+ SSN_BUCKET,
+ EDGE_BUCKET,
+ VOLUME,
+ EXPLORATORY,
+ COMPUTATIONAL
+ }
+}
diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/model/GcpBillingData.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/model/GcpBillingData.java
new file mode 100644
index 0000000..a11dcce
--- /dev/null
+++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/model/GcpBillingData.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.billing.gcp.model;
+
+import lombok.Builder;
+import lombok.Data;
+
+import java.math.BigDecimal;
+import java.time.LocalDate;
+
+@Data
+@Builder
+public class GcpBillingData {
+ private final LocalDate usageDateFrom;
+ private final LocalDate usageDateTo;
+ private final String product;
+ private final String usageType;
+ private final BigDecimal cost;
+ private final String currency;
+ private final String tag;
+}
diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/repository/BillingRepository.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/repository/BillingRepository.java
new file mode 100644
index 0000000..5ae66e2
--- /dev/null
+++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/repository/BillingRepository.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.billing.gcp.repository;
+
+import com.epam.dlab.billing.gcp.model.BillingData;
+import org.springframework.data.mongodb.repository.MongoRepository;
+
+public interface BillingRepository extends MongoRepository<BillingData, String> {
+}
diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/repository/EdgeRepository.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/repository/EdgeRepository.java
new file mode 100644
index 0000000..9622fa7
--- /dev/null
+++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/repository/EdgeRepository.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.billing.gcp.repository;
+
+import com.epam.dlab.billing.gcp.documents.Edge;
+import org.springframework.data.mongodb.repository.MongoRepository;
+
+public interface EdgeRepository extends MongoRepository<Edge, String> {
+}
diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/repository/UserInstanceRepository.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/repository/UserInstanceRepository.java
new file mode 100644
index 0000000..a95d033
--- /dev/null
+++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/repository/UserInstanceRepository.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.billing.gcp.repository;
+
+import com.epam.dlab.billing.gcp.documents.UserInstance;
+import org.springframework.data.mongodb.repository.MongoRepository;
+
+import java.util.Optional;
+
+public interface UserInstanceRepository extends MongoRepository<UserInstance, String> {
+
+ Optional<UserInstance> findByUserAndExploratoryName(String user, String exploratoryName);
+}
diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/scheduler/BillingScheduler.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/scheduler/BillingScheduler.java
new file mode 100644
index 0000000..f649d4c
--- /dev/null
+++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/scheduler/BillingScheduler.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.billing.gcp.scheduler;
+
+import com.epam.dlab.billing.gcp.service.BillingService;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+@Component
+@Slf4j
+public class BillingScheduler {
+
+ private final BillingService billingService;
+
+ @Autowired
+ public BillingScheduler(BillingService billingService) {
+ this.billingService = billingService;
+ }
+
+
+ @Scheduled(cron = "${cron}")
+ public void getBillingReport() {
+ billingService.updateBillingData();
+ }
+}
diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/service/BillingService.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/service/BillingService.java
new file mode 100644
index 0000000..71015aa
--- /dev/null
+++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/service/BillingService.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.billing.gcp.service;
+
+public interface BillingService {
+
+ void updateBillingData();
+}
diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/service/BillingServiceImpl.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/service/BillingServiceImpl.java
new file mode 100644
index 0000000..3c81b60
--- /dev/null
+++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/service/BillingServiceImpl.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.billing.gcp.service;
+
+import com.epam.dlab.billing.gcp.dao.BillingDAO;
+import com.epam.dlab.billing.gcp.documents.Edge;
+import com.epam.dlab.billing.gcp.documents.UserInstance;
+import com.epam.dlab.billing.gcp.model.BillingData;
+import com.epam.dlab.billing.gcp.model.GcpBillingData;
+import com.epam.dlab.billing.gcp.repository.BillingRepository;
+import com.epam.dlab.billing.gcp.repository.EdgeRepository;
+import com.epam.dlab.billing.gcp.repository.UserInstanceRepository;
+import com.epam.dlab.billing.gcp.util.BillingUtils;
+import com.google.common.collect.Iterables;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.data.mongodb.core.MongoTemplate;
+import org.springframework.data.mongodb.core.query.Query;
+import org.springframework.data.mongodb.core.query.Update;
+import org.springframework.stereotype.Service;
+
+import java.math.BigDecimal;
+import java.time.format.DateTimeFormatter;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collector;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static com.epam.dlab.billing.gcp.util.BillingUtils.edgeBillingDataStream;
+import static org.springframework.data.mongodb.core.query.Criteria.where;
+
+@Service
+@Slf4j
+public class BillingServiceImpl implements BillingService {
+
+ private final BillingDAO billingDAO;
+ private final EdgeRepository edgeRepository;
+ private final UserInstanceRepository userInstanceRepository;
+ private final BillingRepository billingRepository;
+ private final MongoTemplate mongoTemplate;
+ @Value("${ssnBaseName}")
+ private String ssnBaseName;
+
+ @Autowired
+ public BillingServiceImpl(BillingDAO billingDAO, EdgeRepository edgeRepository,
+ UserInstanceRepository userInstanceRepository, BillingRepository billingRepository,
+ MongoTemplate mongoTemplate) {
+ this.billingDAO = billingDAO;
+ this.edgeRepository = edgeRepository;
+ this.userInstanceRepository = userInstanceRepository;
+ this.billingRepository = billingRepository;
+ this.mongoTemplate = mongoTemplate;
+ }
+
+ @Override
+ public void updateBillingData() {
+ try {
+
+ final Stream<BillingData> ssnBillingDataStream = BillingUtils.ssnBillingDataStream(ssnBaseName);
+ final Stream<BillingData> billableUserInstances = userInstanceRepository.findAll()
+ .stream()
+ .flatMap(BillingUtils::exploratoryBillingDataStream);
+
+ final Stream<BillingData> billableEdges = edgeRepository.findAll()
+ .stream()
+ .map(Edge::getId)
+ .flatMap(e -> edgeBillingDataStream(e, ssnBaseName));
+
+ final Map<String, BillingData> billableResources = Stream.of(billableUserInstances, billableEdges, ssnBillingDataStream)
+ .flatMap(s -> s)
+ .collect(Collectors.toMap(BillingData::getDlabId, b -> b));
+ log.trace("Billable resources are: {}", billableResources);
+ final List<BillingData> billingDataList = billingDAO.getBillingData(ssnBaseName)
+ .stream()
+ .map(bd -> toBillingData(bd, getOrDefault(billableResources, bd.getTag())))
+ .collect(Collectors.toList());
+
+ billingRepository.insert(billingDataList);
+ updateExploratoryCost(billingDataList);
+
+
+ } catch (Exception e) {
+ log.error("Can not update billing due to: {}", e.getMessage());
+ }
+ }
+
+ private BillingData getOrDefault(Map<String, BillingData> billableResources, String tag) {
+ return billableResources.getOrDefault(tag, BillingData.builder().dlabId(tag).build());
+ }
+
+ private void updateExploratoryCost(List<BillingData> billingDataList) {
+ billingDataList.stream()
+ .filter(this::userAndExploratoryNamePresent)
+ .collect(groupByUserNameExploratoryNameCollector())
+ .forEach(this::updateUserExploratoryBillingData);
+ }
+
+ private void updateUserExploratoryBillingData(String user, Map<String, List<BillingData>> billableExploratoriesMap) {
+ billableExploratoriesMap.forEach((exploratoryName, billingInfoList) ->
+ updateExploratoryBillingData(user, exploratoryName, billingInfoList)
+ );
+ }
+
+ private Collector<BillingData, ?, Map<String, Map<String, List<BillingData>>>> groupByUserNameExploratoryNameCollector() {
+ return Collectors.groupingBy(BillingData::getUser, Collectors.groupingBy(BillingData::getExploratoryName));
+ }
+
+ private boolean userAndExploratoryNamePresent(BillingData bd) {
+ return Objects.nonNull(bd.getUser()) && Objects.nonNull(bd.getExploratoryName());
+ }
+
+ private void updateExploratoryBillingData(String user, String exploratoryName, List<BillingData> billingInfoList) {
+ userInstanceRepository.findByUserAndExploratoryName(user, exploratoryName).ifPresent(userInstance ->
+ mongoTemplate.updateFirst(Query.query(where("user").is(user).and("exploratory_name").is(exploratoryName)),
+ Update.update("cost", getTotalCost(billingInfoList)).set("billing", billingInfoList), UserInstance.class));
+ }
+
+ private double getTotalCost(List<BillingData> billingInfoList) {
+ return new BigDecimal(billingInfoList.stream().mapToDouble(BillingData::getCost).sum())
+ .setScale(2, BigDecimal.ROUND_HALF_UP)
+ .doubleValue();
+
+ }
+
+ private BillingData toBillingData(GcpBillingData bd, BillingData billableResource) {
+
+ return BillingData.builder()
+ .displayName(billableResource.getDisplayName())
+ .cost(bd.getCost().doubleValue())
+ .currency(bd.getCurrency())
+ .product(bd.getProduct())
+ .usageDateTo(bd.getUsageDateTo())
+ .usageDateFrom(bd.getUsageDateFrom())
+ .usageDate(bd.getUsageDateFrom().format((DateTimeFormatter.ofPattern("yyyy-MM-dd"))))
+ .usageType(bd.getUsageType())
+ .user(billableResource.getUser())
+ .exploratoryName(billableResource.getExploratoryName())
+ .computationalName(billableResource.getComputationalName())
+ .dlabId(bd.getTag())
+ .resourceType(billableResource.getResourceType())
+ .build();
+ }
+}
diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/util/BillingUtils.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/util/BillingUtils.java
new file mode 100644
index 0000000..b3200e0
--- /dev/null
+++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/util/BillingUtils.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.billing.gcp.util;
+
+import com.epam.dlab.billing.gcp.documents.UserInstance;
+import com.epam.dlab.billing.gcp.model.BillingData;
+
+import java.util.UUID;
+import java.util.stream.Stream;
+
+public class BillingUtils {
+
+ private static final String EDGE_FORMAT = "%s-%s-edge";
+ private static final String EDGE_VOLUME_FORMAT = "%s-%s-edge-volume-primary";
+ private static final String EDGE_BUCKET_FORMAT = "%s-%s-bucket";
+ private static final String VOLUME_PRIMARY_FORMAT = "%s-volume-primary";
+ private static final String VOLUME_SECONDARY_FORMAT = "%s-volume-secondary";
+ private static final String VOLUME_PRIMARY = "Volume primary";
+ private static final String VOLUME_SECONDARY = "Volume secondary";
+ private static final String SHARED_RESOURCE = "Shared resource";
+ private static final String SSN_BUCKET_FORMAT = "%s-bucket";
+
+ public static Stream<BillingData> edgeBillingDataStream(String user, String sbn) {
+ final String adjustedUserName = adjustUserName(10, user);
+ final String userEdgeId = String.format(EDGE_FORMAT, sbn, adjustedUserName);
+ final String edgeVolumeId = String.format(EDGE_VOLUME_FORMAT, sbn, adjustedUserName);
+ final String edgeBucketId = String.format(EDGE_BUCKET_FORMAT, sbn, adjustedUserName);
+ return Stream.of(
+ BillingData.builder().displayName("EDGE node").user(user).dlabId(userEdgeId).resourceType(BillingData.ResourceType.EDGE).build(),
+ BillingData.builder().displayName("EDGE volume").user(user).dlabId(edgeVolumeId).resourceType(BillingData.ResourceType.VOLUME).build(),
+ BillingData.builder().displayName("EDGE bucket").user(user).dlabId(edgeBucketId).resourceType(BillingData.ResourceType.EDGE_BUCKET).build()
+ );
+ }
+
+ public static Stream<BillingData> ssnBillingDataStream(String sbn) {
+ final String ssnId = sbn + "-ssn";
+ final String bucketName = sbn.replaceAll("_", "-");
+ return Stream.of(
+ BillingData.builder().user(SHARED_RESOURCE).displayName("SSN").dlabId(ssnId).resourceType(BillingData.ResourceType.SSN).build(),
+ BillingData.builder().user(SHARED_RESOURCE).displayName("SSN Volume").dlabId(String.format(VOLUME_PRIMARY_FORMAT, ssnId)).resourceType(BillingData.ResourceType.VOLUME).build(),
+ BillingData.builder().user(SHARED_RESOURCE).displayName("SSN bucket").dlabId(bucketName + "-ssn-bucket").resourceType(BillingData.ResourceType.SSN_BUCKET).build(),
+ BillingData.builder().user(SHARED_RESOURCE).displayName("Collaboration bucket").dlabId(bucketName + "-shared-bucket").resourceType(BillingData.ResourceType.SHARED_BUCKET).build()
+ );
+ }
+
+ public static Stream<BillingData> exploratoryBillingDataStream(UserInstance userInstance) {
+ final Stream<BillingData> computationalStream = userInstance.getComputationalResources()
+ .stream()
+ .flatMap(cr -> Stream.of(computationalBillableResource(userInstance, cr)));
+ final String exploratoryId = userInstance.getExploratoryId();
+ final String primaryVolumeId = String.format(VOLUME_PRIMARY_FORMAT, exploratoryId);
+ final String secondaryVolumeId = String.format(VOLUME_SECONDARY_FORMAT, exploratoryId);
+ final Stream<BillingData> exploratoryStream = Stream.of(
+ withExploratoryName(userInstance).displayName(userInstance.getExploratoryName()).dlabId(exploratoryId).resourceType(BillingData.ResourceType.EXPLORATORY).build(),
+ withExploratoryName(userInstance).displayName(VOLUME_PRIMARY).dlabId(primaryVolumeId).resourceType(BillingData.ResourceType.VOLUME).build(),
+ withExploratoryName(userInstance).displayName(VOLUME_SECONDARY).dlabId(secondaryVolumeId).resourceType(BillingData.ResourceType.VOLUME).build());
+ return Stream.concat(computationalStream, exploratoryStream);
+ }
+
+ private static BillingData computationalBillableResource(UserInstance userInstance, UserInstance.ComputationalResource cr) {
+ return withExploratoryName(userInstance)
+ .dlabId(cr.getComputationalId())
+ .displayName(cr.getComputationalName())
+ .resourceType(BillingData.ResourceType.COMPUTATIONAL)
+ .computationalName(cr.getComputationalName())
+ .build();
+ }
+
+ private static BillingData.BillingDataBuilder withExploratoryName(UserInstance userInstance) {
+ return BillingData.builder().user(userInstance.getUser()).exploratoryName(userInstance.getExploratoryName());
+ }
+
+ private static String adjustUserName(int maxLength, String userName) {
+ String newName = userName.replaceAll("\\s", "_").replaceAll("@.*", "");
+
+ return newName.length() > maxLength ?
+ UUID.nameUUIDFromBytes(newName.getBytes()).toString().substring(0, maxLength) : newName;
+ }
+
+}
diff --git a/services/billing-gcp/src/main/resources/application.yml b/services/billing-gcp/src/main/resources/application.yml
new file mode 100644
index 0000000..be25237
--- /dev/null
+++ b/services/billing-gcp/src/main/resources/application.yml
@@ -0,0 +1,12 @@
+spring:
+ data:
+ mongodb:
+ username: admin
+ password: admin
+ database: DLAB
+ port: 27017
+ host: localhost
+
+ssnBaseName: it78
+tableName: test_dataset.test
+cron: 0 * * * * *
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org