You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by of...@apache.org on 2020/04/03 15:10:35 UTC
[incubator-dlab] branch DLAB-1571 updated (6b955d3 -> 7572611)
This is an automated email from the ASF dual-hosted git repository.
ofuks pushed a change to branch DLAB-1571
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git.
from 6b955d3 Merge remote-tracking branch 'origin/DLAB-1571' into DLAB-1571
new 23d4388 Remote billing
new 329d6a8 Merge remote-tracking branch 'origin/DLAB-1571' into DLAB-1571
add 1c0282c [odahu-integration] Fixed jupyterlab image link
add e1a5368 [DLAB-1679] Fixed bug with custom image creation
add ac222f6 [DLAB-1689] Changes for group while adding endpoint
new 0a8d98d Merge branch 'develop' into DLAB-1571
new 7572611 Remote billing
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../src/general/conf/dlab.ini | 2 +-
.../dlab/dto/exploratory/ExploratoryImageDTO.java | 7 ++++
.../epam/dlab/backendapi/dao/UserRoleDaoImpl.java | 32 +++++++++++++++-
.../service/impl/BillingServiceImpl.java | 44 +++++++++++-----------
.../service/impl/ImageExploratoryServiceImpl.java | 8 +++-
.../epam/dlab/backendapi/util/BillingUtils.java | 22 +++++++----
.../epam/dlab/backendapi/util/RequestBuilder.java | 5 ++-
.../impl/ImageExploratoryServiceImplTest.java | 24 +++++++++---
8 files changed, 101 insertions(+), 43 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org
[incubator-dlab] 02/04: Merge remote-tracking branch
'origin/DLAB-1571' into DLAB-1571
Posted by of...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ofuks pushed a commit to branch DLAB-1571
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 329d6a8bbdb6bde2f41ddf36760ea1fe2872b578
Merge: 23d4388 6b955d3
Author: Oleh Fuks <ol...@gmail.com>
AuthorDate: Fri Apr 3 17:09:58 2020 +0300
Merge remote-tracking branch 'origin/DLAB-1571' into DLAB-1571
.../resources-grid/resources-grid.component.html | 8 +++++---
.../resources-grid/resources-grid.component.scss | 4 ++++
.../resources-grid/resources-grid.component.ts | 6 ------
.../resources-grid/resources-grid.model.ts | 2 ++
.../src/app/resources/resources.component.html | 2 +-
.../webapp/src/app/resources/resources.component.ts | 11 ++---------
.../multi-level-select-dropdown.component.html | 18 ++++++++++++++----
.../multi-level-select-dropdown.component.scss | 21 ++++++++++-----------
8 files changed, 38 insertions(+), 34 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org
[incubator-dlab] 04/04: Remote billing
Posted by of...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ofuks pushed a commit to branch DLAB-1571
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 757261106f6d51a705f9aa025d22431a9587f6e4
Author: Oleh Fuks <ol...@gmail.com>
AuthorDate: Fri Apr 3 18:10:21 2020 +0300
Remote billing
---
.../epam/dlab/backendapi/dao/UserRoleDaoImpl.java | 2 +-
.../service/impl/BillingServiceImpl.java | 33 +++++++++-------------
.../epam/dlab/backendapi/util/BillingUtils.java | 4 ---
3 files changed, 14 insertions(+), 25 deletions(-)
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/UserRoleDaoImpl.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/UserRoleDaoImpl.java
index 7bd7b94..174e71f 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/UserRoleDaoImpl.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/UserRoleDaoImpl.java
@@ -108,7 +108,7 @@ public class UserRoleDaoImpl extends BaseDAO implements UserRoleDao {
.noneMatch(id -> id.equals(u.getId())))
.forEach(this::insert);
- addGroupToRole(aggregateRolesByGroup()
+ addGroupToRole(aggregateRolesByGroup(false)
.stream()
.map(UserGroupDto::getGroup)
.collect(Collectors.toSet()),
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BillingServiceImpl.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BillingServiceImpl.java
index 794ac33..7d6ab11 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BillingServiceImpl.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BillingServiceImpl.java
@@ -140,7 +140,7 @@ public class BillingServiceImpl implements BillingService {
projects.addAll(projectService.getUserProjects(user, false));
}
- final Map<String, BillingReportLine> billableResources = getBillableResources(user, projects);
+ final Map<String, BillingReportLine> billableResources = getBillableResources(projects);
List<BillingReportLine> billingReport = getRemoteBillingData(user, filter)
.stream()
@@ -153,8 +153,17 @@ public class BillingServiceImpl implements BillingService {
return billingReport;
}
- private Map<String, BillingReportLine> getBillableResources(UserInfo user, Set<ProjectDTO> projects) {
- Stream<BillingReportLine> billableAdminResources = Stream.empty();
+ private Map<String, BillingReportLine> getBillableResources(Set<ProjectDTO> projects) {
+ final Stream<BillingReportLine> ssnBillingDataStream = BillingUtils.ssnBillingDataStream(sbn);
+ final Stream<BillingReportLine> billableEdges = projects
+ .stream()
+ .collect(Collectors.toMap(ProjectDTO::getName, ProjectDTO::getEndpoints))
+ .entrySet()
+ .stream()
+ .flatMap(e -> projectEdges(sbn, e.getKey(), e.getValue()));
+ final Stream<BillingReportLine> billableSharedEndpoints = endpointService.getEndpoints()
+ .stream()
+ .flatMap(endpoint -> BillingUtils.sharedEndpointBillingDataStream(endpoint.getName(), sbn));
final Stream<BillingReportLine> billableUserInstances = exploratoryService.findAll(projects)
.stream()
.filter(userInstance -> Objects.nonNull(userInstance.getExploratoryId()))
@@ -165,23 +174,7 @@ public class BillingServiceImpl implements BillingService {
.flatMap(Collection::stream)
.flatMap(i -> BillingUtils.customImageBillingDataStream(i, sbn));
- if (UserRoles.isAdmin(user)) {
- final Stream<BillingReportLine> ssnBillingDataStream = BillingUtils.ssnBillingDataStream(sbn);
- final Stream<BillingReportLine> billableEdges = projects
- .stream()
- .collect(Collectors.toMap(ProjectDTO::getName, ProjectDTO::getEndpoints))
- .entrySet()
- .stream()
- .flatMap(e -> projectEdges(sbn, e.getKey(), e.getValue()));
- final Stream<BillingReportLine> billableSharedEndpoints = endpointService.getEndpoints()
- .stream()
- .flatMap(endpoint -> BillingUtils.sharedEndpointBillingDataStream(endpoint.getName(), sbn));
-
- billableAdminResources = Stream.of(billableEdges, ssnBillingDataStream, billableSharedEndpoints)
- .flatMap(s -> s);
- }
-
- final Map<String, BillingReportLine> billableResources = Stream.of(billableUserInstances, customImages, billableAdminResources)
+ final Map<String, BillingReportLine> billableResources = Stream.of(ssnBillingDataStream, billableEdges, billableSharedEndpoints, billableUserInstances, customImages)
.flatMap(s -> s)
.collect(Collectors.toMap(BillingReportLine::getDlabId, b -> b));
log.debug("Billable resources are: {}", billableResources);
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java
index cc77ff1..c8454b2 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java
@@ -62,9 +62,7 @@ public class BillingUtils {
private static final String VOLUME_SECONDARY_FORMAT = "%s-volume-secondary";
private static final String VOLUME_SECONDARY_COMPUTATIONAL_FORMAT = "%s-%s-volume-secondary";
private static final String IMAGE_STANDARD_FORMAT1 = "%s-%s-%s-%s-notebook-image";
- private static final String IMAGE_STANDARD_FORMAT2 = "%s-%s-%s-notebook-image";
private static final String IMAGE_CUSTOM_FORMAT = "%s-%s-%s-%s-%s";
- private static final String IMAGE_NAME_PREFIX = "docker.dlab-";
private static final String VOLUME_PRIMARY = "Volume primary";
private static final String VOLUME_SECONDARY = "Volume secondary";
@@ -171,8 +169,6 @@ public class BillingUtils {
for (String notebook : AVAILABLE_NOTEBOOKS) {
list.add(BillingReportLine.builder().resourceName(IMAGE_NAME).dlabId(String.format(IMAGE_STANDARD_FORMAT1, sbn, project, endpoint, notebook))
.project(SHARED_RESOURCE).resourceType(IMAGE).build());
- list.add(BillingReportLine.builder().resourceName(IMAGE_NAME).dlabId(String.format(IMAGE_STANDARD_FORMAT2, sbn, endpoint, notebook))
- .project(SHARED_RESOURCE).resourceType(IMAGE).build());
}
return list.stream();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org
[incubator-dlab] 03/04: Merge branch 'develop' into DLAB-1571
Posted by of...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ofuks pushed a commit to branch DLAB-1571
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 0a8d98d558d1b65464e704f07bbab295d4d4caf3
Merge: 329d6a8 ac222f6
Author: Oleh Fuks <ol...@gmail.com>
AuthorDate: Fri Apr 3 17:10:26 2020 +0300
Merge branch 'develop' into DLAB-1571
.../src/general/conf/dlab.ini | 2 +-
.../dlab/dto/exploratory/ExploratoryImageDTO.java | 7 +++++
.../epam/dlab/backendapi/dao/UserRoleDaoImpl.java | 32 ++++++++++++++++++++--
.../service/impl/ImageExploratoryServiceImpl.java | 8 ++++--
.../epam/dlab/backendapi/util/RequestBuilder.java | 5 ++--
.../impl/ImageExploratoryServiceImplTest.java | 24 ++++++++++++----
6 files changed, 65 insertions(+), 13 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org
[incubator-dlab] 01/04: Remote billing
Posted by of...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ofuks pushed a commit to branch DLAB-1571
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 23d43885609cd8c084da6b39bcd4b4c7217815a7
Author: Oleh Fuks <ol...@gmail.com>
AuthorDate: Fri Apr 3 14:57:07 2020 +0300
Remote billing
---
.../service/impl/BillingServiceImpl.java | 15 ++++++++++-----
.../epam/dlab/backendapi/util/BillingUtils.java | 22 +++++++++++++++++-----
2 files changed, 27 insertions(+), 10 deletions(-)
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BillingServiceImpl.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BillingServiceImpl.java
index b687e62..794ac33 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BillingServiceImpl.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BillingServiceImpl.java
@@ -132,8 +132,13 @@ public class BillingServiceImpl implements BillingService {
@Override
public List<BillingReportLine> getBillingReportLines(UserInfo user, BillingFilter filter) {
setUserFilter(user, filter);
- Set<ProjectDTO> projects = new HashSet<>(projectService.getProjects(user));
- projects.addAll(projectService.getUserProjects(user, false));
+ Set<ProjectDTO> projects;
+ if (isFullReport(user)) {
+ projects = new HashSet<>(projectService.getProjects());
+ } else {
+ projects = new HashSet<>(projectService.getProjects(user));
+ projects.addAll(projectService.getUserProjects(user, false));
+ }
final Map<String, BillingReportLine> billableResources = getBillableResources(user, projects);
@@ -154,20 +159,20 @@ public class BillingServiceImpl implements BillingService {
.stream()
.filter(userInstance -> Objects.nonNull(userInstance.getExploratoryId()))
.flatMap(ui -> BillingUtils.exploratoryBillingDataStream(ui, configuration.getMaxSparkInstanceCount(), sbn));
- final Stream<BillingReportLine> billingReportLineStream = projects
+ final Stream<BillingReportLine> customImages = projects
.stream()
.map(p -> imageExploratoryDao.getImagesForProject(p.getName()))
.flatMap(Collection::stream)
.flatMap(i -> BillingUtils.customImageBillingDataStream(i, sbn));
if (UserRoles.isAdmin(user)) {
+ final Stream<BillingReportLine> ssnBillingDataStream = BillingUtils.ssnBillingDataStream(sbn);
final Stream<BillingReportLine> billableEdges = projects
.stream()
.collect(Collectors.toMap(ProjectDTO::getName, ProjectDTO::getEndpoints))
.entrySet()
.stream()
.flatMap(e -> projectEdges(sbn, e.getKey(), e.getValue()));
- final Stream<BillingReportLine> ssnBillingDataStream = BillingUtils.ssnBillingDataStream(sbn);
final Stream<BillingReportLine> billableSharedEndpoints = endpointService.getEndpoints()
.stream()
.flatMap(endpoint -> BillingUtils.sharedEndpointBillingDataStream(endpoint.getName(), sbn));
@@ -176,7 +181,7 @@ public class BillingServiceImpl implements BillingService {
.flatMap(s -> s);
}
- final Map<String, BillingReportLine> billableResources = Stream.of(billableUserInstances, billingReportLineStream, billableAdminResources)
+ final Map<String, BillingReportLine> billableResources = Stream.of(billableUserInstances, customImages, billableAdminResources)
.flatMap(s -> s)
.collect(Collectors.toMap(BillingReportLine::getDlabId, b -> b));
log.debug("Billable resources are: {}", billableResources);
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java
index 2c5b87f..cc77ff1 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java
@@ -47,6 +47,7 @@ import static com.epam.dlab.dto.billing.BillingResourceType.SSN;
import static com.epam.dlab.dto.billing.BillingResourceType.VOLUME;
public class BillingUtils {
+ private static final String[] AVAILABLE_NOTEBOOKS = {"zeppelin", "tensor-rstudio", "rstudio", "tensor", "superset", "jupyterlab", "jupyter", "deeplearning"};
private static final String[] REPORT_HEADERS = {"DLab ID", "User", "Project", "DLab Resource Type", "Shape", "Product", "Cost"};
private static final String REPORT_FIRST_LINE = "Service base name: %s. Available reporting period from: %s to: %s";
private static final String TOTAL_LINE = "Total: %s %s";
@@ -78,10 +79,12 @@ public class BillingUtils {
final String edgeVolumeId = String.format(EDGE_VOLUME_FORMAT, sbn, project.toLowerCase(), endpoint);
final String endpointBucketId = String.format(PROJECT_ENDPOINT_BUCKET_FORMAT, sbn, project.toLowerCase(), endpoint);
- return Stream.of(
+ return Stream.concat(Stream.of(
BillingReportLine.builder().resourceName("EDGE node").user(SHARED_RESOURCE).project(project).dlabId(userEdgeId).resourceType(EDGE).status(UserInstanceStatus.of(status)).build(),
BillingReportLine.builder().resourceName("EDGE volume").user(SHARED_RESOURCE).project(project).dlabId(edgeVolumeId).resourceType(VOLUME).build(),
BillingReportLine.builder().resourceName("Project endpoint shared bucket").user(SHARED_RESOURCE).project(project).dlabId(endpointBucketId).resourceType(BUCKET).build()
+ ),
+ standardImageBillingDataStream(sbn, project, endpoint)
);
}
@@ -117,16 +120,13 @@ public class BillingUtils {
getSlaveVolumes(userInstance, cr, maxSparkInstanceCount)
));
final String exploratoryId = userInstance.getExploratoryId();
- final String imageId1 = String.format(IMAGE_STANDARD_FORMAT1, sbn, userInstance.getProject(), userInstance.getEndpoint(), userInstance.getImageName().replace(IMAGE_NAME_PREFIX, ""));
- final String imageId2 = String.format(IMAGE_STANDARD_FORMAT2, sbn, userInstance.getEndpoint(), userInstance.getImageName().replace(IMAGE_NAME_PREFIX, ""));
final String primaryVolumeId = String.format(VOLUME_PRIMARY_FORMAT, exploratoryId);
final String secondaryVolumeId = String.format(VOLUME_SECONDARY_FORMAT, exploratoryId);
final Stream<BillingReportLine> exploratoryStream = Stream.of(
withUserProject(userInstance).resourceName(userInstance.getExploratoryName()).dlabId(exploratoryId).resourceType(EXPLORATORY).status(UserInstanceStatus.of(userInstance.getStatus())).shape(userInstance.getShape()).build(),
- BillingReportLine.builder().resourceName(IMAGE_NAME).dlabId(imageId1).project(SHARED_RESOURCE).resourceType(IMAGE).build(),
- BillingReportLine.builder().resourceName(IMAGE_NAME).dlabId(imageId2).project(SHARED_RESOURCE).resourceType(IMAGE).build(),
withUserProject(userInstance).resourceName(VOLUME_PRIMARY).dlabId(primaryVolumeId).resourceType(VOLUME).build(),
withUserProject(userInstance).resourceName(VOLUME_SECONDARY).dlabId(secondaryVolumeId).resourceType(VOLUME).build());
+
return Stream.concat(computationalStream, exploratoryStream);
}
@@ -166,6 +166,18 @@ public class BillingUtils {
String.format(DATAENGINE_SERVICE_NAME_FORMAT, resource.getMasterNodeShape(), System.lineSeparator(), null, null);
}
+ public static Stream<BillingReportLine> standardImageBillingDataStream(String sbn, String project, String endpoint) {
+ List<BillingReportLine> list = new ArrayList<>();
+ for (String notebook : AVAILABLE_NOTEBOOKS) {
+ list.add(BillingReportLine.builder().resourceName(IMAGE_NAME).dlabId(String.format(IMAGE_STANDARD_FORMAT1, sbn, project, endpoint, notebook))
+ .project(SHARED_RESOURCE).resourceType(IMAGE).build());
+ list.add(BillingReportLine.builder().resourceName(IMAGE_NAME).dlabId(String.format(IMAGE_STANDARD_FORMAT2, sbn, endpoint, notebook))
+ .project(SHARED_RESOURCE).resourceType(IMAGE).build());
+ }
+
+ return list.stream();
+ }
+
public static String getFirstLine(String sbn, LocalDate from, LocalDate to) {
return CSVFormatter.formatLine(Lists.newArrayList(String.format(REPORT_FIRST_LINE, sbn,
Optional.ofNullable(from).map(date -> date.format(DateTimeFormatter.ISO_DATE)).orElse(StringUtils.EMPTY),
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org