You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by of...@apache.org on 2020/06/01 14:12:53 UTC
[incubator-dlab] branch develop updated: [DLAB-1815] Improve
performance of bucket browser
This is an automated email from the ASF dual-hosted git repository.
ofuks pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/develop by this push:
new a1a14e0 [DLAB-1815] Improve performance of bucket browser
a1a14e0 is described below
commit a1a14e08b572928db7cbfab6b0490f939daaceda
Author: Oleh Fuks <ol...@gmail.com>
AuthorDate: Mon Jun 1 17:11:57 2020 +0300
[DLAB-1815] Improve performance of bucket browser
---
pom.xml | 1 +
services/provisioning-service/pom.xml | 5 ++
.../dlab/backendapi/resources/BucketResource.java | 54 ++++++++++++++++---
services/self-service/pom.xml | 5 ++
.../dlab/backendapi/resources/BucketResource.java | 60 +++++++++++++++++++---
.../service/impl/EndpointServiceImpl.java | 6 +--
6 files changed, 113 insertions(+), 18 deletions(-)
diff --git a/pom.xml b/pom.xml
index b6808a8..e24bfd9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -86,6 +86,7 @@
<lombok.version>1.16.18</lombok.version>
<hibernate.validator.version>5.4.2.Final</hibernate.validator.version>
<logback.version>1.2.3</logback.version>
+ <commons-fileupload.version>1.3.3</commons-fileupload.version>
</properties>
<dependencyManagement>
<dependencies>
diff --git a/services/provisioning-service/pom.xml b/services/provisioning-service/pom.xml
index 043f596..0cec6a5 100644
--- a/services/provisioning-service/pom.xml
+++ b/services/provisioning-service/pom.xml
@@ -117,6 +117,11 @@
<version>2.6</version>
</dependency>
<dependency>
+ <groupId>commons-fileupload</groupId>
+ <artifactId>commons-fileupload</artifactId>
+ <version>${commons-fileupload.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>${org.mockito.version}</version>
diff --git a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/resources/BucketResource.java b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/resources/BucketResource.java
index 180d324..4495f5d 100644
--- a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/resources/BucketResource.java
+++ b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/resources/BucketResource.java
@@ -22,11 +22,16 @@ package com.epam.dlab.backendapi.resources;
import com.epam.dlab.auth.UserInfo;
import com.epam.dlab.backendapi.service.BucketService;
import com.epam.dlab.dto.bucket.BucketDeleteDTO;
+import com.epam.dlab.exceptions.DlabException;
import com.google.inject.Inject;
import io.dropwizard.auth.Auth;
import lombok.extern.slf4j.Slf4j;
-import org.glassfish.jersey.media.multipart.FormDataParam;
+import org.apache.commons.fileupload.FileItemIterator;
+import org.apache.commons.fileupload.FileItemStream;
+import org.apache.commons.fileupload.servlet.ServletFileUpload;
+import org.apache.commons.fileupload.util.Streams;
+import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.Consumes;
import javax.ws.rs.GET;
@@ -42,6 +47,10 @@ import java.io.InputStream;
@Slf4j
@Path("/bucket")
public class BucketResource {
+ private static final String OBJECT_FORM_FIELD = "object";
+ private static final String BUCKET_FORM_FIELD = "bucket";
+ private static final String SIZE_FORM_FIELD = "file-size";
+
private final BucketService bucketService;
@Inject
@@ -62,12 +71,8 @@ public class BucketResource {
@Path("/upload")
@Consumes(MediaType.MULTIPART_FORM_DATA)
@Produces(MediaType.APPLICATION_JSON)
- public Response uploadObject(@Auth UserInfo userInfo,
- @FormDataParam("object") String object,
- @FormDataParam("bucket") String bucket,
- @FormDataParam("file") InputStream inputStream,
- @FormDataParam("file-size") long fileSize) {
- bucketService.uploadObject(bucket, object, inputStream, fileSize);
+ public Response uploadObject(@Auth UserInfo userInfo, @Context HttpServletRequest request) {
+ upload(request);
return Response.ok().build();
}
@@ -90,4 +95,39 @@ public class BucketResource {
bucketService.deleteObjects(bucketDeleteDTO.getBucket(), bucketDeleteDTO.getObjects());
return Response.ok().build();
}
+
+ private void upload(HttpServletRequest request) {
+ String object = null;
+ String bucket = null;
+ long fileSize = 0;
+
+ ServletFileUpload upload = new ServletFileUpload();
+ try {
+ FileItemIterator iterStream = upload.getItemIterator(request);
+ while (iterStream.hasNext()) {
+ FileItemStream item = iterStream.next();
+ try (InputStream stream = item.openStream()) {
+ if (item.isFormField()) {
+ if (OBJECT_FORM_FIELD.equals(item.getFieldName())) {
+ object = Streams.asString(stream);
+ }
+ if (BUCKET_FORM_FIELD.equals(item.getFieldName())) {
+ bucket = Streams.asString(stream);
+ }
+ if (SIZE_FORM_FIELD.equals(item.getFieldName())) {
+ fileSize = Long.parseLong(Streams.asString(stream));
+ }
+ } else {
+ bucketService.uploadObject(bucket, object, stream, fileSize);
+ }
+ } catch (Exception e) {
+ log.error("Cannot upload object {} to bucket {}. {}", object, bucket, e.getMessage(), e);
+ throw new DlabException(String.format("Cannot upload object %s to bucket %s. %s", object, bucket, e.getMessage()));
+ }
+ }
+ } catch (Exception e) {
+ log.error("Cannot upload object {} to bucket {}. {}", object, bucket, e.getMessage(), e);
+ throw new DlabException(String.format("Cannot upload object %s to bucket %s. %s", object, bucket, e.getMessage()));
+ }
+ }
}
diff --git a/services/self-service/pom.xml b/services/self-service/pom.xml
index d974c69..f21caf9 100644
--- a/services/self-service/pom.xml
+++ b/services/self-service/pom.xml
@@ -195,6 +195,11 @@
<artifactId>commons-collections4</artifactId>
<version>4.4</version>
</dependency>
+ <dependency>
+ <groupId>commons-fileupload</groupId>
+ <artifactId>commons-fileupload</artifactId>
+ <version>${commons-fileupload.version}</version>
+ </dependency>
</dependencies>
<build>
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/BucketResource.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/BucketResource.java
index f757950..6ffc7f9 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/BucketResource.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/BucketResource.java
@@ -22,12 +22,17 @@ package com.epam.dlab.backendapi.resources;
import com.epam.dlab.auth.UserInfo;
import com.epam.dlab.backendapi.resources.dto.BucketDeleteDTO;
import com.epam.dlab.backendapi.service.BucketService;
+import com.epam.dlab.exceptions.DlabException;
import com.google.inject.Inject;
import io.dropwizard.auth.Auth;
import lombok.extern.slf4j.Slf4j;
-import org.glassfish.jersey.media.multipart.FormDataParam;
+import org.apache.commons.fileupload.FileItemIterator;
+import org.apache.commons.fileupload.FileItemStream;
+import org.apache.commons.fileupload.servlet.ServletFileUpload;
+import org.apache.commons.fileupload.util.Streams;
import javax.annotation.security.RolesAllowed;
+import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.validation.Valid;
import javax.ws.rs.Consumes;
@@ -46,6 +51,11 @@ import java.nio.file.Paths;
@Path("/bucket")
@Slf4j
public class BucketResource {
+ private static final String OBJECT_FORM_FIELD = "object";
+ private static final String BUCKET_FORM_FIELD = "bucket";
+ private static final String ENDPOINT_FORM_FIELD = "endpoint";
+ private static final String SIZE_FORM_FIELD = "size";
+
private final BucketService bucketService;
@Inject
@@ -69,13 +79,8 @@ public class BucketResource {
@Consumes(MediaType.MULTIPART_FORM_DATA)
@Produces(MediaType.APPLICATION_JSON)
@RolesAllowed("/api/bucket/upload")
- public Response uploadObject(@Auth UserInfo userInfo,
- @FormDataParam("object") String object,
- @FormDataParam("bucket") String bucket,
- @FormDataParam("endpoint") String endpoint,
- @FormDataParam("file") InputStream fileInputStream,
- @FormDataParam("size") long fileSize) {
- bucketService.uploadObjects(userInfo, bucket, object, endpoint, fileInputStream, fileSize);
+ public Response uploadObject(@Auth UserInfo userInfo, @Context HttpServletRequest request) {
+ upload(userInfo, request);
return Response.ok().build();
}
@@ -103,4 +108,43 @@ public class BucketResource {
bucketService.deleteObjects(userInfo, bucketDto.getBucket(), bucketDto.getObjects(), bucketDto.getEndpoint());
return Response.ok().build();
}
+
+ private void upload(UserInfo userInfo, HttpServletRequest request) {
+ String object = null;
+ String bucket = null;
+ String endpoint = null;
+ long fileSize = 0;
+
+ ServletFileUpload upload = new ServletFileUpload();
+ try {
+ FileItemIterator iterStream = upload.getItemIterator(request);
+ while (iterStream.hasNext()) {
+ FileItemStream item = iterStream.next();
+ try (InputStream stream = item.openStream()) {
+ if (item.isFormField()) {
+ if (OBJECT_FORM_FIELD.equals(item.getFieldName())) {
+ object = Streams.asString(stream);
+ }
+ if (BUCKET_FORM_FIELD.equals(item.getFieldName())) {
+ bucket = Streams.asString(stream);
+ }
+ if (ENDPOINT_FORM_FIELD.equals(item.getFieldName())) {
+ endpoint = Streams.asString(stream);
+ }
+ if (SIZE_FORM_FIELD.equals(item.getFieldName())) {
+ fileSize = Long.parseLong(Streams.asString(stream));
+ }
+ } else {
+ bucketService.uploadObjects(userInfo, bucket, object, endpoint, stream, fileSize);
+ }
+ } catch (Exception e) {
+ log.error("Cannot upload object {} to bucket {}. {}", object, bucket, e.getMessage(), e);
+ throw new DlabException(String.format("Cannot upload object %s to bucket %s. %s", object, bucket, e.getMessage()));
+ }
+ }
+ } catch (Exception e) {
+ log.error("User {} cannot upload object {} to bucket {}. {}", userInfo.getName(), object, bucket, e.getMessage(), e);
+ throw new DlabException(String.format("User %s cannot upload object %s to bucket %s. %s", userInfo.getName(), object, bucket, e.getMessage()));
+ }
+ }
}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/EndpointServiceImpl.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/EndpointServiceImpl.java
index 645f84b..fcfe789 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/EndpointServiceImpl.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/EndpointServiceImpl.java
@@ -137,9 +137,9 @@ public class EndpointServiceImpl implements EndpointService {
response = provisioningService.get(url + HEALTH_CHECK, userInfo.getAccessToken(), Response.class);
cloudProvider = response.readEntity(CloudProvider.class);
} catch (Exception e) {
- log.error("Cannot connect to url '{}'. {}", url, e.getMessage());
- throw new DlabException(String.format("Cannot connect to url '%s'. %s", url, e.getMessage()));
- }
+ log.error("Cannot connect to url '{}'. {}", url, e.getMessage());
+ throw new DlabException(String.format("Cannot connect to url '%s'.", url));
+ }
if (response.getStatus() != 200) {
log.warn("Endpoint url {} is not valid", url);
throw new ResourceNotFoundException(String.format("Endpoint url '%s' is not valid", url));
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org