You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by of...@apache.org on 2020/05/08 09:41:48 UTC
[incubator-dlab] branch develop updated: added streaming download
feature
This is an automated email from the ASF dual-hosted git repository.
ofuks pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/develop by this push:
new 47cc42b added streaming download feature
new 1a6f2a4 Merge remote-tracking branch 'origin/develop' into develop
47cc42b is described below
commit 47cc42b4eee1c7953d9f3aa02681dedb40939eb7
Author: Oleh Fuks <ol...@gmail.com>
AuthorDate: Fri May 8 12:04:34 2020 +0300
added streaming download feature
---
.../dlab/backendapi/resources/BucketResource.java | 7 +++++--
.../epam/dlab/backendapi/service/BucketService.java | 3 ++-
.../service/impl/aws/BucketServiceAwsImpl.java | 8 +++++---
.../service/impl/azure/BucketServiceAzureImpl.java | 4 ++--
.../service/impl/gcp/BucketServiceGcpImpl.java | 12 ++++++------
services/self-service/pom.xml | 5 +++++
.../dlab/backendapi/resources/BucketResource.java | 7 +++++--
.../epam/dlab/backendapi/service/BucketService.java | 3 ++-
.../backendapi/service/impl/BucketServiceImpl.java | 19 +++++++++++++------
9 files changed, 45 insertions(+), 23 deletions(-)
diff --git a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/resources/BucketResource.java b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/resources/BucketResource.java
index 28e51b3..d81bf61 100644
--- a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/resources/BucketResource.java
+++ b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/resources/BucketResource.java
@@ -27,12 +27,14 @@ import io.dropwizard.auth.Auth;
import lombok.extern.slf4j.Slf4j;
import org.glassfish.jersey.media.multipart.FormDataParam;
+import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.Consumes;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
+import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import java.io.InputStream;
@@ -72,10 +74,11 @@ public class BucketResource {
@Path("/{bucket}/object/{object}/download")
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_OCTET_STREAM)
- public Response downloadObject(@Auth UserInfo userInfo,
+ public Response downloadObject(@Auth UserInfo userInfo, @Context HttpServletResponse resp,
@PathParam("object") String object,
@PathParam("bucket") String bucket) {
- return Response.ok(bucketService.downloadObject(bucket, object)).build();
+ bucketService.downloadObject(bucket, object, resp);
+ return Response.ok().build();
}
@POST
diff --git a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/BucketService.java b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/BucketService.java
index fed9aef..edaf29c 100644
--- a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/BucketService.java
+++ b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/BucketService.java
@@ -21,6 +21,7 @@ package com.epam.dlab.backendapi.service;
import com.epam.dlab.dto.bucket.BucketDTO;
+import javax.servlet.http.HttpServletResponse;
import java.io.InputStream;
import java.util.List;
@@ -31,7 +32,7 @@ public interface BucketService {
void uploadObject(String bucket, String object, InputStream stream);
- byte[] downloadObject(String bucket, String object);
+ void downloadObject(String bucket, String object, HttpServletResponse resp);
void deleteObjects(String bucket, List<String> objects);
}
diff --git a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/aws/BucketServiceAwsImpl.java b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/aws/BucketServiceAwsImpl.java
index 94755f2..83f65e3 100644
--- a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/aws/BucketServiceAwsImpl.java
+++ b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/aws/BucketServiceAwsImpl.java
@@ -37,6 +37,8 @@ import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.model.S3Object;
+import javax.servlet.ServletOutputStream;
+import javax.servlet.http.HttpServletResponse;
import java.io.InputStream;
import java.text.SimpleDateFormat;
import java.util.Date;
@@ -82,15 +84,15 @@ public class BucketServiceAwsImpl implements BucketService {
}
@Override
- public byte[] downloadObject(String bucket, String object) {
- try {
+ public void downloadObject(String bucket, String object, HttpServletResponse resp) {
+ try (ServletOutputStream outputStream = resp.getOutputStream()) {
S3Client s3 = S3Client.create();
GetObjectRequest downloadRequest = GetObjectRequest
.builder()
.bucket(bucket)
.key(object)
.build();
- return s3.getObject(downloadRequest, ResponseTransformer.toBytes()).asByteArray();
+ s3.getObject(downloadRequest, ResponseTransformer.toOutputStream(outputStream));
} catch (Exception e) {
log.error("Cannot download object {} from bucket {}. Reason: {}", object, bucket, e.getMessage());
throw new DlabException(String.format("Cannot download object %s from bucket %s. Reason: %s", object, bucket, e.getMessage()));
diff --git a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/azure/BucketServiceAzureImpl.java b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/azure/BucketServiceAzureImpl.java
index 87ac6d4..1112b5e 100644
--- a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/azure/BucketServiceAzureImpl.java
+++ b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/azure/BucketServiceAzureImpl.java
@@ -22,6 +22,7 @@ package com.epam.dlab.backendapi.service.impl.azure;
import com.epam.dlab.backendapi.service.BucketService;
import com.epam.dlab.dto.bucket.BucketDTO;
+import javax.servlet.http.HttpServletResponse;
import java.io.InputStream;
import java.util.List;
@@ -37,8 +38,7 @@ public class BucketServiceAzureImpl implements BucketService {
}
@Override
- public byte[] downloadObject(String bucket, String object) {
- return new byte[0];
+ public void downloadObject(String bucket, String object, HttpServletResponse resp) {
}
@Override
diff --git a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/gcp/BucketServiceGcpImpl.java b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/gcp/BucketServiceGcpImpl.java
index e01f9ba..dc068fc 100644
--- a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/gcp/BucketServiceGcpImpl.java
+++ b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/gcp/BucketServiceGcpImpl.java
@@ -31,7 +31,8 @@ import com.google.cloud.storage.StorageOptions;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
-import java.io.ByteArrayOutputStream;
+import javax.servlet.ServletOutputStream;
+import javax.servlet.http.HttpServletResponse;
import java.io.InputStream;
import java.text.SimpleDateFormat;
import java.util.Date;
@@ -72,14 +73,13 @@ public class BucketServiceGcpImpl implements BucketService {
}
@Override
- public byte[] downloadObject(String bucket, String object) {
+ public void downloadObject(String bucket, String object, HttpServletResponse resp) {
log.info("Downloading file {} from bucket {}", object, bucket);
- try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) {
+ try (ServletOutputStream outputStream = resp.getOutputStream()) {
Storage storage = StorageOptions.getDefaultInstance().getService();
Blob blob = storage.get(BlobId.of(bucket, object));
- blob.downloadTo(outputStream); //todo add check for blob != null and throw exception
- log.info("Downloading uploading file {} from bucket {}", object, bucket);
- return outputStream.toByteArray();
+ blob.downloadTo(outputStream);
+ log.info("Finished downloading file {} from bucket {}", object, bucket);
} catch (Exception e) {
log.error("Cannot download object {} from bucket {}. Reason: {}", object, bucket, e.getMessage());
throw new DlabException(String.format("Cannot download object %s from bucket %s. Reason: %s", object, bucket, e.getMessage()));
diff --git a/services/self-service/pom.xml b/services/self-service/pom.xml
index 382dc7c..d974c69 100644
--- a/services/self-service/pom.xml
+++ b/services/self-service/pom.xml
@@ -186,6 +186,11 @@
</dependency>
<dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>2.6</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId>
<version>4.4</version>
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/BucketResource.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/BucketResource.java
index 5f189f8..f34c3f3 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/BucketResource.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/BucketResource.java
@@ -28,6 +28,7 @@ import lombok.extern.slf4j.Slf4j;
import org.glassfish.jersey.media.multipart.FormDataParam;
import javax.annotation.security.RolesAllowed;
+import javax.servlet.http.HttpServletResponse;
import javax.validation.Valid;
import javax.ws.rs.Consumes;
import javax.ws.rs.GET;
@@ -35,6 +36,7 @@ import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
+import javax.ws.rs.core.Context;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
@@ -81,11 +83,12 @@ public class BucketResource {
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_OCTET_STREAM)
@RolesAllowed("/api/bucket/download")
- public Response downloadObject(@Auth UserInfo userInfo,
+ public Response downloadObject(@Auth UserInfo userInfo, @Context HttpServletResponse resp,
@PathParam("bucket") String bucket,
@PathParam("object") String object,
@PathParam("endpoint") String endpoint) {
- return Response.ok(bucketService.downloadObject(userInfo, bucket, object, endpoint))
+ bucketService.downloadObject(userInfo, bucket, object, endpoint, resp);
+ return Response.ok()
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + Paths.get(object).getFileName() + "\"")
.build();
}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/BucketService.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/BucketService.java
index 3fdba99..62a0e2e 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/BucketService.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/BucketService.java
@@ -22,6 +22,7 @@ package com.epam.dlab.backendapi.service;
import com.epam.dlab.auth.UserInfo;
import com.epam.dlab.dto.bucket.BucketDTO;
+import javax.servlet.http.HttpServletResponse;
import java.io.InputStream;
import java.util.List;
@@ -30,7 +31,7 @@ public interface BucketService {
void uploadObjects(UserInfo userInfo, String bucket, String object, String endpoint, InputStream inputStream);
- byte[] downloadObject(UserInfo userInfo, String bucket, String object, String endpoint);
+ void downloadObject(UserInfo userInfo, String bucket, String object, String endpoint, HttpServletResponse resp);
void deleteObjects(UserInfo userInfo, String bucket, List<String> objects, String endpoint);
}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BucketServiceImpl.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BucketServiceImpl.java
index 3cc9413..1372ada 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BucketServiceImpl.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BucketServiceImpl.java
@@ -31,9 +31,13 @@ import com.epam.dlab.rest.client.RESTService;
import com.google.inject.Inject;
import com.google.inject.name.Named;
import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.IOUtils;
import org.apache.http.HttpStatus;
import org.glassfish.jersey.media.multipart.FormDataMultiPart;
+import org.glassfish.jersey.media.multipart.file.StreamDataBodyPart;
+import javax.servlet.ServletOutputStream;
+import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.core.GenericType;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
@@ -92,11 +96,13 @@ public class BucketServiceImpl implements BucketService {
}
@Override
- public byte[] downloadObject(UserInfo userInfo, String bucket, String object, String endpoint) {
- try {
- EndpointDTO endpointDTO = endpointService.get(endpoint);
- return provisioningService.getWithMediaTypes(String.format(BUCKET_DOWNLOAD_OBJECT, endpointDTO.getUrl(), bucket, encodeObject(object)), userInfo.getAccessToken(), byte[].class,
- APPLICATION_JSON, APPLICATION_OCTET_STREAM);
+ public void downloadObject(UserInfo userInfo, String bucket, String object, String endpoint, HttpServletResponse resp) {
+ log.info("Downloading file {} for user {} from bucket {}", object, userInfo.getName(), bucket);
+ EndpointDTO endpointDTO = endpointService.get(endpoint);
+ try (InputStream inputStream = provisioningService.getWithMediaTypes(String.format(BUCKET_DOWNLOAD_OBJECT, endpointDTO.getUrl(), bucket, encodeObject(object)), userInfo.getAccessToken(),
+ InputStream.class, APPLICATION_JSON, APPLICATION_OCTET_STREAM); ServletOutputStream outputStream = resp.getOutputStream()) {
+ IOUtils.copyLarge(inputStream, outputStream);
+ log.info("Finished downloading file {} for user {} from bucket {}", object, userInfo.getName(), bucket);
} catch (Exception e) {
log.error("Cannot upload object {} from bucket {} for user {}, endpoint {}. Reason {}", object, bucket, userInfo.getName(), endpoint, e.getMessage());
throw new DlabException(String.format("Cannot download object %s from bucket %s for user %s, endpoint %s. Reason %s", object, bucket, userInfo.getName(), endpoint, e.getMessage()));
@@ -123,10 +129,11 @@ public class BucketServiceImpl implements BucketService {
}
private FormDataMultiPart getFormDataMultiPart(String bucket, String object, InputStream inputStream) {
+ StreamDataBodyPart filePart = new StreamDataBodyPart("file", inputStream, object, MediaType.valueOf(APPLICATION_OCTET_STREAM));
FormDataMultiPart formData = new FormDataMultiPart();
- formData.field("file", inputStream, MediaType.valueOf(APPLICATION_OCTET_STREAM));
formData.field("bucket", bucket);
formData.field("object", object);
+ formData.bodyPart(filePart);
return formData;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org