You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by of...@apache.org on 2020/05/15 11:51:53 UTC
[incubator-dlab] branch bucket-browser-azure updated: [DLAB-1552,
DLAB-1803] Azure bucket browser
This is an automated email from the ASF dual-hosted git repository.
ofuks pushed a commit to branch bucket-browser-azure
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/bucket-browser-azure by this push:
new 54d19cd [DLAB-1552, DLAB-1803] Azure bucket browser
54d19cd is described below
commit 54d19cddb93b538ad7e5c26b0b5640007aaf5696
Author: Oleh Fuks <ol...@gmail.com>
AuthorDate: Fri May 15 14:50:28 2020 +0300
[DLAB-1552, DLAB-1803] Azure bucket browser
---
services/provisioning-service/pom.xml | 11 +++-
.../dlab/backendapi/resources/BucketResource.java | 5 +-
.../dlab/backendapi/service/BucketService.java | 2 +-
.../service/impl/aws/BucketServiceAwsImpl.java | 9 ++--
.../service/impl/azure/BucketServiceAzureImpl.java | 63 ++++++++++++++++++++--
.../service/impl/gcp/BucketServiceGcpImpl.java | 2 +-
.../dlab/backendapi/resources/BucketResource.java | 5 +-
.../dlab/backendapi/service/BucketService.java | 2 +-
.../backendapi/service/impl/BucketServiceImpl.java | 7 +--
9 files changed, 86 insertions(+), 20 deletions(-)
diff --git a/services/provisioning-service/pom.xml b/services/provisioning-service/pom.xml
index fc01af1..043f596 100644
--- a/services/provisioning-service/pom.xml
+++ b/services/provisioning-service/pom.xml
@@ -101,7 +101,16 @@
<artifactId>httpclient</artifactId>
<version>4.5.9</version>
</dependency>
-
+ <dependency>
+ <groupId>com.azure</groupId>
+ <artifactId>azure-storage-blob</artifactId>
+ <version>12.6.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ <version>2.11.0</version>
+ </dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
diff --git a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/resources/BucketResource.java b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/resources/BucketResource.java
index d81bf61..180d324 100644
--- a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/resources/BucketResource.java
+++ b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/resources/BucketResource.java
@@ -65,8 +65,9 @@ public class BucketResource {
public Response uploadObject(@Auth UserInfo userInfo,
@FormDataParam("object") String object,
@FormDataParam("bucket") String bucket,
- @FormDataParam("file") InputStream inputStream) {
- bucketService.uploadObject(bucket, object, inputStream);
+ @FormDataParam("file") InputStream inputStream,
+ @FormDataParam("file-size") long fileSize) {
+ bucketService.uploadObject(bucket, object, inputStream, fileSize);
return Response.ok().build();
}
diff --git a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/BucketService.java b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/BucketService.java
index edaf29c..f69dfe7 100644
--- a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/BucketService.java
+++ b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/BucketService.java
@@ -30,7 +30,7 @@ public interface BucketService {
List<BucketDTO> getObjects(String bucket);
- void uploadObject(String bucket, String object, InputStream stream);
+ void uploadObject(String bucket, String object, InputStream stream, long fileSize);
void downloadObject(String bucket, String object, HttpServletResponse resp);
diff --git a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/aws/BucketServiceAwsImpl.java b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/aws/BucketServiceAwsImpl.java
index 83f65e3..d7cef73 100644
--- a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/aws/BucketServiceAwsImpl.java
+++ b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/aws/BucketServiceAwsImpl.java
@@ -24,8 +24,6 @@ import com.epam.dlab.dto.bucket.BucketDTO;
import com.epam.dlab.exceptions.DlabException;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOUtils;
-import software.amazon.awssdk.awscore.exception.AwsServiceException;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.core.sync.ResponseTransformer;
import software.amazon.awssdk.services.s3.S3Client;
@@ -68,7 +66,7 @@ public class BucketServiceAwsImpl implements BucketService {
}
@Override
- public void uploadObject(String bucket, String object, InputStream stream) {
+ public void uploadObject(String bucket, String object, InputStream stream, long fileSize) {
try {
S3Client s3 = S3Client.create();
PutObjectRequest uploadRequest = PutObjectRequest
@@ -76,7 +74,7 @@ public class BucketServiceAwsImpl implements BucketService {
.bucket(bucket)
.key(object)
.build();
- s3.putObject(uploadRequest, RequestBody.fromBytes(IOUtils.toByteArray(stream)));
+ s3.putObject(uploadRequest, RequestBody.fromInputStream(stream, fileSize));
} catch (Exception e) {
log.error("Cannot upload object {} to bucket {}. Reason: {}", object, bucket, e.getMessage());
throw new DlabException(String.format("Cannot upload object %s to bucket %s. Reason: %s", object, bucket, e.getMessage()));
@@ -118,8 +116,7 @@ public class BucketServiceAwsImpl implements BucketService {
.build();
s3.deleteObjects(deleteObjectsRequests);
-
- } catch (AwsServiceException e) {
+ } catch (Exception e) {
log.error("Cannot delete objects {} from bucket {}. Reason: {}", objects, bucket, e.getMessage());
throw new DlabException(String.format("Cannot delete objects %s from bucket %s. Reason: %s", objects, bucket, e.getMessage()));
}
diff --git a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/azure/BucketServiceAzureImpl.java b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/azure/BucketServiceAzureImpl.java
index 1112b5e..c650c6e 100644
--- a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/azure/BucketServiceAzureImpl.java
+++ b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/azure/BucketServiceAzureImpl.java
@@ -19,30 +19,87 @@
package com.epam.dlab.backendapi.service.impl.azure;
+import com.azure.storage.blob.BlobClient;
+import com.azure.storage.blob.BlobContainerClient;
+import com.azure.storage.blob.BlobServiceClient;
+import com.azure.storage.blob.BlobServiceClientBuilder;
+import com.azure.storage.blob.models.BlobItem;
import com.epam.dlab.backendapi.service.BucketService;
import com.epam.dlab.dto.bucket.BucketDTO;
+import com.epam.dlab.exceptions.DlabException;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.FileUtils;
+import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletResponse;
import java.io.InputStream;
+import java.time.format.DateTimeFormatter;
import java.util.List;
+import java.util.stream.Collectors;
+@Slf4j
public class BucketServiceAzureImpl implements BucketService {
@Override
public List<BucketDTO> getObjects(String bucket) {
- return null;
+ try {
+ BlobServiceClient blobServiceClient = new BlobServiceClientBuilder().connectionString(System.getenv("AZURE_STORAGE_CONNECTION_STRING")).buildClient();
+ BlobContainerClient blobContainerClient = blobServiceClient.getBlobContainerClient(bucket);
+ return blobContainerClient.listBlobs()
+ .stream()
+ .map(blob -> toBucketDTO(bucket, blob))
+ .collect(Collectors.toList());
+ } catch (Exception e) {
+ log.error("Cannot retrieve objects from bucket {}. Reason: {}", bucket, e.getMessage());
+ throw new DlabException(String.format("Cannot retrieve objects from bucket %s. Reason: %s", bucket, e.getMessage()));
+ }
}
@Override
- public void uploadObject(String bucket, String object, InputStream stream) {
-
+ public void uploadObject(String bucket, String object, InputStream stream, long fileSize) {
+ try {
+ BlobServiceClient blobServiceClient = new BlobServiceClientBuilder().connectionString(System.getenv("AZURE_STORAGE_CONNECTION_STRING")).buildClient();
+ BlobContainerClient blobContainerClient = blobServiceClient.getBlobContainerClient(bucket);
+ BlobClient blobClient = blobContainerClient.getBlobClient(object);
+ blobClient.upload(stream, fileSize);
+ } catch (Exception e) {
+ log.error("Cannot upload object {} to bucket {}. Reason: {}", object, bucket, e.getMessage());
+ throw new DlabException(String.format("Cannot upload object %s to bucket %s. Reason: %s", object, bucket, e.getMessage()));
+ }
}
@Override
public void downloadObject(String bucket, String object, HttpServletResponse resp) {
+ try (ServletOutputStream outputStream = resp.getOutputStream()) {
+ BlobServiceClient blobServiceClient = new BlobServiceClientBuilder().connectionString(System.getenv("AZURE_STORAGE_CONNECTION_STRING")).buildClient();
+ BlobContainerClient blobContainerClient = blobServiceClient.getBlobContainerClient(bucket);
+ BlobClient blobClient = blobContainerClient.getBlobClient(object);
+ blobClient.download(outputStream);
+ } catch (Exception e) {
+ log.error("Cannot download object {} from bucket {}. Reason: {}", object, bucket, e.getMessage());
+ throw new DlabException(String.format("Cannot download object %s from bucket %s. Reason: %s", object, bucket, e.getMessage()));
+ }
}
@Override
public void deleteObjects(String bucket, List<String> objects) {
+ try {
+ BlobServiceClient blobServiceClient = new BlobServiceClientBuilder().connectionString(System.getenv("AZURE_STORAGE_CONNECTION_STRING")).buildClient();
+ BlobContainerClient blobContainerClient = blobServiceClient.getBlobContainerClient(bucket);
+ objects.forEach(object -> blobContainerClient.getBlobClient(object).delete());
+ } catch (Exception e) {
+ log.error("Cannot delete objects {} from bucket {}. Reason: {}", objects, bucket, e.getMessage());
+ throw new DlabException(String.format("Cannot delete objects %s from bucket %s. Reason: %s", objects, bucket, e.getMessage()));
+ }
+ }
+ private BucketDTO toBucketDTO(String bucket, BlobItem blob) {
+ final String size = FileUtils.byteCountToDisplaySize(blob.getProperties().getContentLength());
+ String lastModifiedDate = blob.getProperties().getLastModified().format(DateTimeFormatter.ofPattern(DATE_FORMAT));
+ return BucketDTO.builder()
+ .bucket(bucket)
+ .object(blob.getName())
+ .lastModifiedDate(lastModifiedDate)
+ .size(size)
+ .build();
}
}
diff --git a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/gcp/BucketServiceGcpImpl.java b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/gcp/BucketServiceGcpImpl.java
index dc068fc..bd9abf6 100644
--- a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/gcp/BucketServiceGcpImpl.java
+++ b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/gcp/BucketServiceGcpImpl.java
@@ -58,7 +58,7 @@ public class BucketServiceGcpImpl implements BucketService {
}
@Override
- public void uploadObject(String bucket, String object, InputStream stream) {
+ public void uploadObject(String bucket, String object, InputStream stream, long fileSize) {
log.info("Uploading file {} to bucket {}", object, bucket);
try {
Storage storage = StorageOptions.getDefaultInstance().getService();
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/BucketResource.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/BucketResource.java
index f34c3f3..f757950 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/BucketResource.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/BucketResource.java
@@ -73,8 +73,9 @@ public class BucketResource {
@FormDataParam("object") String object,
@FormDataParam("bucket") String bucket,
@FormDataParam("endpoint") String endpoint,
- @FormDataParam("file") InputStream fileInputStream) {
- bucketService.uploadObjects(userInfo, bucket, object, endpoint, fileInputStream);
+ @FormDataParam("file") InputStream fileInputStream,
+ @FormDataParam("size") long fileSize) {
+ bucketService.uploadObjects(userInfo, bucket, object, endpoint, fileInputStream, fileSize);
return Response.ok().build();
}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/BucketService.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/BucketService.java
index 62a0e2e..6e5345b 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/BucketService.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/BucketService.java
@@ -29,7 +29,7 @@ import java.util.List;
public interface BucketService {
List<BucketDTO> getObjects(UserInfo userInfo, String bucket, String endpoint);
- void uploadObjects(UserInfo userInfo, String bucket, String object, String endpoint, InputStream inputStream);
+ void uploadObjects(UserInfo userInfo, String bucket, String object, String endpoint, InputStream inputStream, long fileSize);
void downloadObject(UserInfo userInfo, String bucket, String object, String endpoint, HttpServletResponse resp);
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BucketServiceImpl.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BucketServiceImpl.java
index 1372ada..d305532 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BucketServiceImpl.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BucketServiceImpl.java
@@ -79,11 +79,11 @@ public class BucketServiceImpl implements BucketService {
}
@Override
- public void uploadObjects(UserInfo userInfo, String bucket, String object, String endpoint, InputStream inputStream) {
+ public void uploadObjects(UserInfo userInfo, String bucket, String object, String endpoint, InputStream inputStream, long fileSize) {
log.info("Uploading file {} for user {} to bucket {}", object, userInfo.getName(), bucket);
try {
EndpointDTO endpointDTO = endpointService.get(endpoint);
- FormDataMultiPart formData = getFormDataMultiPart(bucket, object, inputStream);
+ FormDataMultiPart formData = getFormDataMultiPart(bucket, object, inputStream, fileSize);
Response response = provisioningService.postForm(String.format(BUCKET_UPLOAD_OBJECT, endpointDTO.getUrl()), userInfo.getAccessToken(), formData, Response.class);
if (response.getStatus() != HttpStatus.SC_OK) {
throw new DlabException(String.format("Something went wrong. Response status is %s ", response.getStatus()));
@@ -128,11 +128,12 @@ public class BucketServiceImpl implements BucketService {
return URLEncoder.encode(object, StandardCharsets.UTF_8.toString()).replace("+", "%20");
}
- private FormDataMultiPart getFormDataMultiPart(String bucket, String object, InputStream inputStream) {
+ private FormDataMultiPart getFormDataMultiPart(String bucket, String object, InputStream inputStream, long fileSize) {
StreamDataBodyPart filePart = new StreamDataBodyPart("file", inputStream, object, MediaType.valueOf(APPLICATION_OCTET_STREAM));
FormDataMultiPart formData = new FormDataMultiPart();
formData.field("bucket", bucket);
formData.field("object", object);
+ formData.field("file-size", String.valueOf(fileSize));
formData.bodyPart(filePart);
return formData;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org