You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by yk...@apache.org on 2021/10/25 11:59:55 UTC
[incubator-datalab] branch develop updated: [DATALAB-2563] -- fixed
bucket browser for azure (resolve dependency )
This is an automated email from the ASF dual-hosted git repository.
ykinash pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
The following commit(s) were added to refs/heads/develop by this push:
new b869808 [DATALAB-2563] -- fixed bucket browser for azure (resolve dependency )
b869808 is described below
commit b86980868f2a64f632a41209319f329cdb36aa1f
Author: KinashYurii <ur...@gmail.com>
AuthorDate: Mon Oct 25 14:59:22 2021 +0300
[DATALAB-2563] -- fixed bucket browser for azure (resolve dependency )
---
services/provisioning-service/pom.xml | 26 +++++++++++++++++++---
.../service/impl/azure/BucketServiceAzureImpl.java | 20 +----------------
.../CheckInfrastructureStatusScheduler.java | 6 ++---
3 files changed, 27 insertions(+), 25 deletions(-)
diff --git a/services/provisioning-service/pom.xml b/services/provisioning-service/pom.xml
index 13eb127..a6f16e9 100644
--- a/services/provisioning-service/pom.xml
+++ b/services/provisioning-service/pom.xml
@@ -45,6 +45,13 @@
<type>pom</type>
<scope>import</scope>
</dependency>
+ <dependency>
+ <groupId>com.azure</groupId>
+ <artifactId>azure-sdk-bom</artifactId>
+ <version>1.0.3</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
</dependencies>
</dependencyManagement>
@@ -104,17 +111,30 @@
<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-storage-blob</artifactId>
- <version>12.14.1</version>
+ </dependency>
+ <dependency>
+ <groupId>com.azure</groupId>
+ <artifactId>azure-security-keyvault-secrets</artifactId>
</dependency>
<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-identity</artifactId>
- <version>1.0.6</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
- <version>2.11.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>io.projectreactor.netty</groupId>
+ <artifactId>reactor-netty</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>commons-io</groupId>
diff --git a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/service/impl/azure/BucketServiceAzureImpl.java b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/service/impl/azure/BucketServiceAzureImpl.java
index 28bea5a..0db7f0c 100644
--- a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/service/impl/azure/BucketServiceAzureImpl.java
+++ b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/service/impl/azure/BucketServiceAzureImpl.java
@@ -61,17 +61,9 @@ public class BucketServiceAzureImpl implements BucketService {
@Override
public List<BucketDTO> getObjects(String bucket) {
try {
- log.info("TEST LOG!!!: bucket: {}", bucket);
-
AzureStorageAccount account = getAzureStorageAccount(bucket);
- log.info("TEST LOG!!!: account: {}", account);
-
BlobServiceClient blobServiceClient = getBlobServiceClient(account.getStorageAccount());
- log.info("TEST LOG!!!: blobServiceClient: {}", blobServiceClient);
-
BlobContainerClient blobContainerClient = blobServiceClient.getBlobContainerClient(account.getContainer());
- log.info("TEST LOG!!!: blobContainerClient: {}", blobContainerClient);
-
return blobContainerClient.listBlobs()
.stream()
.map(blob -> toBucketDTO(account.getContainer(), blob))
@@ -133,11 +125,6 @@ public class BucketServiceAzureImpl implements BucketService {
}
private BucketDTO toBucketDTO(String bucket, BlobItem blob) {
- log.info("TEST LOG!!!: toBucketDTO: ");
-
- log.info("TEST LOG!!!: bucket: {}", bucket);
- log.info("TEST LOG!!!: blob: {}", blob);
-
return BucketDTO.builder()
.bucket(bucket)
.object(blob.getName())
@@ -165,8 +152,7 @@ public class BucketServiceAzureImpl implements BucketService {
}
private BlobServiceClient getBlobServiceClient(String storageAccount) {
- final String endpoint = String.format("https://%s.blob.core.windows.net", storageAccount);
- log.info("TEST LOG!!!: endpoint: {}", endpoint);
+ final String endpoint = String.format("https://%s.blob.core.windows.net", storageAccount);;
return new BlobServiceClientBuilder()
.endpoint(endpoint)
.credential(new ClientSecretCredentialBuilder()
@@ -175,10 +161,6 @@ public class BucketServiceAzureImpl implements BucketService {
.tenantId(azureAuthFile.getTenantId())
.build())
.buildClient();
-// return new BlobClientBuilder()
-// .endpoint()
-// .sasToken()
-// .containerName()
}
private AzureStorageAccount getAzureStorageAccount(String bucket) {
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/CheckInfrastructureStatusScheduler.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/CheckInfrastructureStatusScheduler.java
index 00162f6..91dda92 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/CheckInfrastructureStatusScheduler.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/CheckInfrastructureStatusScheduler.java
@@ -110,7 +110,7 @@ public class CheckInfrastructureStatusScheduler implements Job {
.stream()
.filter(c -> DataEngineType.SPARK_STANDALONE == DataEngineType.fromDockerImageName(c.getImageName()))
.filter(c -> statusesToCheck.contains(UserInstanceStatus.of(c.getStatus())))
- .filter(c -> c.getComputationalId() != null && c.getInstanceId() != null)
+// .filter(c -> c.getComputationalId() != null && c.getInstanceId() != null)
.map(r -> new EnvResource()
.withId(r.getInstanceId())
.withName(r.getComputationalName())
@@ -135,7 +135,7 @@ public class CheckInfrastructureStatusScheduler implements Job {
return userInstanceDTO.getResources().stream()
.filter(c -> DataEngineType.CLOUD_SERVICE == DataEngineType.fromDockerImageName(c.getImageName()))
.filter(c -> statusesToCheck.contains(UserInstanceStatus.of(c.getStatus())))
- .filter(c -> c.getComputationalId() != null && c.getInstanceId() != null)
+// .filter(c -> c.getComputationalId() != null && c.getInstanceId() != null)
.filter(this::noEmrCreating)
.map(r -> new EnvResource()
.withId(r.getInstanceId())
@@ -148,7 +148,7 @@ public class CheckInfrastructureStatusScheduler implements Job {
}
private boolean noEmrCreating(UserComputationalResource c) {
- return !c.getStatus().equals(CREATING.name()) && !c.getTemplateName().contains(AWS_EMR_CLUSTER);
+ return ! (c.getStatus().equals(CREATING.name()) && c.getTemplateName().contains(AWS_EMR_CLUSTER));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org