You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by re...@apache.org on 2025/07/29 08:29:57 UTC
(jackrabbit-oak) branch OAK-11267 updated: OAK-11267: imported diffs from https://github.com/seropian/jackrabbit-oak/tree/upgrade_azure_sdk_12
This is an automated email from the ASF dual-hosted git repository.
reschke pushed a commit to branch OAK-11267
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/OAK-11267 by this push:
new 9b94c6f0d9 OAK-11267: imported diffs from https://github.com/seropian/jackrabbit-oak/tree/upgrade_azure_sdk_12
9b94c6f0d9 is described below
commit 9b94c6f0d94ab035301b37da5e9b96deafc6fd36
Author: Julian Reschke <re...@apache.org>
AuthorDate: Tue Jul 29 09:29:43 2025 +0100
OAK-11267: imported diffs from https://github.com/seropian/jackrabbit-oak/tree/upgrade_azure_sdk_12
---
oak-blob-cloud-azure/pom.xml | 17 +
.../blobstorage/AbstractAzureBlobStoreBackend.java | 45 ++
.../blobstorage/AzureBlobContainerProvider.java | 231 ++----
.../azure/blobstorage/AzureBlobStoreBackend.java | 819 +++++++++------------
.../cloud/azure/blobstorage/AzureConstants.java | 82 ++-
.../cloud/azure/blobstorage/AzureDataStore.java | 12 +-
.../azure/blobstorage/AzureDataStoreService.java | 3 +-
.../oak/blob/cloud/azure/blobstorage/Utils.java | 126 ++--
.../AzureBlobContainerProviderV8.java} | 21 +-
.../AzureBlobStoreBackendV8.java} | 288 ++++----
.../blobstorage/{Utils.java => v8/UtilsV8.java} | 83 +--
.../blobstorage/AzureBlobStoreBackendTest.java | 78 +-
.../AzureDataRecordAccessProviderIT.java | 2 +-
.../AzureDataRecordAccessProviderTest.java | 8 +-
.../azure/blobstorage/AzureDataStoreUtils.java | 13 +-
.../cloud/azure/blobstorage/AzuriteDockerRule.java | 16 +-
.../blob/cloud/azure/blobstorage/TestAzureDS.java | 5 +-
.../blobstorage/TestAzureDSWithSmallCache.java | 2 +-
.../azure/blobstorage/TestAzureDsCacheOff.java | 2 +-
.../blob/cloud/azure/blobstorage/UtilsTest.java | 71 ++
.../AzureBlobContainerProviderV8Test.java} | 44 +-
.../AzureBlobStoreBackendV8Test.java} | 32 +-
.../{UtilsTest.java => v8/UtilsV8Test.java} | 44 +-
.../src/test/resources/azure.properties | 4 +-
...e.jackrabbit.oak.jcr.osgi.RepositoryManager.cfg | 30 +-
...krabbit.oak.segment.SegmentNodeStoreService.cfg | 32 +-
.../fixtures/datastore/AzureDataStoreFixture.java | 40 +-
.../jackrabbit/oak/fixture/DataStoreUtils.java | 8 +-
.../jackrabbit/oak/fixture/DataStoreUtilsTest.java | 56 +-
oak-run-elastic/pom.xml | 3 +-
30 files changed, 1101 insertions(+), 1116 deletions(-)
diff --git a/oak-blob-cloud-azure/pom.xml b/oak-blob-cloud-azure/pom.xml
index b01e13a2ff..c806980420 100644
--- a/oak-blob-cloud-azure/pom.xml
+++ b/oak-blob-cloud-azure/pom.xml
@@ -41,10 +41,15 @@
com.fasterxml.jackson.annotation;resolution:=optional,
com.fasterxml.jackson.databind*;resolution:=optional,
com.fasterxml.jackson.dataformat.xml;resolution:=optional,
+ com.fasterxml.jackson.dataformat.xml.annotation;resolution:=optional,
com.fasterxml.jackson.datatype*;resolution:=optional,
com.azure.identity.broker.implementation;resolution:=optional,
com.azure.xml;resolution:=optional,
+ com.azure.storage.common*;resolution:=optional,
+ com.azure.storage.internal*;resolution:=optional,
+ com.microsoft.aad.*;resolution:=optional,
com.microsoft.aad.msal4jextensions*;resolution:=optional,
+ com.microsoft.aad.msal4jextensions.persistence*;resolution:=optional,
com.sun.net.httpserver;resolution:=optional,
sun.misc;resolution:=optional,
net.jcip.annotations;resolution:=optional,
@@ -68,6 +73,13 @@
azure-core,
azure-identity,
azure-json,
+ azure-xml,
+ azure-storage-blob,
+ azure-storage-common,
+ azure-storage-internal-avro,
+ com.microsoft.aad,
+ com.microsoft.aad.msal4jextensions,
+ com.microsoft.aad.msal4jextensions.persistence,
guava,
jsr305,
reactive-streams,
@@ -170,6 +182,11 @@
<groupId>com.microsoft.azure</groupId>
<artifactId>azure-storage</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.azure</groupId>
+ <artifactId>azure-storage-blob</artifactId>
+ <version>12.27.1</version>
+ </dependency>
<dependency>
<groupId>com.microsoft.azure</groupId>
<artifactId>azure-keyvault-core</artifactId>
diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AbstractAzureBlobStoreBackend.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AbstractAzureBlobStoreBackend.java
new file mode 100644
index 0000000000..9e40a18799
--- /dev/null
+++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AbstractAzureBlobStoreBackend.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;
+
+import org.apache.jackrabbit.core.data.DataIdentifier;
+import org.apache.jackrabbit.core.data.DataRecord;
+import org.apache.jackrabbit.core.data.DataStoreException;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordDownloadOptions;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUpload;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadException;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadOptions;
+import org.apache.jackrabbit.oak.spi.blob.AbstractSharedBackend;
+import org.jetbrains.annotations.NotNull;
+
+import java.net.URI;
+import java.util.Properties;
+
+
+public abstract class AbstractAzureBlobStoreBackend extends AbstractSharedBackend {
+
+ protected abstract DataRecordUpload initiateHttpUpload(long maxUploadSizeInBytes, int maxNumberOfURIs, @NotNull final DataRecordUploadOptions options);
+ protected abstract DataRecord completeHttpUpload(@NotNull String uploadTokenStr) throws DataRecordUploadException, DataStoreException;
+ protected abstract void setHttpDownloadURIExpirySeconds(int seconds);
+ protected abstract void setHttpUploadURIExpirySeconds(int seconds);
+ protected abstract void setHttpDownloadURICacheSize(int maxSize);
+ protected abstract URI createHttpDownloadURI(@NotNull DataIdentifier identifier, @NotNull DataRecordDownloadOptions downloadOptions);
+ public abstract void setProperties(final Properties properties);
+
+}
diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobContainerProvider.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobContainerProvider.java
index 01522248b0..a79cd2381e 100644
--- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobContainerProvider.java
+++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobContainerProvider.java
@@ -18,24 +18,19 @@
*/
package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;
-import com.azure.core.credential.AccessToken;
-import com.azure.core.credential.TokenRequestContext;
import com.azure.identity.ClientSecretCredential;
import com.azure.identity.ClientSecretCredentialBuilder;
-import com.microsoft.azure.storage.CloudStorageAccount;
-import com.microsoft.azure.storage.StorageCredentialsToken;
-import com.microsoft.azure.storage.StorageException;
-import com.microsoft.azure.storage.UserDelegationKey;
-import com.microsoft.azure.storage.blob.BlobRequestOptions;
-import com.microsoft.azure.storage.blob.CloudBlobClient;
-import com.microsoft.azure.storage.blob.CloudBlobContainer;
-import com.microsoft.azure.storage.blob.CloudBlockBlob;
-import com.microsoft.azure.storage.blob.SharedAccessBlobHeaders;
-import com.microsoft.azure.storage.blob.SharedAccessBlobPermissions;
-import com.microsoft.azure.storage.blob.SharedAccessBlobPolicy;
+import com.azure.storage.blob.BlobContainerClient;
+import com.azure.storage.blob.BlobContainerClientBuilder;
+import com.azure.storage.blob.BlobServiceClient;
+import com.azure.storage.blob.BlobServiceClientBuilder;
+import com.azure.storage.blob.models.UserDelegationKey;
+import com.azure.storage.blob.sas.BlobSasPermission;
+import com.azure.storage.blob.sas.BlobServiceSasSignatureValues;
+import com.azure.storage.blob.specialized.BlockBlobClient;
+import com.azure.storage.common.policy.RequestRetryOptions;
import org.apache.commons.lang3.StringUtils;
import org.apache.jackrabbit.core.data.DataStoreException;
-import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
@@ -44,23 +39,13 @@ import org.slf4j.LoggerFactory;
import java.io.Closeable;
import java.net.URISyntaxException;
import java.security.InvalidKeyException;
-import java.time.Instant;
-import java.time.LocalDateTime;
import java.time.OffsetDateTime;
-import java.time.format.DateTimeFormatter;
-import java.util.Date;
-import java.util.EnumSet;
-import java.util.Objects;
-import java.util.Optional;
+import java.time.ZoneOffset;
import java.util.Properties;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
public class AzureBlobContainerProvider implements Closeable {
private static final Logger log = LoggerFactory.getLogger(AzureBlobContainerProvider.class);
private static final String DEFAULT_ENDPOINT_SUFFIX = "core.windows.net";
- private static final String AZURE_DEFAULT_SCOPE = "https://storage.azure.com/.default";
private final String azureConnectionString;
private final String accountName;
private final String containerName;
@@ -70,12 +55,6 @@ public class AzureBlobContainerProvider implements Closeable {
private final String tenantId;
private final String clientId;
private final String clientSecret;
- private ClientSecretCredential clientSecretCredential;
- private AccessToken accessToken;
- private StorageCredentialsToken storageCredentialsToken;
- private static final long TOKEN_REFRESHER_INITIAL_DELAY = 45L;
- private static final long TOKEN_REFRESHER_DELAY = 1L;
- private final ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
private AzureBlobContainerProvider(Builder builder) {
this.azureConnectionString = builder.azureConnectionString;
@@ -89,6 +68,8 @@ public class AzureBlobContainerProvider implements Closeable {
this.clientSecret = builder.clientSecret;
}
+ @Override
+ public void close() {}
public static class Builder {
private final String containerName;
@@ -171,141 +152,64 @@ public class AzureBlobContainerProvider implements Closeable {
return containerName;
}
+ public String getAzureConnectionString() {
+ return azureConnectionString;
+ }
+
@NotNull
- public CloudBlobContainer getBlobContainer() throws DataStoreException {
- return this.getBlobContainer(null);
+ public BlobContainerClient getBlobContainer() throws DataStoreException {
+ return this.getBlobContainer(null, new Properties());
}
@NotNull
- public CloudBlobContainer getBlobContainer(@Nullable BlobRequestOptions blobRequestOptions) throws DataStoreException {
+ public BlobContainerClient getBlobContainer(@Nullable RequestRetryOptions retryOptions, Properties properties) throws DataStoreException {
// connection string will be given preference over service principals / sas / account key
if (StringUtils.isNotBlank(azureConnectionString)) {
log.debug("connecting to azure blob storage via azureConnectionString");
- return Utils.getBlobContainer(azureConnectionString, containerName, blobRequestOptions);
+ return Utils.getBlobContainerFromConnectionString(getAzureConnectionString(), containerName);
} else if (authenticateViaServicePrincipal()) {
log.debug("connecting to azure blob storage via service principal credentials");
- return getBlobContainerFromServicePrincipals(blobRequestOptions);
+ return getBlobContainerFromServicePrincipals(accountName, retryOptions);
} else if (StringUtils.isNotBlank(sasToken)) {
log.debug("connecting to azure blob storage via sas token");
final String connectionStringWithSasToken = Utils.getConnectionStringForSas(sasToken, blobEndpoint, accountName);
- return Utils.getBlobContainer(connectionStringWithSasToken, containerName, blobRequestOptions);
+ return Utils.getBlobContainer(connectionStringWithSasToken, containerName, retryOptions, properties);
}
log.debug("connecting to azure blob storage via access key");
final String connectionStringWithAccountKey = Utils.getConnectionString(accountName, accountKey, blobEndpoint);
- return Utils.getBlobContainer(connectionStringWithAccountKey, containerName, blobRequestOptions);
- }
-
- @NotNull
- private CloudBlobContainer getBlobContainerFromServicePrincipals(@Nullable BlobRequestOptions blobRequestOptions) throws DataStoreException {
- StorageCredentialsToken storageCredentialsToken = getStorageCredentials();
- try {
- CloudStorageAccount cloud = new CloudStorageAccount(storageCredentialsToken, true, DEFAULT_ENDPOINT_SUFFIX, accountName);
- CloudBlobClient cloudBlobClient = cloud.createCloudBlobClient();
- if (blobRequestOptions != null) {
- cloudBlobClient.setDefaultRequestOptions(blobRequestOptions);
- }
- return cloudBlobClient.getContainerReference(containerName);
- } catch (URISyntaxException | StorageException e) {
- throw new DataStoreException(e);
- }
- }
-
- @NotNull
- private StorageCredentialsToken getStorageCredentials() {
- boolean isAccessTokenGenerated = false;
- /* generate access token, the same token will be used for subsequent access
- * generated token is valid for 1 hour only and will be refreshed in background
- * */
- if (accessToken == null) {
- clientSecretCredential = new ClientSecretCredentialBuilder()
- .clientId(clientId)
- .clientSecret(clientSecret)
- .tenantId(tenantId)
- .build();
- accessToken = clientSecretCredential.getTokenSync(new TokenRequestContext().addScopes(AZURE_DEFAULT_SCOPE));
- if (accessToken == null || StringUtils.isBlank(accessToken.getToken())) {
- log.error("Access token is null or empty");
- throw new IllegalArgumentException("Could not connect to azure storage, access token is null or empty");
- }
- storageCredentialsToken = new StorageCredentialsToken(accountName, accessToken.getToken());
- isAccessTokenGenerated = true;
- }
-
- Objects.requireNonNull(storageCredentialsToken, "storage credentials token cannot be null");
-
- // start refresh token executor only when the access token is first generated
- if (isAccessTokenGenerated) {
- log.info("starting refresh token task at: {}", OffsetDateTime.now());
- TokenRefresher tokenRefresher = new TokenRefresher();
- executorService.scheduleWithFixedDelay(tokenRefresher, TOKEN_REFRESHER_INITIAL_DELAY, TOKEN_REFRESHER_DELAY, TimeUnit.MINUTES);
- }
- return storageCredentialsToken;
+ return Utils.getBlobContainer(connectionStringWithAccountKey, containerName, retryOptions, properties);
}
@NotNull
- public String generateSharedAccessSignature(BlobRequestOptions requestOptions,
+ public String generateSharedAccessSignature(RequestRetryOptions retryOptions,
String key,
- EnumSet<SharedAccessBlobPermissions> permissions,
+ BlobSasPermission blobSasPermissions,
int expirySeconds,
- SharedAccessBlobHeaders optionalHeaders) throws DataStoreException, URISyntaxException, StorageException, InvalidKeyException {
- SharedAccessBlobPolicy policy = new SharedAccessBlobPolicy();
- Date expiry = Date.from(Instant.now().plusSeconds(expirySeconds));
- policy.setSharedAccessExpiryTime(expiry);
- policy.setPermissions(permissions);
+ Properties properties) throws DataStoreException, URISyntaxException, InvalidKeyException {
+
+ OffsetDateTime expiry = OffsetDateTime.now().plusSeconds(expirySeconds);
+ BlobServiceSasSignatureValues serviceSasSignatureValues = new BlobServiceSasSignatureValues(expiry, blobSasPermissions);
- CloudBlockBlob blob = getBlobContainer(requestOptions).getBlockBlobReference(key);
+ BlockBlobClient blob = getBlobContainer(retryOptions, properties).getBlobClient(key).getBlockBlobClient();
if (authenticateViaServicePrincipal()) {
- return generateUserDelegationKeySignedSas(blob, policy, optionalHeaders, expiry);
+ return generateUserDelegationKeySignedSas(blob, serviceSasSignatureValues, expiry);
}
- return generateSas(blob, policy, optionalHeaders);
- }
-
- @NotNull
- private String generateUserDelegationKeySignedSas(CloudBlockBlob blob,
- SharedAccessBlobPolicy policy,
- SharedAccessBlobHeaders optionalHeaders,
- Date expiry) throws StorageException {
- fillEmptyHeaders(optionalHeaders);
- UserDelegationKey userDelegationKey = blob.getServiceClient().getUserDelegationKey(Date.from(Instant.now().minusSeconds(900)),
- expiry);
- return optionalHeaders == null ? blob.generateUserDelegationSharedAccessSignature(userDelegationKey, policy) :
- blob.generateUserDelegationSharedAccessSignature(userDelegationKey, policy, optionalHeaders, null, null);
- }
-
- /* set empty headers as blank string due to a bug in Azure SDK
- * Azure SDK considers null headers as 'null' string which corrupts the string to sign and generates an invalid
- * sas token
- * */
- private void fillEmptyHeaders(SharedAccessBlobHeaders sharedAccessBlobHeaders) {
- final String EMPTY_STRING = "";
- Optional.ofNullable(sharedAccessBlobHeaders)
- .ifPresent(headers -> {
- if (StringUtils.isBlank(headers.getCacheControl())) {
- headers.setCacheControl(EMPTY_STRING);
- }
- if (StringUtils.isBlank(headers.getContentDisposition())) {
- headers.setContentDisposition(EMPTY_STRING);
- }
- if (StringUtils.isBlank(headers.getContentEncoding())) {
- headers.setContentEncoding(EMPTY_STRING);
- }
- if (StringUtils.isBlank(headers.getContentLanguage())) {
- headers.setContentLanguage(EMPTY_STRING);
- }
- if (StringUtils.isBlank(headers.getContentType())) {
- headers.setContentType(EMPTY_STRING);
- }
- });
+ return generateSas(blob, serviceSasSignatureValues);
}
@NotNull
- private String generateSas(CloudBlockBlob blob,
- SharedAccessBlobPolicy policy,
- SharedAccessBlobHeaders optionalHeaders) throws InvalidKeyException, StorageException {
- return optionalHeaders == null ? blob.generateSharedAccessSignature(policy, null) :
- blob.generateSharedAccessSignature(policy,
- optionalHeaders, null, null, null, true);
+ public String generateUserDelegationKeySignedSas(BlockBlobClient blobClient,
+ BlobServiceSasSignatureValues serviceSasSignatureValues,
+ OffsetDateTime expiryTime) {
+
+ BlobServiceClient blobServiceClient = new BlobServiceClientBuilder()
+ .endpoint(String.format(String.format("https://%s.%s", accountName, DEFAULT_ENDPOINT_SUFFIX)))
+ .credential(getClientSecretCredential())
+ .buildClient();
+ OffsetDateTime startTime = OffsetDateTime.now(ZoneOffset.UTC);
+ UserDelegationKey userDelegationKey = blobServiceClient.getUserDelegationKey(startTime, expiryTime);
+ return blobClient.generateUserDelegationSas(serviceSasSignatureValues, userDelegationKey);
}
private boolean authenticateViaServicePrincipal() {
@@ -313,34 +217,27 @@ public class AzureBlobContainerProvider implements Closeable {
StringUtils.isNoneBlank(accountName, tenantId, clientId, clientSecret);
}
- private class TokenRefresher implements Runnable {
- @Override
- public void run() {
- try {
- log.debug("Checking for azure access token expiry at: {}", LocalDateTime.now());
- OffsetDateTime tokenExpiryThreshold = OffsetDateTime.now().plusMinutes(5);
- if (accessToken.getExpiresAt() != null && accessToken.getExpiresAt().isBefore(tokenExpiryThreshold)) {
- log.info("Access token is about to expire (5 minutes or less) at: {}. New access token will be generated",
- accessToken.getExpiresAt().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME));
- AccessToken newToken = clientSecretCredential.getTokenSync(new TokenRequestContext().addScopes(AZURE_DEFAULT_SCOPE));
- log.info("New azure access token generated at: {}", LocalDateTime.now());
- if (newToken == null || StringUtils.isBlank(newToken.getToken())) {
- log.error("New access token is null or empty");
- return;
- }
- // update access token with newly generated token
- accessToken = newToken;
- storageCredentialsToken.updateToken(accessToken.getToken());
- }
- } catch (Exception e) {
- log.error("Error while acquiring new access token: ", e);
- }
- }
+ private ClientSecretCredential getClientSecretCredential() {
+ return new ClientSecretCredentialBuilder()
+ .clientId(clientId)
+ .clientSecret(clientSecret)
+ .tenantId(tenantId)
+ .build();
}
- @Override
- public void close() {
- new ExecutorCloser(executorService).close();
- log.info("Refresh token executor service shutdown completed");
+ @NotNull
+ private BlobContainerClient getBlobContainerFromServicePrincipals(String accountName, RequestRetryOptions retryOptions) {
+ ClientSecretCredential clientSecretCredential = getClientSecretCredential();
+ return new BlobContainerClientBuilder()
+ .endpoint(String.format(String.format("https://%s.%s", accountName, DEFAULT_ENDPOINT_SUFFIX)))
+ .credential(clientSecretCredential)
+ .retryOptions(retryOptions)
+ .buildClient();
+ }
+
+ @NotNull
+ private String generateSas(BlockBlobClient blob,
+ BlobServiceSasSignatureValues blobServiceSasSignatureValues) {
+ return blob.generateSas(blobServiceSasSignatureValues, null);
}
-}
+}
\ No newline at end of file
diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
index c60ad2e868..f61ecb63f0 100644
--- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
+++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
@@ -18,15 +18,42 @@
*/
package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;
-import static java.lang.Thread.currentThread;
-
-import java.io.BufferedInputStream;
-import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
+import com.azure.core.http.rest.Response;
+import com.azure.core.util.BinaryData;
+import com.azure.storage.blob.BlobClient;
+import com.azure.storage.blob.BlobContainerClient;
+import com.azure.storage.blob.models.BlobContainerProperties;
+import com.azure.storage.blob.models.BlobItem;
+import com.azure.storage.blob.models.BlobProperties;
+import com.azure.storage.blob.models.BlobStorageException;
+import com.azure.storage.blob.models.Block;
+import com.azure.storage.blob.models.BlockBlobItem;
+import com.azure.storage.blob.models.BlockListType;
+import com.azure.storage.blob.models.ListBlobsOptions;
+import com.azure.storage.blob.models.ParallelTransferOptions;
+import com.azure.storage.blob.options.BlobUploadFromFileOptions;
+import com.azure.storage.blob.sas.BlobSasPermission;
+import com.azure.storage.blob.specialized.BlockBlobClient;
+import com.azure.storage.common.policy.RequestRetryOptions;
+import com.microsoft.azure.storage.RetryPolicy;
+import org.apache.commons.io.IOUtils;
+import org.apache.jackrabbit.core.data.DataIdentifier;
+import org.apache.jackrabbit.core.data.DataRecord;
+import org.apache.jackrabbit.core.data.DataStoreException;
+import com.google.common.base.Strings;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.jackrabbit.oak.commons.PropertiesUtil;
+import org.apache.jackrabbit.oak.spi.blob.AbstractDataRecord;
+import org.apache.jackrabbit.oak.spi.blob.AbstractSharedBackend;
+import org.apache.jackrabbit.util.Base64;
+import org.jetbrains.annotations.NotNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URLEncoder;
@@ -35,85 +62,48 @@ import java.security.InvalidKeyException;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.EnumSet;
-import java.util.HashMap;
import java.util.Iterator;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Properties;
-import java.util.Queue;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
-import java.util.function.Function;
+
+import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
-import org.apache.jackrabbit.guava.common.cache.Cache;
-import org.apache.jackrabbit.guava.common.cache.CacheBuilder;
-import org.apache.jackrabbit.guava.common.collect.AbstractIterator;
-import com.microsoft.azure.storage.AccessCondition;
-import com.microsoft.azure.storage.LocationMode;
-import com.microsoft.azure.storage.ResultContinuation;
-import com.microsoft.azure.storage.ResultSegment;
-import com.microsoft.azure.storage.RetryPolicy;
-import com.microsoft.azure.storage.StorageException;
-import com.microsoft.azure.storage.blob.BlobListingDetails;
-import com.microsoft.azure.storage.blob.BlobRequestOptions;
-import com.microsoft.azure.storage.blob.BlockEntry;
-import com.microsoft.azure.storage.blob.BlockListingFilter;
-import com.microsoft.azure.storage.blob.CloudBlob;
-import com.microsoft.azure.storage.blob.CloudBlobContainer;
-import com.microsoft.azure.storage.blob.CloudBlobDirectory;
-import com.microsoft.azure.storage.blob.CloudBlockBlob;
-import com.microsoft.azure.storage.blob.CopyStatus;
-import com.microsoft.azure.storage.blob.ListBlobItem;
-import com.microsoft.azure.storage.blob.SharedAccessBlobHeaders;
-import com.microsoft.azure.storage.blob.SharedAccessBlobPermissions;
-import org.apache.commons.io.IOUtils;
-import org.apache.jackrabbit.core.data.DataIdentifier;
-import org.apache.jackrabbit.core.data.DataRecord;
-import org.apache.jackrabbit.core.data.DataStoreException;
-import org.apache.jackrabbit.oak.commons.PropertiesUtil;
+
import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordDownloadOptions;
import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUpload;
import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadException;
import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadOptions;
import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadToken;
-import org.apache.jackrabbit.oak.spi.blob.AbstractDataRecord;
-import org.apache.jackrabbit.oak.spi.blob.AbstractSharedBackend;
-import org.apache.jackrabbit.util.Base64;
-import org.jetbrains.annotations.NotNull;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-public class AzureBlobStoreBackend extends AbstractSharedBackend {
+import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_BUFFERED_STREAM_THRESHOLD;
+import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT;
+import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_LAST_MODIFIED_KEY;
+import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MAX_ALLOWABLE_UPLOAD_URIS;
+import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MAX_BINARY_UPLOAD_SIZE;
+import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT;
+import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MAX_MULTIPART_UPLOAD_PART_SIZE;
+import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MAX_SINGLE_PUT_UPLOAD_SIZE;
+import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_META_KEY_PREFIX;
+import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE;
+import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_REF_KEY;
+
+
+public class AzureBlobStoreBackend extends AbstractAzureBlobStoreBackend {
private static final Logger LOG = LoggerFactory.getLogger(AzureBlobStoreBackend.class);
private static final Logger LOG_STREAMS_DOWNLOAD = LoggerFactory.getLogger("oak.datastore.download.streams");
private static final Logger LOG_STREAMS_UPLOAD = LoggerFactory.getLogger("oak.datastore.upload.streams");
- private static final String META_DIR_NAME = "META";
- private static final String META_KEY_PREFIX = META_DIR_NAME + "/";
-
- private static final String REF_KEY = "reference.key";
- private static final String LAST_MODIFIED_KEY = "lastModified";
-
- private static final long BUFFERED_STREAM_THRESHOLD = 1024 * 1024;
- static final long MIN_MULTIPART_UPLOAD_PART_SIZE = 1024 * 1024 * 10; // 10MB
- static final long MAX_MULTIPART_UPLOAD_PART_SIZE = 1024 * 1024 * 100; // 100MB
- static final long MAX_SINGLE_PUT_UPLOAD_SIZE = 1024 * 1024 * 256; // 256MB, Azure limit
- static final long MAX_BINARY_UPLOAD_SIZE = (long) Math.floor(1024L * 1024L * 1024L * 1024L * 4.75); // 4.75TB, Azure limit
- private static final int MAX_ALLOWABLE_UPLOAD_URIS = 50000; // Azure limit
- private static final int MAX_UNIQUE_RECORD_TRIES = 10;
- private static final int DEFAULT_CONCURRENT_REQUEST_COUNT = 2;
- private static final int MAX_CONCURRENT_REQUEST_COUNT = 50;
-
private Properties properties;
private AzureBlobContainerProvider azureBlobContainerProvider;
- private int concurrentRequestCount = DEFAULT_CONCURRENT_REQUEST_COUNT;
- private RetryPolicy retryPolicy;
+ private int concurrentRequestCount = AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT;
+ private RequestRetryOptions retryOptions;
private Integer requestTimeout;
private int httpDownloadURIExpirySeconds = 0; // disabled by default
private int httpUploadURIExpirySeconds = 0; // disabled by default
@@ -121,7 +111,6 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
private String downloadDomainOverride = null;
private boolean createBlobContainer = true;
private boolean presignedDownloadURIVerifyExists = true;
- private boolean enableSecondaryLocation = AzureConstants.AZURE_BLOB_ENABLE_SECONDARY_LOCATION_DEFAULT;
private Cache<String, URI> httpDownloadURICache;
@@ -130,36 +119,19 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
public void setProperties(final Properties properties) {
this.properties = properties;
}
+ private volatile BlobContainerClient azureContainer = null;
- private volatile CloudBlobContainer azureContainer = null;
-
- protected CloudBlobContainer getAzureContainer() throws DataStoreException {
+ protected BlobContainerClient getAzureContainer() throws DataStoreException {
if (azureContainer == null) {
synchronized (this) {
if (azureContainer == null) {
- azureContainer = azureBlobContainerProvider.getBlobContainer(getBlobRequestOptions());
+ azureContainer = azureBlobContainerProvider.getBlobContainer(retryOptions, properties);
}
}
}
return azureContainer;
}
- @NotNull
- protected BlobRequestOptions getBlobRequestOptions() {
- BlobRequestOptions requestOptions = new BlobRequestOptions();
- if (null != retryPolicy) {
- requestOptions.setRetryPolicyFactory(retryPolicy);
- }
- if (null != requestTimeout) {
- requestOptions.setTimeoutIntervalInMs(requestTimeout);
- }
- requestOptions.setConcurrentRequestCount(concurrentRequestCount);
- if (enableSecondaryLocation) {
- requestOptions.setLocationMode(LocationMode.PRIMARY_THEN_SECONDARY);
- }
- return requestOptions;
- }
-
@Override
public void init() throws DataStoreException {
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
@@ -171,47 +143,42 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
if (null == properties) {
try {
properties = Utils.readConfig(Utils.DEFAULT_CONFIG_FILE);
- }
- catch (IOException e) {
+ } catch (IOException e) {
throw new DataStoreException("Unable to initialize Azure Data Store from " + Utils.DEFAULT_CONFIG_FILE, e);
}
}
try {
- Utils.setProxyIfNeeded(properties);
createBlobContainer = PropertiesUtil.toBoolean(
org.apache.jackrabbit.oak.commons.StringUtils.emptyToNull(properties.getProperty(AzureConstants.AZURE_CREATE_CONTAINER)), true);
initAzureDSConfig();
concurrentRequestCount = PropertiesUtil.toInteger(
properties.getProperty(AzureConstants.AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION),
- DEFAULT_CONCURRENT_REQUEST_COUNT);
- if (concurrentRequestCount < DEFAULT_CONCURRENT_REQUEST_COUNT) {
+ AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT);
+ if (concurrentRequestCount < AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT) {
LOG.warn("Invalid setting [{}] for concurrentRequestsPerOperation (too low); resetting to {}",
concurrentRequestCount,
- DEFAULT_CONCURRENT_REQUEST_COUNT);
- concurrentRequestCount = DEFAULT_CONCURRENT_REQUEST_COUNT;
- } else if (concurrentRequestCount > MAX_CONCURRENT_REQUEST_COUNT) {
+ AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT);
+ concurrentRequestCount = AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT;
+ } else if (concurrentRequestCount > AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT) {
LOG.warn("Invalid setting [{}] for concurrentRequestsPerOperation (too high); resetting to {}",
concurrentRequestCount,
- MAX_CONCURRENT_REQUEST_COUNT);
- concurrentRequestCount = MAX_CONCURRENT_REQUEST_COUNT;
+ AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT);
+ concurrentRequestCount = AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT;
}
LOG.info("Using concurrentRequestsPerOperation={}", concurrentRequestCount);
- retryPolicy = Utils.getRetryPolicy(properties.getProperty(AzureConstants.AZURE_BLOB_MAX_REQUEST_RETRY));
if (properties.getProperty(AzureConstants.AZURE_BLOB_REQUEST_TIMEOUT) != null) {
requestTimeout = PropertiesUtil.toInteger(properties.getProperty(AzureConstants.AZURE_BLOB_REQUEST_TIMEOUT), RetryPolicy.DEFAULT_CLIENT_RETRY_COUNT);
}
+
+ retryOptions = Utils.getRetryOptions(properties.getProperty(AzureConstants.AZURE_BLOB_MAX_REQUEST_RETRY), requestTimeout, computeSecondaryLocationEndpoint());
+
presignedDownloadURIVerifyExists = PropertiesUtil.toBoolean(
org.apache.jackrabbit.oak.commons.StringUtils.emptyToNull(properties.getProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_VERIFY_EXISTS)), true);
- enableSecondaryLocation = PropertiesUtil.toBoolean(
- properties.getProperty(AzureConstants.AZURE_BLOB_ENABLE_SECONDARY_LOCATION_NAME),
- AzureConstants.AZURE_BLOB_ENABLE_SECONDARY_LOCATION_DEFAULT
- );
-
- CloudBlobContainer azureContainer = getAzureContainer();
+ BlobContainerClient azureContainer = getAzureContainer();
if (createBlobContainer && !azureContainer.exists()) {
azureContainer.create();
@@ -232,8 +199,7 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
String cacheMaxSize = properties.getProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_CACHE_MAX_SIZE);
if (null != cacheMaxSize) {
this.setHttpDownloadURICacheSize(Integer.parseInt(cacheMaxSize));
- }
- else {
+ } else {
this.setHttpDownloadURICacheSize(0); // default
}
}
@@ -243,16 +209,13 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
// Initialize reference key secret
boolean createRefSecretOnInit = PropertiesUtil.toBoolean(
org.apache.jackrabbit.oak.commons.StringUtils.emptyToNull(properties.getProperty(AzureConstants.AZURE_REF_ON_INIT)), true);
-
if (createRefSecretOnInit) {
getOrCreateReferenceKey();
}
- }
- catch (StorageException e) {
+ } catch (BlobStorageException e) {
throw new DataStoreException(e);
}
- }
- finally {
+ } finally {
Thread.currentThread().setContextClassLoader(contextClassLoader);
}
}
@@ -280,7 +243,7 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
try {
Thread.currentThread().setContextClassLoader(
getClass().getClassLoader());
- CloudBlockBlob blob = getAzureContainer().getBlockBlobReference(key);
+ BlockBlobClient blob = getAzureContainer().getBlobClient(key).getBlockBlobClient();
if (!blob.exists()) {
throw new DataStoreException(String.format("Trying to read missing blob. identifier=%s", key));
}
@@ -288,18 +251,13 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
InputStream is = blob.openInputStream();
LOG.debug("Got input stream for blob. identifier={} duration={}", key, (System.currentTimeMillis() - start));
if (LOG_STREAMS_DOWNLOAD.isDebugEnabled()) {
- // Log message, with exception so we can get a trace to see where the call came from
+ // Log message, with exception, so we can get a trace to see where the call came from
LOG_STREAMS_DOWNLOAD.debug("Binary downloaded from Azure Blob Storage - identifier={}", key, new Exception());
}
return is;
- }
- catch (StorageException e) {
+ } catch (BlobStorageException e) {
LOG.info("Error reading blob. identifier={}", key);
throw new DataStoreException(String.format("Cannot read blob. identifier=%s", key), e);
- }
- catch (URISyntaxException e) {
- LOG.debug("Error reading blob. identifier={}", key);
- throw new DataStoreException(String.format("Cannot read blob. identifier=%s", key), e);
} finally {
if (contextClassLoader != null) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
@@ -307,12 +265,40 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
}
}
+ private void uploadBlob(BlockBlobClient client, File file, long len, long start, String key) throws IOException {
+
+ boolean useBufferedStream = len < AZURE_BLOB_BUFFERED_STREAM_THRESHOLD;
+ try (InputStream in = useBufferedStream ?
+ new BufferedInputStream(new FileInputStream(file))
+ : new FileInputStream(file)) {
+
+ ParallelTransferOptions parallelTransferOptions = new ParallelTransferOptions()
+ .setBlockSizeLong(len)
+ .setMaxConcurrency(concurrentRequestCount)
+ .setMaxSingleUploadSizeLong(AZURE_BLOB_MAX_SINGLE_PUT_UPLOAD_SIZE);
+ BlobUploadFromFileOptions options = new BlobUploadFromFileOptions(file.toString());
+ options.setParallelTransferOptions(parallelTransferOptions);
+ try {
+ BlobClient blobClient = client.getContainerClient().getBlobClient(file.getName());
+ Response<BlockBlobItem> blockBlob = blobClient.uploadFromFileWithResponse(options, null, null);
+ LOG.debug("Upload status is {} for blob {}", blockBlob.getStatusCode(), key);
+ } catch (UncheckedIOException ex) {
+ System.err.printf("Failed to upload from file: %s%n", ex.getMessage());
+ }
+ LOG.debug("Blob created. identifier={} length={} duration={} buffered={}", key, len, (System.currentTimeMillis() - start), useBufferedStream);
+ if (LOG_STREAMS_UPLOAD.isDebugEnabled()) {
+ // Log message, with exception, so we can get a trace to see where the call came from
+ LOG_STREAMS_UPLOAD.debug("Binary uploaded to Azure Blob Storage - identifier={}", key, new Exception());
+ }
+ }
+ }
+
@Override
public void write(DataIdentifier identifier, File file) throws DataStoreException {
- if (null == identifier) {
+ if (identifier == null) {
throw new NullPointerException("identifier");
}
- if (null == file) {
+ if (file == null) {
throw new NullPointerException("file");
}
String key = getKeyName(identifier);
@@ -324,110 +310,40 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
long len = file.length();
LOG.debug("Blob write started. identifier={} length={}", key, len);
- CloudBlockBlob blob = getAzureContainer().getBlockBlobReference(key);
+ BlockBlobClient blob = getAzureContainer().getBlobClient(key).getBlockBlobClient();
if (!blob.exists()) {
- addLastModified(blob);
-
- BlobRequestOptions options = new BlobRequestOptions();
- options.setConcurrentRequestCount(concurrentRequestCount);
- boolean useBufferedStream = len < BUFFERED_STREAM_THRESHOLD;
- final InputStream in = useBufferedStream ? new BufferedInputStream(new FileInputStream(file)) : new FileInputStream(file);
- try {
- blob.upload(in, len, null, options, null);
- LOG.debug("Blob created. identifier={} length={} duration={} buffered={}", key, len, (System.currentTimeMillis() - start), useBufferedStream);
- if (LOG_STREAMS_UPLOAD.isDebugEnabled()) {
- // Log message, with exception so we can get a trace to see where the call came from
- LOG_STREAMS_UPLOAD.debug("Binary uploaded to Azure Blob Storage - identifier={}", key, new Exception());
- }
- } finally {
- in.close();
- }
+ updateLastModifiedMetadata(blob);
+ uploadBlob(blob, file, len, start, key);
return;
}
- blob.downloadAttributes();
- if (blob.getProperties().getLength() != len) {
+ if (blob.getProperties().getBlobSize() != len) {
throw new DataStoreException("Length Collision. identifier=" + key +
- " new length=" + len +
- " old length=" + blob.getProperties().getLength());
+ " new length=" + len +
+ " old length=" + blob.getProperties().getBlobSize());
}
LOG.trace("Blob already exists. identifier={} lastModified={}", key, getLastModified(blob));
- addLastModified(blob);
- blob.uploadMetadata();
+ updateLastModifiedMetadata(blob);
LOG.debug("Blob updated. identifier={} lastModified={} duration={}", key,
- getLastModified(blob), (System.currentTimeMillis() - start));
- }
- catch (StorageException e) {
+ getLastModified(blob), (System.currentTimeMillis() - start));
+ } catch (BlobStorageException e) {
LOG.info("Error writing blob. identifier={}", key, e);
throw new DataStoreException(String.format("Cannot write blob. identifier=%s", key), e);
- }
- catch (URISyntaxException | IOException e) {
+ } catch (IOException e) {
LOG.debug("Error writing blob. identifier={}", key, e);
throw new DataStoreException(String.format("Cannot write blob. identifier=%s", key), e);
} finally {
- if (null != contextClassLoader) {
+ if (contextClassLoader != null) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
}
}
}
- private static boolean waitForCopy(CloudBlob blob) throws StorageException, InterruptedException {
- boolean continueLoop = true;
- CopyStatus status = CopyStatus.PENDING;
- while (continueLoop) {
- blob.downloadAttributes();
- status = blob.getCopyState().getStatus();
- continueLoop = status == CopyStatus.PENDING;
- // Sleep if retry is needed
- if (continueLoop) {
- Thread.sleep(500);
- }
- }
- return status == CopyStatus.SUCCESS;
- }
-
- @Override
- public byte[] getOrCreateReferenceKey() throws DataStoreException {
- try {
- if (secret != null && secret.length != 0) {
- return secret;
- } else {
- byte[] key;
- // Try reading from the metadata folder if it exists
- key = readMetadataBytes(REF_KEY);
- if (key == null) {
- key = super.getOrCreateReferenceKey();
- addMetadataRecord(new ByteArrayInputStream(key), REF_KEY);
- key = readMetadataBytes(REF_KEY);
- }
- secret = key;
- return secret;
- }
- } catch (IOException e) {
- throw new DataStoreException("Unable to get or create key " + e);
- }
- }
-
- private byte[] readMetadataBytes(String name) throws IOException, DataStoreException {
- DataRecord rec = getMetadataRecord(name);
- byte[] key = null;
- if (rec != null) {
- InputStream stream = null;
- try {
- stream = rec.getStream();
- return IOUtils.toByteArray(stream);
- } finally {
- IOUtils.closeQuietly(stream);
- }
- }
- return key;
- }
-
@Override
public DataRecord getRecord(DataIdentifier identifier) throws DataStoreException {
- if (null == identifier) {
+ if (identifier == null) {
throw new NullPointerException("identifier");
}
String key = getKeyName(identifier);
@@ -436,30 +352,23 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
try {
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
- CloudBlockBlob blob = getAzureContainer().getBlockBlobReference(key);
- blob.downloadAttributes();
+ BlockBlobClient blob = getAzureContainer().getBlobClient(key).getBlockBlobClient();
AzureBlobStoreDataRecord record = new AzureBlobStoreDataRecord(
this,
azureBlobContainerProvider,
- new DataIdentifier(getIdentifierName(blob.getName())),
+ new DataIdentifier(getIdentifierName(blob.getBlobName())),
getLastModified(blob),
- blob.getProperties().getLength());
+ blob.getProperties().getBlobSize());
LOG.debug("Data record read for blob. identifier={} duration={} record={}",
- key, (System.currentTimeMillis() - start), record);
+ key, (System.currentTimeMillis() - start), record);
return record;
- }
- catch (StorageException e) {
- if (404 == e.getHttpStatusCode()) {
+ } catch (BlobStorageException e) {
+ if (e.getStatusCode() == 404) {
LOG.debug("Unable to get record for blob; blob does not exist. identifier={}", key);
- }
- else {
+ } else {
LOG.info("Error getting data record for blob. identifier={}", key, e);
}
throw new DataStoreException(String.format("Cannot retrieve blob. identifier=%s", key), e);
- }
- catch (URISyntaxException e) {
- LOG.debug("Error getting data record for blob. identifier={}", key, e);
- throw new DataStoreException(String.format("Cannot retrieve blob. identifier=%s", key), e);
} finally {
if (contextClassLoader != null) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
@@ -468,24 +377,24 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
}
@Override
- public Iterator<DataIdentifier> getAllIdentifiers() {
- return new RecordsIterator<>(
- input -> new DataIdentifier(getIdentifierName(input.getName())));
+ public Iterator<DataIdentifier> getAllIdentifiers() throws DataStoreException {
+ return getAzureContainer().listBlobs().stream()
+ .map(blobItem -> new DataIdentifier(getIdentifierName(blobItem.getName())))
+ .iterator();
}
-
-
@Override
- public Iterator<DataRecord> getAllRecords() {
+ public Iterator<DataRecord> getAllRecords() throws DataStoreException {
final AbstractSharedBackend backend = this;
- return new RecordsIterator<>(
- input -> new AzureBlobStoreDataRecord(
+ final BlobContainerClient containerClient = getAzureContainer();
+ return containerClient.listBlobs().stream()
+ .map(blobItem -> (DataRecord) new AzureBlobStoreDataRecord(
backend,
azureBlobContainerProvider,
- new DataIdentifier(getIdentifierName(input.getName())),
- input.getLastModified(),
- input.getLength())
- );
+ new DataIdentifier(getIdentifierName(blobItem.getName())),
+ getLastModified(containerClient.getBlobClient(blobItem.getName()).getBlockBlobClient()),
+ blobItem.getProperties().getContentLength()))
+ .iterator();
}
@Override
@@ -496,22 +405,20 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
try {
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
- boolean exists =getAzureContainer().getBlockBlobReference(key).exists();
+ boolean exists = getAzureContainer().getBlobClient(key).getBlockBlobClient().exists();
LOG.debug("Blob exists={} identifier={} duration={}", exists, key, (System.currentTimeMillis() - start));
return exists;
- }
- catch (Exception e) {
+ } catch (Exception e) {
throw new DataStoreException(e);
- }
- finally {
- if (null != contextClassLoader) {
+ } finally {
+ if (contextClassLoader != null) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
}
}
}
@Override
- public void close() throws DataStoreException {
+ public void close(){
azureBlobContainerProvider.close();
LOG.info("AzureBlobBackend closed.");
}
@@ -526,17 +433,13 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
try {
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
- boolean result = getAzureContainer().getBlockBlobReference(key).deleteIfExists();
+ boolean result = getAzureContainer().getBlobClient(key).getBlockBlobClient().deleteIfExists();
LOG.debug("Blob {}. identifier={} duration={}",
result ? "deleted" : "delete requested, but it does not exist (perhaps already deleted)",
key, (System.currentTimeMillis() - start));
- }
- catch (StorageException e) {
+ } catch (BlobStorageException e) {
LOG.info("Error deleting blob. identifier={}", key, e);
throw new DataStoreException(e);
- }
- catch (URISyntaxException e) {
- throw new DataStoreException(e);
} finally {
if (contextClassLoader != null) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
@@ -546,7 +449,7 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
@Override
public void addMetadataRecord(InputStream input, String name) throws DataStoreException {
- if (null == input) {
+ if (input == null) {
throw new NullPointerException("input");
}
if (StringUtils.isEmpty(name)) {
@@ -557,11 +460,11 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
try {
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
- addMetadataRecordImpl(input, name, -1L);
+ addMetadataRecordImpl(input, name, -1);
LOG.debug("Metadata record added. metadataName={} duration={}", name, (System.currentTimeMillis() - start));
}
finally {
- if (null != contextClassLoader) {
+ if (contextClassLoader != null) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
}
}
@@ -569,7 +472,7 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
@Override
public void addMetadataRecord(File input, String name) throws DataStoreException {
- if (null == input) {
+ if (input == null) {
throw new NullPointerException("input");
}
if (StringUtils.isEmpty(name)) {
@@ -582,30 +485,29 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
addMetadataRecordImpl(new FileInputStream(input), name, input.length());
LOG.debug("Metadata record added. metadataName={} duration={}", name, (System.currentTimeMillis() - start));
- }
- catch (FileNotFoundException e) {
+ } catch (FileNotFoundException e) {
throw new DataStoreException(e);
- }
- finally {
- if (null != contextClassLoader) {
+ } finally {
+ if (contextClassLoader != null) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
}
}
}
+ private BlockBlobClient getMetaBlobClient(String name) throws DataStoreException {
+ return getAzureContainer().getBlobClient(AzureConstants.AZUre_BlOB_META_DIR_NAME + "/" + name).getBlockBlobClient();
+ }
+
private void addMetadataRecordImpl(final InputStream input, String name, long recordLength) throws DataStoreException {
try {
- CloudBlobDirectory metaDir = getAzureContainer().getDirectoryReference(META_DIR_NAME);
- CloudBlockBlob blob = metaDir.getBlockBlobReference(name);
- addLastModified(blob);
- blob.upload(input, recordLength);
- }
- catch (StorageException e) {
+ BlockBlobClient blockBlobClient = getMetaBlobClient(name);
+ updateLastModifiedMetadata(blockBlobClient);
+ blockBlobClient.upload(BinaryData.fromBytes(input.readAllBytes()));
+ } catch (BlobStorageException e) {
LOG.info("Error adding metadata record. metadataName={} length={}", name, recordLength, e);
throw new DataStoreException(e);
- }
- catch (URISyntaxException | IOException e) {
- throw new DataStoreException(e);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
}
}
@@ -616,15 +518,14 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
try {
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
- CloudBlobDirectory metaDir = getAzureContainer().getDirectoryReference(META_DIR_NAME);
- CloudBlockBlob blob = metaDir.getBlockBlobReference(name);
- if (!blob.exists()) {
+ BlockBlobClient blockBlobClient = getMetaBlobClient(name);
+ if (!blockBlobClient.exists()) {
LOG.warn("Trying to read missing metadata. metadataName={}", name);
return null;
}
- blob.downloadAttributes();
- long lastModified = getLastModified(blob);
- long length = blob.getProperties().getLength();
+
+ long lastModified = getLastModified(blockBlobClient);
+ long length = blockBlobClient.getProperties().getBlobSize();
AzureBlobStoreDataRecord record = new AzureBlobStoreDataRecord(this,
azureBlobContainerProvider,
new DataIdentifier(name),
@@ -633,15 +534,14 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
true);
LOG.debug("Metadata record read. metadataName={} duration={} record={}", name, (System.currentTimeMillis() - start), record);
return record;
-
- } catch (StorageException e) {
+ } catch (BlobStorageException e) {
LOG.info("Error reading metadata record. metadataName={}", name, e);
throw new RuntimeException(e);
} catch (Exception e) {
LOG.debug("Error reading metadata record. metadataName={}", name, e);
throw new RuntimeException(e);
} finally {
- if (null != contextClassLoader) {
+ if (contextClassLoader != null) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
}
}
@@ -658,30 +558,27 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
try {
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
- CloudBlobDirectory metaDir = getAzureContainer().getDirectoryReference(META_DIR_NAME);
- for (ListBlobItem item : metaDir.listBlobs(prefix)) {
- if (item instanceof CloudBlob) {
- CloudBlob blob = (CloudBlob) item;
- blob.downloadAttributes();
- records.add(new AzureBlobStoreDataRecord(
- this,
- azureBlobContainerProvider,
- new DataIdentifier(stripMetaKeyPrefix(blob.getName())),
- getLastModified(blob),
- blob.getProperties().getLength(),
- true));
- }
+ ListBlobsOptions listBlobsOptions = new ListBlobsOptions();
+ listBlobsOptions.setPrefix(AzureConstants.AZUre_BlOB_META_DIR_NAME);
+
+ for (BlobItem blobItem : getAzureContainer().listBlobs(listBlobsOptions, null)) {
+ BlobClient blobClient = getAzureContainer().getBlobClient(blobItem.getName());
+ BlobProperties properties = blobClient.getProperties();
+
+ records.add(new AzureBlobStoreDataRecord(this,
+ azureBlobContainerProvider,
+ new DataIdentifier(stripMetaKeyPrefix(blobClient.getBlobName())),
+ getLastModified(blobClient.getBlockBlobClient()),
+ properties.getBlobSize(),
+ true));
}
LOG.debug("Metadata records read. recordsRead={} metadataFolder={} duration={}", records.size(), prefix, (System.currentTimeMillis() - start));
- }
- catch (StorageException e) {
+ } catch (BlobStorageException e) {
LOG.info("Error reading all metadata records. metadataFolder={}", prefix, e);
- }
- catch (DataStoreException | URISyntaxException e) {
+ } catch (DataStoreException e) {
LOG.debug("Error reading all metadata records. metadataFolder={}", prefix, e);
- }
- finally {
- if (null != contextClassLoader) {
+ } finally {
+ if (contextClassLoader != null) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
}
}
@@ -695,21 +592,17 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
try {
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
- CloudBlockBlob blob = getAzureContainer().getBlockBlobReference(addMetaKeyPrefix(name));
+ BlobClient blob = getAzureContainer().getBlobClient(addMetaKeyPrefix(name));
boolean result = blob.deleteIfExists();
LOG.debug("Metadata record {}. metadataName={} duration={}",
result ? "deleted" : "delete requested, but it does not exist (perhaps already deleted)",
name, (System.currentTimeMillis() - start));
return result;
-
- }
- catch (StorageException e) {
+ } catch (BlobStorageException e) {
LOG.info("Error deleting metadata record. metadataName={}", name, e);
- }
- catch (DataStoreException | URISyntaxException e) {
+ } catch (DataStoreException e) {
LOG.debug("Error deleting metadata record. metadataName={}", name, e);
- }
- finally {
+ } finally {
if (contextClassLoader != null) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
}
@@ -727,26 +620,25 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
try {
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
- CloudBlobDirectory metaDir = getAzureContainer().getDirectoryReference(META_DIR_NAME);
int total = 0;
- for (ListBlobItem item : metaDir.listBlobs(prefix)) {
- if (item instanceof CloudBlob) {
- if (((CloudBlob)item).deleteIfExists()) {
- total++;
- }
+
+ ListBlobsOptions listBlobsOptions = new ListBlobsOptions();
+ listBlobsOptions.setPrefix(AzureConstants.AZUre_BlOB_META_DIR_NAME);
+
+ for (BlobItem blobItem : getAzureContainer().listBlobs(listBlobsOptions, null)) {
+ BlobClient blobClient = getAzureContainer().getBlobClient(blobItem.getName());
+ if (blobClient.deleteIfExists()) {
+ total++;
}
}
LOG.debug("Metadata records deleted. recordsDeleted={} metadataFolder={} duration={}",
total, prefix, (System.currentTimeMillis() - start));
- }
- catch (StorageException e) {
+ } catch (BlobStorageException e) {
LOG.info("Error deleting all metadata records. metadataFolder={}", prefix, e);
- }
- catch (DataStoreException | URISyntaxException e) {
+ } catch (DataStoreException e) {
LOG.debug("Error deleting all metadata records. metadataFolder={}", prefix, e);
- }
- finally {
+ } finally {
if (null != contextClassLoader) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
}
@@ -759,15 +651,13 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
try {
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
- CloudBlockBlob blob = getAzureContainer().getBlockBlobReference(addMetaKeyPrefix(name));
+ BlobClient blob = getAzureContainer().getBlobClient(addMetaKeyPrefix(name));
boolean exists = blob.exists();
LOG.debug("Metadata record {} exists {}. duration={}", name, exists, (System.currentTimeMillis() - start));
return exists;
- }
- catch (DataStoreException | StorageException | URISyntaxException e) {
+ } catch (DataStoreException | BlobStorageException e) {
LOG.debug("Error checking existence of metadata record = {}", name, e);
- }
- finally {
+ } finally {
if (contextClassLoader != null) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
}
@@ -775,7 +665,6 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
return false;
}
-
/**
* Get key from data identifier. Object is stored with key in ADS.
*/
@@ -790,39 +679,43 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
private static String getIdentifierName(String key) {
if (!key.contains(Utils.DASH)) {
return null;
- } else if (key.contains(META_KEY_PREFIX)) {
+ } else if (key.contains(AZURE_BLOB_META_KEY_PREFIX)) {
return key;
}
return key.substring(0, 4) + key.substring(5);
}
private static String addMetaKeyPrefix(final String key) {
- return META_KEY_PREFIX + key;
+ return AZURE_BLOB_META_KEY_PREFIX + key;
}
private static String stripMetaKeyPrefix(String name) {
- if (name.startsWith(META_KEY_PREFIX)) {
- return name.substring(META_KEY_PREFIX.length());
+ if (name.startsWith(AZURE_BLOB_META_KEY_PREFIX)) {
+ return name.substring(AZURE_BLOB_META_KEY_PREFIX.length());
}
return name;
}
- private static void addLastModified(CloudBlockBlob blob) {
- blob.getMetadata().put(LAST_MODIFIED_KEY, String.valueOf(System.currentTimeMillis()));
+ private static void updateLastModifiedMetadata(BlockBlobClient blockBlobClient) {
+ BlobContainerClient blobContainerClient = blockBlobClient.getContainerClient();
+ Map<String, String> metadata = blobContainerClient.getProperties().getMetadata();
+ metadata.put(AZURE_BLOB_LAST_MODIFIED_KEY, String.valueOf(System.currentTimeMillis()));
+ blobContainerClient.setMetadata(metadata);
}
- private static long getLastModified(CloudBlob blob) {
- if (blob.getMetadata().containsKey(LAST_MODIFIED_KEY)) {
- return Long.parseLong(blob.getMetadata().get(LAST_MODIFIED_KEY));
+ private static long getLastModified(BlockBlobClient blobClient) {
+ BlobContainerProperties blobProperties = blobClient.getContainerClient().getProperties();
+ if (blobProperties.getMetadata().containsKey(AZURE_BLOB_LAST_MODIFIED_KEY)) {
+ return Long.parseLong(blobProperties.getMetadata().get(AZURE_BLOB_LAST_MODIFIED_KEY));
}
- return blob.getProperties().getLastModified().getTime();
+ return blobProperties.getLastModified().toInstant().toEpochMilli();
}
- void setHttpDownloadURIExpirySeconds(int seconds) {
+ protected void setHttpDownloadURIExpirySeconds(int seconds) {
httpDownloadURIExpirySeconds = seconds;
}
- void setHttpDownloadURICacheSize(int maxSize) {
+ protected void setHttpDownloadURICacheSize(int maxSize) {
// max size 0 or smaller is used to turn off the cache
if (maxSize > 0) {
LOG.info("presigned GET URI cache enabled, maxSize = {} items, expiry = {} seconds", maxSize, httpDownloadURIExpirySeconds / 2);
@@ -836,22 +729,22 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
}
}
- URI createHttpDownloadURI(@NotNull DataIdentifier identifier,
- @NotNull DataRecordDownloadOptions downloadOptions) {
+ protected URI createHttpDownloadURI(@NotNull DataIdentifier identifier,
+ @NotNull DataRecordDownloadOptions downloadOptions) {
URI uri = null;
// When running unit test from Maven, it doesn't always honor the @NotNull decorators
- if (null == identifier) throw new NullPointerException("identifier");
- if (null == downloadOptions) throw new NullPointerException("downloadOptions");
+ if (identifier == null) throw new NullPointerException("identifier");
+ if (downloadOptions == null) throw new NullPointerException("downloadOptions");
if (httpDownloadURIExpirySeconds > 0) {
String domain = getDirectDownloadBlobStorageDomain(downloadOptions.isDomainOverrideIgnored());
- if (null == domain) {
+ if (domain == null) {
throw new NullPointerException("Could not determine domain for direct download");
}
- String cacheKey = identifier.toString()
+ String cacheKey = identifier
+ domain
+ Objects.toString(downloadOptions.getContentTypeHeader(), "")
+ Objects.toString(downloadOptions.getContentDispositionHeader(), "");
@@ -874,24 +767,9 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
}
String key = getKeyName(identifier);
- SharedAccessBlobHeaders headers = new SharedAccessBlobHeaders();
- headers.setCacheControl(String.format("private, max-age=%d, immutable", httpDownloadURIExpirySeconds));
-
- String contentType = downloadOptions.getContentTypeHeader();
- if (! StringUtils.isEmpty(contentType)) {
- headers.setContentType(contentType);
- }
-
- String contentDisposition =
- downloadOptions.getContentDispositionHeader();
- if (! StringUtils.isEmpty(contentDisposition)) {
- headers.setContentDisposition(contentDisposition);
- }
-
uri = createPresignedURI(key,
- EnumSet.of(SharedAccessBlobPermissions.READ),
+ new BlobSasPermission().setReadPermission(true),
httpDownloadURIExpirySeconds,
- headers,
domain);
if (uri != null && httpDownloadURICache != null) {
httpDownloadURICache.put(cacheKey, uri);
@@ -901,44 +779,42 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
return uri;
}
- void setHttpUploadURIExpirySeconds(int seconds) { httpUploadURIExpirySeconds = seconds; }
+ protected void setHttpUploadURIExpirySeconds(int seconds) {
+ httpUploadURIExpirySeconds = seconds;
+ }
private DataIdentifier generateSafeRandomIdentifier() {
return new DataIdentifier(
String.format("%s-%d",
- UUID.randomUUID().toString(),
+ UUID.randomUUID(),
Instant.now().toEpochMilli()
)
);
}
- DataRecordUpload initiateHttpUpload(long maxUploadSizeInBytes, int maxNumberOfURIs, @NotNull final DataRecordUploadOptions options) {
- List<URI> uploadPartURIs = new ArrayList<>();
- long minPartSize = MIN_MULTIPART_UPLOAD_PART_SIZE;
- long maxPartSize = MAX_MULTIPART_UPLOAD_PART_SIZE;
+ protected DataRecordUpload initiateHttpUpload(long maxUploadSizeInBytes, int maxNumberOfURIs, @NotNull final DataRecordUploadOptions options) {
+ List<URI> uploadPartURIs = Lists.newArrayList();
+ long minPartSize = AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE;
+ long maxPartSize = AZURE_BLOB_MAX_MULTIPART_UPLOAD_PART_SIZE;
if (0L >= maxUploadSizeInBytes) {
throw new IllegalArgumentException("maxUploadSizeInBytes must be > 0");
- }
- else if (0 == maxNumberOfURIs) {
+ } else if (0 == maxNumberOfURIs) {
throw new IllegalArgumentException("maxNumberOfURIs must either be > 0 or -1");
- }
- else if (-1 > maxNumberOfURIs) {
+ } else if (-1 > maxNumberOfURIs) {
throw new IllegalArgumentException("maxNumberOfURIs must either be > 0 or -1");
- }
- else if (maxUploadSizeInBytes > MAX_SINGLE_PUT_UPLOAD_SIZE &&
+ } else if (maxUploadSizeInBytes > AZURE_BLOB_MAX_SINGLE_PUT_UPLOAD_SIZE &&
maxNumberOfURIs == 1) {
throw new IllegalArgumentException(
String.format("Cannot do single-put upload with file size %d - exceeds max single-put upload size of %d",
maxUploadSizeInBytes,
- MAX_SINGLE_PUT_UPLOAD_SIZE)
+ AZURE_BLOB_MAX_SINGLE_PUT_UPLOAD_SIZE)
);
- }
- else if (maxUploadSizeInBytes > MAX_BINARY_UPLOAD_SIZE) {
+ } else if (maxUploadSizeInBytes > AZURE_BLOB_MAX_BINARY_UPLOAD_SIZE) {
throw new IllegalArgumentException(
String.format("Cannot do upload with file size %d - exceeds max upload size of %d",
maxUploadSizeInBytes,
- MAX_BINARY_UPLOAD_SIZE)
+ AZURE_BLOB_MAX_BINARY_UPLOAD_SIZE)
);
}
@@ -973,7 +849,7 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
maxNumberOfURIs,
Math.min(
(long) Math.ceil(((double) maxUploadSizeInBytes) / ((double) minPartSize)),
- MAX_ALLOWABLE_UPLOAD_URIS
+ AZURE_BLOB_MAX_ALLOWABLE_UPLOAD_URIS
)
);
} else {
@@ -981,10 +857,9 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
String.format("Cannot do multi-part upload with requested part size %d", requestedPartSize)
);
}
- }
- else {
- long maximalNumParts = (long) Math.ceil(((double) maxUploadSizeInBytes) / ((double) MIN_MULTIPART_UPLOAD_PART_SIZE));
- numParts = Math.min(maximalNumParts, MAX_ALLOWABLE_UPLOAD_URIS);
+ } else {
+ long maximalNumParts = (long) Math.ceil(((double) maxUploadSizeInBytes) / ((double) AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE));
+ numParts = Math.min(maximalNumParts, AZURE_BLOB_MAX_ALLOWABLE_UPLOAD_URIS);
}
String key = getKeyName(newIdentifier);
@@ -993,8 +868,9 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
throw new NullPointerException("Could not determine domain for direct upload");
}
- EnumSet<SharedAccessBlobPermissions> perms = EnumSet.of(SharedAccessBlobPermissions.WRITE);
- Map<String, String> presignedURIRequestParams = new HashMap<>();
+ BlobSasPermission perms = new BlobSasPermission()
+ .setWritePermission(true);
+ Map<String, String> presignedURIRequestParams = Maps.newHashMap();
// see https://docs.microsoft.com/en-us/rest/api/storageservices/put-block#uri-parameters
presignedURIRequestParams.put("comp", "block");
for (long blockId = 1; blockId <= numParts; ++blockId) {
@@ -1043,7 +919,18 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
return null;
}
- DataRecord completeHttpUpload(@NotNull String uploadTokenStr)
+ private Long getUncommittedBlocksListSize(BlockBlobClient client) throws DataStoreException {
+ List<Block> blocks = client.listBlocks(BlockListType.UNCOMMITTED).getUncommittedBlocks();
+ updateLastModifiedMetadata(client);
+ client.commitBlockList(blocks.stream().map(Block::getName).collect(Collectors.toList()));
+ long size = 0L;
+ for (Block block : blocks) {
+ size += block.getSize();
+ }
+ return size;
+ }
+
+ protected DataRecord completeHttpUpload(@NotNull String uploadTokenStr)
throws DataRecordUploadException, DataStoreException {
if (StringUtils.isEmpty(uploadTokenStr)) {
@@ -1060,32 +947,19 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
// If this succeeds this means either it was a "single put" upload
// (we don't need to do anything in this case - blob is already uploaded)
// or it was completed before with the same token.
- }
- catch (DataStoreException e1) {
+ } catch (DataStoreException e1) {
// record doesn't exist - so this means we are safe to do the complete request
try {
if (uploadToken.getUploadId().isPresent()) {
- CloudBlockBlob blob = getAzureContainer().getBlockBlobReference(key);
- // An existing upload ID means this is a multi-part upload
- List<BlockEntry> blocks = blob.downloadBlockList(
- BlockListingFilter.UNCOMMITTED,
- AccessCondition.generateEmptyCondition(),
- null,
- null);
- addLastModified(blob);
- blob.commitBlockList(blocks);
- long size = 0L;
- for (BlockEntry block : blocks) {
- size += block.getSize();
- }
+ BlockBlobClient blockBlobClient = getAzureContainer().getBlobClient(key).getBlockBlobClient();
+ long size = getUncommittedBlocksListSize(blockBlobClient);
record = new AzureBlobStoreDataRecord(
this,
azureBlobContainerProvider,
blobId,
- getLastModified(blob),
+ getLastModified(blockBlobClient),
size);
- }
- else {
+ } else {
// Something is wrong - upload ID missing from upload token
// but record doesn't exist already, so this is invalid
throw new DataRecordUploadException(
@@ -1093,7 +967,7 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
blobId)
);
}
- } catch (URISyntaxException | StorageException e2) {
+ } catch (BlobStorageException e2) {
throw new DataRecordUploadException(
String.format("Unable to finalize direct write of binary %s", blobId),
e2
@@ -1134,36 +1008,26 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
}
private URI createPresignedURI(String key,
- EnumSet<SharedAccessBlobPermissions> permissions,
+ BlobSasPermission blobSasPermissions,
int expirySeconds,
- SharedAccessBlobHeaders optionalHeaders,
String domain) {
- return createPresignedURI(key, permissions, expirySeconds, new HashMap<>(), optionalHeaders, domain);
+ return createPresignedURI(key, blobSasPermissions, expirySeconds, Maps.newHashMap(), domain);
}
private URI createPresignedURI(String key,
- EnumSet<SharedAccessBlobPermissions> permissions,
+ BlobSasPermission blobSasPermissions,
int expirySeconds,
Map<String, String> additionalQueryParams,
String domain) {
- return createPresignedURI(key, permissions, expirySeconds, additionalQueryParams, null, domain);
- }
-
- private URI createPresignedURI(String key,
- EnumSet<SharedAccessBlobPermissions> permissions,
- int expirySeconds,
- Map<String, String> additionalQueryParams,
- SharedAccessBlobHeaders optionalHeaders,
- String domain) {
- if (StringUtils.isEmpty(domain)) {
+ if (Strings.isNullOrEmpty(domain)) {
LOG.warn("Can't generate presigned URI - no Azure domain provided (is Azure account name configured?)");
return null;
}
URI presignedURI = null;
try {
- String sharedAccessSignature = azureBlobContainerProvider.generateSharedAccessSignature(getBlobRequestOptions(), key,
- permissions, expirySeconds, optionalHeaders);
+ String sharedAccessSignature = azureBlobContainerProvider.generateSharedAccessSignature(retryOptions, key,
+ blobSasPermissions, expirySeconds, properties);
// Shared access signature is returned encoded already.
String uriString = String.format("https://%s/%s/%s?%s",
@@ -1172,7 +1036,7 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
key,
sharedAccessSignature);
- if (! additionalQueryParams.isEmpty()) {
+ if (!additionalQueryParams.isEmpty()) {
StringBuilder builder = new StringBuilder();
for (Map.Entry<String, String> e : additionalQueryParams.entrySet()) {
builder.append("&");
@@ -1184,21 +1048,18 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
}
presignedURI = new URI(uriString);
- }
- catch (DataStoreException e) {
+ } catch (DataStoreException e) {
LOG.error("No connection to Azure Blob Storage", e);
- }
- catch (URISyntaxException | InvalidKeyException e) {
+ } catch (URISyntaxException | InvalidKeyException e) {
LOG.error("Can't generate a presigned URI for key {}", key, e);
- }
- catch (StorageException e) {
+ } catch (BlobStorageException e) {
LOG.error("Azure request to create presigned Azure Blob Storage {} URI failed. " +
"Key: {}, Error: {}, HTTP Code: {}, Azure Error Code: {}",
- permissions.contains(SharedAccessBlobPermissions.READ) ? "GET" :
- (permissions.contains(SharedAccessBlobPermissions.WRITE) ? "PUT" : ""),
+ blobSasPermissions.hasReadPermission() ? "GET" :
+ ((blobSasPermissions.hasWritePermission()) ? "PUT" : ""),
key,
e.getMessage(),
- e.getHttpStatusCode(),
+ e.getStatusCode(),
e.getErrorCode());
}
@@ -1228,70 +1089,10 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
return length;
}
- public static AzureBlobInfo fromCloudBlob(CloudBlob cloudBlob) throws StorageException {
- cloudBlob.downloadAttributes();
- return new AzureBlobInfo(cloudBlob.getName(),
+ public static AzureBlobInfo fromCloudBlob(BlockBlobClient cloudBlob) throws BlobStorageException {
+ return new AzureBlobInfo(cloudBlob.getBlobName(),
AzureBlobStoreBackend.getLastModified(cloudBlob),
- cloudBlob.getProperties().getLength());
- }
- }
-
- private class RecordsIterator<T> extends AbstractIterator<T> {
- // Seems to be thread-safe (in 5.0.0)
- ResultContinuation resultContinuation;
- boolean firstCall = true;
- final Function<AzureBlobInfo, T> transformer;
- final Queue<AzureBlobInfo> items = new LinkedList<>();
-
- public RecordsIterator (Function<AzureBlobInfo, T> transformer) {
- this.transformer = transformer;
- }
-
- @Override
- protected T computeNext() {
- if (items.isEmpty()) {
- loadItems();
- }
- if (!items.isEmpty()) {
- return transformer.apply(items.remove());
- }
- return endOfData();
- }
-
- private boolean loadItems() {
- long start = System.currentTimeMillis();
- ClassLoader contextClassLoader = currentThread().getContextClassLoader();
- try {
- currentThread().setContextClassLoader(getClass().getClassLoader());
-
- CloudBlobContainer container = azureBlobContainerProvider.getBlobContainer();
- if (!firstCall && (resultContinuation == null || !resultContinuation.hasContinuation())) {
- LOG.trace("No more records in container. containerName={}", container);
- return false;
- }
- firstCall = false;
- ResultSegment<ListBlobItem> results = container.listBlobsSegmented(null, false, EnumSet.noneOf(BlobListingDetails.class), null, resultContinuation, null, null);
- resultContinuation = results.getContinuationToken();
- for (ListBlobItem item : results.getResults()) {
- if (item instanceof CloudBlob) {
- items.add(AzureBlobInfo.fromCloudBlob((CloudBlob)item));
- }
- }
- LOG.debug("Container records batch read. batchSize={} containerName={} duration={}",
- results.getLength(), getContainerName(), (System.currentTimeMillis() - start));
- return results.getLength() > 0;
- }
- catch (StorageException e) {
- LOG.info("Error listing blobs. containerName={}", getContainerName(), e);
- }
- catch (DataStoreException e) {
- LOG.debug("Cannot list blobs. containerName={}", getContainerName(), e);
- } finally {
- if (contextClassLoader != null) {
- currentThread().setContextClassLoader(contextClassLoader);
- }
- }
- return false;
+ cloudBlob.getProperties().getBlobSize());
}
}
@@ -1323,20 +1124,20 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
@Override
public InputStream getStream() throws DataStoreException {
String id = getKeyName(getIdentifier());
- CloudBlobContainer container = azureBlobContainerProvider.getBlobContainer();
+ BlobContainerClient container = azureBlobContainerProvider.getBlobContainer();
if (isMeta) {
id = addMetaKeyPrefix(getIdentifier().toString());
}
else {
// Don't worry about stream logging for metadata records
if (LOG_STREAMS_DOWNLOAD.isDebugEnabled()) {
- // Log message, with exception so we can get a trace to see where the call came from
+ // Log message, with exception, so we can get a trace to see where the call came from
LOG_STREAMS_DOWNLOAD.debug("Binary downloaded from Azure Blob Storage - identifier={} ", id, new Exception());
}
}
try {
- return container.getBlockBlobReference(id).openInputStream();
- } catch (StorageException | URISyntaxException e) {
+ return container.getBlobClient(id).openInputStream();
+ } catch (Exception e) {
throw new DataStoreException(e);
}
}
@@ -1362,4 +1163,54 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
.map(AzureBlobContainerProvider::getContainerName)
.orElse(null);
}
+
+ @Override
+ public byte[] getOrCreateReferenceKey() throws DataStoreException {
+ try {
+ if (secret != null && secret.length != 0) {
+ return secret;
+ } else {
+ byte[] key;
+ // Try reading from the metadata folder if it exists
+ key = readMetadataBytes(AZURE_BLOB_REF_KEY);
+ if (key == null) {
+ key = super.getOrCreateReferenceKey();
+ addMetadataRecord(new ByteArrayInputStream(key), AZURE_BLOB_REF_KEY);
+ key = readMetadataBytes(AZURE_BLOB_REF_KEY);
+ }
+ secret = key;
+ return secret;
+ }
+ } catch (IOException e) {
+ throw new DataStoreException("Unable to get or create key " + e);
+ }
+ }
+
+ protected byte[] readMetadataBytes(String name) throws IOException, DataStoreException {
+ DataRecord rec = getMetadataRecord(name);
+ byte[] key = null;
+ if (rec != null) {
+ InputStream stream = null;
+ try {
+ stream = rec.getStream();
+ return IOUtils.toByteArray(stream);
+ } finally {
+ IOUtils.closeQuietly(stream);
+ }
+ }
+ return key;
+ }
+
+ private String computeSecondaryLocationEndpoint() {
+ String accountName = properties.getProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, "");
+
+ boolean enableSecondaryLocation = PropertiesUtil.toBoolean(properties.getProperty(AzureConstants.AZURE_BLOB_ENABLE_SECONDARY_LOCATION_NAME),
+ AzureConstants.AZURE_BLOB_ENABLE_SECONDARY_LOCATION_DEFAULT);
+
+ if(enableSecondaryLocation) {
+ return String.format("https://%s-secondary.blob.core.windows.net", accountName);
+ }
+
+ return null;
+ }
}
diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureConstants.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureConstants.java
index a7fa4249d2..87acdffce2 100644
--- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureConstants.java
+++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureConstants.java
@@ -148,6 +148,86 @@ public final class AzureConstants {
* Property to enable/disable creation of reference secret on init.
*/
public static final String AZURE_REF_ON_INIT = "refOnInit";
-
+
+ /**
+ * Directory name for storing metadata files in the blob storage
+ */
+ public static final String AZUre_BlOB_META_DIR_NAME = "META";
+
+ /**
+ * Key prefix for metadata entries, includes trailing slash for directory structure
+ */
+ public static final String AZURE_BLOB_META_KEY_PREFIX = AZUre_BlOB_META_DIR_NAME + "/";
+
+ /**
+ * Key name for storing blob reference information
+ */
+ public static final String AZURE_BLOB_REF_KEY = "reference.key";
+
+ /**
+ * Key name for storing last modified timestamp metadata
+ */
+ public static final String AZURE_BLOB_LAST_MODIFIED_KEY = "lastModified";
+
+ /**
+ * Threshold size (8 MiB) above which streams are buffered to disk during upload operations
+ */
+ public static final long AZURE_BLOB_BUFFERED_STREAM_THRESHOLD = 8L * 1024L * 1024L;
+
+ /**
+ * Minimum part size (256 KiB) required for Azure Blob Storage multipart uploads
+ */
+ public static final long AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE = 256L * 1024L;
+
+ /**
+ * Maximum part size (4000 MiB / 4 GiB) allowed by Azure Blob Storage for multipart uploads
+ */
+ public static final long AZURE_BLOB_MAX_MULTIPART_UPLOAD_PART_SIZE = 4000L * 1024L * 1024L;
+
+ /**
+ * Maximum size (256 MiB) for single PUT operations in Azure Blob Storage
+ */
+ public static final long AZURE_BLOB_MAX_SINGLE_PUT_UPLOAD_SIZE = 256L * 1024L * 1024L;
+
+ /**
+ * Maximum total binary size (~190.7 TiB) that can be uploaded to Azure Blob Storage
+ */
+ public static final long AZURE_BLOB_MAX_BINARY_UPLOAD_SIZE = 190L * 1024L * 1024L * 1024L * 1024L;
+
+ /**
+ * Maximum number of blocks (50,000) allowed per blob in Azure Blob Storage
+ */
+ public static final int AZURE_BLOB_MAX_ALLOWABLE_UPLOAD_URIS = 50000;
+
+ /**
+ * Maximum number of attempts to generate a unique record identifier
+ */
+ public static final int AZURE_BLOB_MAX_UNIQUE_RECORD_TRIES = 10;
+
+ /**
+ * Default number of concurrent requests (5) for optimal performance with Azure SDK 12
+ */
+ public static final int AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT = 5;
+
+ /**
+ * Maximum recommended concurrent requests (10) for optimal performance with Azure SDK 12
+ */
+ public static final int AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT = 10;
+
+ /**
+ * Maximum block size (100 MiB) supported by Azure Blob Storage SDK 12
+ */
+ public static final long AZURE_BLOB_MAX_BLOCK_SIZE = 100L * 1024L * 1024L;
+
+ /**
+ * Default number of retry attempts (4) for failed requests in Azure SDK 12
+ */
+ public static final int AZURE_BLOB_MAX_RETRY_REQUESTS = 4;
+
+ /**
+ * Default timeout duration (60 seconds) for Azure Blob Storage operations
+ */
+ public static final int AZURE_BLOB_DEFAULT_TIMEOUT_SECONDS = 60;
+
private AzureConstants() { }
}
diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStore.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStore.java
index 11575ad966..6b1c7421fc 100644
--- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStore.java
+++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStore.java
@@ -25,6 +25,7 @@ import java.util.Properties;
import org.apache.jackrabbit.core.data.DataIdentifier;
import org.apache.jackrabbit.core.data.DataRecord;
import org.apache.jackrabbit.core.data.DataStoreException;
+import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v8.AzureBlobStoreBackendV8;
import org.apache.jackrabbit.oak.plugins.blob.AbstractSharedCachingDataStore;
import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.ConfigurableDataRecordAccessProvider;
import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadException;
@@ -41,11 +42,18 @@ public class AzureDataStore extends AbstractSharedCachingDataStore implements Co
protected Properties properties;
- private AzureBlobStoreBackend azureBlobStoreBackend;
+ private AbstractAzureBlobStoreBackend azureBlobStoreBackend;
+
+ private final boolean useAzureSdkV12 = Boolean.getBoolean("blob.azure.v12.enabled");
@Override
protected AbstractSharedBackend createBackend() {
- azureBlobStoreBackend = new AzureBlobStoreBackend();
+ if (useAzureSdkV12) {
+ azureBlobStoreBackend = new AzureBlobStoreBackend();
+ } else {
+ azureBlobStoreBackend = new AzureBlobStoreBackendV8();
+ }
+
if (null != properties) {
azureBlobStoreBackend.setProperties(properties);
}
diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreService.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreService.java
index 3ad2e5e46e..36469401b9 100644
--- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreService.java
+++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreService.java
@@ -20,6 +20,7 @@
package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;
import org.apache.jackrabbit.oak.stats.StatisticsProvider;
+import org.jetbrains.annotations.NotNull;
import org.osgi.service.component.annotations.Component;
import org.osgi.service.component.annotations.ConfigurationPolicy;
import org.osgi.service.component.annotations.Reference;
@@ -32,7 +33,7 @@ public class AzureDataStoreService extends AbstractAzureDataStoreService {
public static final String NAME = "org.apache.jackrabbit.oak.plugins.blob.datastore.AzureDataStore";
- protected StatisticsProvider getStatisticsProvider(){
+ protected @NotNull StatisticsProvider getStatisticsProvider(){
return statisticsProvider;
}
diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/Utils.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/Utils.java
index 185816f7c0..430b13745d 100644
--- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/Utils.java
+++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/Utils.java
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;
import java.io.File;
@@ -24,21 +23,18 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.InetSocketAddress;
-import java.net.Proxy;
-import java.net.SocketAddress;
-import java.net.URISyntaxException;
-import java.security.InvalidKeyException;
import java.util.Properties;
-import com.microsoft.azure.storage.CloudStorageAccount;
-import com.microsoft.azure.storage.OperationContext;
-import com.microsoft.azure.storage.RetryExponentialRetry;
-import com.microsoft.azure.storage.RetryNoRetry;
-import com.microsoft.azure.storage.RetryPolicy;
-import com.microsoft.azure.storage.StorageException;
-import com.microsoft.azure.storage.blob.BlobRequestOptions;
-import com.microsoft.azure.storage.blob.CloudBlobClient;
-import com.microsoft.azure.storage.blob.CloudBlobContainer;
+import com.azure.core.http.HttpClient;
+import com.azure.core.http.ProxyOptions;
+import com.azure.core.http.netty.NettyAsyncHttpClientBuilder;
+import com.azure.storage.blob.BlobContainerClient;
+import com.azure.storage.blob.BlobContainerClientBuilder;
+import com.azure.storage.blob.BlobServiceClient;
+import com.azure.storage.blob.BlobServiceClientBuilder;
+import com.azure.storage.common.policy.RequestRetryOptions;
+import com.azure.storage.common.policy.RetryPolicyType;
+import com.google.common.base.Strings;
import org.apache.commons.lang3.StringUtils;
import org.apache.jackrabbit.core.data.DataStoreException;
import org.apache.jackrabbit.oak.commons.PropertiesUtil;
@@ -46,84 +42,62 @@ import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
public final class Utils {
-
+ public static final String DASH = "-";
public static final String DEFAULT_CONFIG_FILE = "azure.properties";
- public static final String DASH = "-";
+ private Utils() {}
- /**
- * private constructor so that class cannot initialized from outside.
- */
- private Utils() {
- }
+ public static BlobContainerClient getBlobContainer(@NotNull final String connectionString,
+ @NotNull final String containerName,
+ @Nullable final RequestRetryOptions retryOptions,
+ final Properties properties) throws DataStoreException {
+ try {
+ BlobServiceClientBuilder builder = new BlobServiceClientBuilder()
+ .connectionString(connectionString)
+ .retryOptions(retryOptions);
- /**
- * Create CloudBlobClient from properties.
- *
- * @param connectionString connectionString to configure @link {@link CloudBlobClient}
- * @return {@link CloudBlobClient}
- */
- public static CloudBlobClient getBlobClient(@NotNull final String connectionString) throws URISyntaxException, InvalidKeyException {
- return getBlobClient(connectionString, null);
- }
+ HttpClient httpClient = new NettyAsyncHttpClientBuilder()
+ .proxy(computeProxyOptions(properties))
+ .build();
- public static CloudBlobClient getBlobClient(@NotNull final String connectionString,
- @Nullable final BlobRequestOptions requestOptions) throws URISyntaxException, InvalidKeyException {
- CloudStorageAccount account = CloudStorageAccount.parse(connectionString);
- CloudBlobClient client = account.createCloudBlobClient();
- if (null != requestOptions) {
- client.setDefaultRequestOptions(requestOptions);
- }
- return client;
- }
+ builder.httpClient(httpClient);
- public static CloudBlobContainer getBlobContainer(@NotNull final String connectionString,
- @NotNull final String containerName) throws DataStoreException {
- return getBlobContainer(connectionString, containerName, null);
- }
+ BlobServiceClient blobServiceClient = builder.buildClient();
+ return blobServiceClient.getBlobContainerClient(containerName);
- public static CloudBlobContainer getBlobContainer(@NotNull final String connectionString,
- @NotNull final String containerName,
- @Nullable final BlobRequestOptions requestOptions) throws DataStoreException {
- try {
- CloudBlobClient client = (
- (null == requestOptions)
- ? Utils.getBlobClient(connectionString)
- : Utils.getBlobClient(connectionString, requestOptions)
- );
- return client.getContainerReference(containerName);
- } catch (InvalidKeyException | URISyntaxException | StorageException e) {
+ } catch (Exception e) {
throw new DataStoreException(e);
}
}
- public static void setProxyIfNeeded(final Properties properties) {
+ public static ProxyOptions computeProxyOptions(final Properties properties) {
String proxyHost = properties.getProperty(AzureConstants.PROXY_HOST);
String proxyPort = properties.getProperty(AzureConstants.PROXY_PORT);
- if (!StringUtils.isEmpty(proxyHost) &&
- StringUtils.isEmpty(proxyPort)) {
- int port = Integer.parseInt(proxyPort);
- SocketAddress proxyAddr = new InetSocketAddress(proxyHost, port);
- Proxy proxy = new Proxy(Proxy.Type.HTTP, proxyAddr);
- OperationContext.setDefaultProxy(proxy);
+ if(!Strings.isNullOrEmpty(proxyHost) && Strings.isNullOrEmpty(proxyPort)) {
+ return new ProxyOptions(ProxyOptions.Type.HTTP,
+ new InetSocketAddress(proxyHost, Integer.parseInt(proxyPort)));
}
+ return null;
}
- public static RetryPolicy getRetryPolicy(final String maxRequestRetry) {
- int retries = PropertiesUtil.toInteger(maxRequestRetry, -1);
- if (retries < 0) {
+ public static RequestRetryOptions getRetryOptions(final String maxRequestRetryCount, Integer requestTimeout, String secondaryLocation) {
+ int retries = PropertiesUtil.toInteger(maxRequestRetryCount, -1);
+ if(retries < 0) {
return null;
}
+
if (retries == 0) {
- return new RetryNoRetry();
+ return new RequestRetryOptions(RetryPolicyType.FIXED, 1,
+ requestTimeout, null, null,
+ secondaryLocation);
}
- return new RetryExponentialRetry(RetryPolicy.DEFAULT_CLIENT_BACKOFF, retries);
+ return new RequestRetryOptions(RetryPolicyType.EXPONENTIAL, retries,
+ requestTimeout, null, null,
+ secondaryLocation);
}
-
public static String getConnectionStringFromProperties(Properties properties) {
-
String sasUri = properties.getProperty(AzureConstants.AZURE_SAS, "");
String blobEndpoint = properties.getProperty(AzureConstants.AZURE_BLOB_ENDPOINT, "");
String connectionString = properties.getProperty(AzureConstants.AZURE_CONNECTION_STRING, "");
@@ -140,7 +114,7 @@ public final class Utils {
return getConnectionString(
accountName,
- accountKey,
+ accountKey,
blobEndpoint);
}
@@ -152,21 +126,23 @@ public final class Utils {
}
}
- public static String getConnectionString(final String accountName, final String accountKey) {
- return getConnectionString(accountName, accountKey, null);
- }
-
public static String getConnectionString(final String accountName, final String accountKey, String blobEndpoint) {
StringBuilder connString = new StringBuilder("DefaultEndpointsProtocol=https");
connString.append(";AccountName=").append(accountName);
connString.append(";AccountKey=").append(accountKey);
-
- if (!StringUtils.isEmpty(blobEndpoint)) {
+ if (!Strings.isNullOrEmpty(blobEndpoint)) {
connString.append(";BlobEndpoint=").append(blobEndpoint);
}
return connString.toString();
}
+ public static BlobContainerClient getBlobContainerFromConnectionString(final String azureConnectionString, final String containerName) {
+ return new BlobContainerClientBuilder()
+ .connectionString(azureConnectionString)
+ .containerName(containerName)
+ .buildClient();
+ }
+
/**
* Read a configuration properties file. If the file name ends with ";burn",
* the file is deleted after reading.
diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobContainerProvider.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobContainerProviderV8.java
similarity index 95%
copy from oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobContainerProvider.java
copy to oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobContainerProviderV8.java
index 01522248b0..9eddee3aa0 100644
--- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobContainerProvider.java
+++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobContainerProviderV8.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;
+package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v8;
import com.azure.core.credential.AccessToken;
import com.azure.core.credential.TokenRequestContext;
@@ -35,6 +35,8 @@ import com.microsoft.azure.storage.blob.SharedAccessBlobPermissions;
import com.microsoft.azure.storage.blob.SharedAccessBlobPolicy;
import org.apache.commons.lang3.StringUtils;
import org.apache.jackrabbit.core.data.DataStoreException;
+import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants;
+import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.Utils;
import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -57,8 +59,8 @@ import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
-public class AzureBlobContainerProvider implements Closeable {
- private static final Logger log = LoggerFactory.getLogger(AzureBlobContainerProvider.class);
+public class AzureBlobContainerProviderV8 implements Closeable {
+ private static final Logger log = LoggerFactory.getLogger(AzureBlobContainerProviderV8.class);
private static final String DEFAULT_ENDPOINT_SUFFIX = "core.windows.net";
private static final String AZURE_DEFAULT_SCOPE = "https://storage.azure.com/.default";
private final String azureConnectionString;
@@ -77,7 +79,7 @@ public class AzureBlobContainerProvider implements Closeable {
private static final long TOKEN_REFRESHER_DELAY = 1L;
private final ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
- private AzureBlobContainerProvider(Builder builder) {
+ private AzureBlobContainerProviderV8(Builder builder) {
this.azureConnectionString = builder.azureConnectionString;
this.accountName = builder.accountName;
this.containerName = builder.containerName;
@@ -89,7 +91,6 @@ public class AzureBlobContainerProvider implements Closeable {
this.clientSecret = builder.clientSecret;
}
-
public static class Builder {
private final String containerName;
@@ -162,8 +163,8 @@ public class AzureBlobContainerProvider implements Closeable {
return this;
}
- public AzureBlobContainerProvider build() {
- return new AzureBlobContainerProvider(this);
+ public AzureBlobContainerProviderV8 build() {
+ return new AzureBlobContainerProviderV8(this);
}
}
@@ -181,18 +182,18 @@ public class AzureBlobContainerProvider implements Closeable {
// connection string will be given preference over service principals / sas / account key
if (StringUtils.isNotBlank(azureConnectionString)) {
log.debug("connecting to azure blob storage via azureConnectionString");
- return Utils.getBlobContainer(azureConnectionString, containerName, blobRequestOptions);
+ return UtilsV8.getBlobContainer(azureConnectionString, containerName, blobRequestOptions);
} else if (authenticateViaServicePrincipal()) {
log.debug("connecting to azure blob storage via service principal credentials");
return getBlobContainerFromServicePrincipals(blobRequestOptions);
} else if (StringUtils.isNotBlank(sasToken)) {
log.debug("connecting to azure blob storage via sas token");
final String connectionStringWithSasToken = Utils.getConnectionStringForSas(sasToken, blobEndpoint, accountName);
- return Utils.getBlobContainer(connectionStringWithSasToken, containerName, blobRequestOptions);
+ return UtilsV8.getBlobContainer(connectionStringWithSasToken, containerName, blobRequestOptions);
}
log.debug("connecting to azure blob storage via access key");
final String connectionStringWithAccountKey = Utils.getConnectionString(accountName, accountKey, blobEndpoint);
- return Utils.getBlobContainer(connectionStringWithAccountKey, containerName, blobRequestOptions);
+ return UtilsV8.getBlobContainer(connectionStringWithAccountKey, containerName, blobRequestOptions);
}
@NotNull
diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobStoreBackendV8.java
similarity index 87%
copy from oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
copy to oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobStoreBackendV8.java
index c60ad2e868..cc2368f2e2 100644
--- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
+++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobStoreBackendV8.java
@@ -16,9 +16,21 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;
+package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v8;
import static java.lang.Thread.currentThread;
+import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_BUFFERED_STREAM_THRESHOLD;
+import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT;
+import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_LAST_MODIFIED_KEY;
+import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MAX_ALLOWABLE_UPLOAD_URIS;
+import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MAX_BINARY_UPLOAD_SIZE;
+import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT;
+import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MAX_MULTIPART_UPLOAD_PART_SIZE;
+import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MAX_SINGLE_PUT_UPLOAD_SIZE;
+import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZUre_BlOB_META_DIR_NAME;
+import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_META_KEY_PREFIX;
+import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE;
+import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_REF_KEY;
import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
@@ -33,12 +45,9 @@ import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.security.InvalidKeyException;
import java.time.Instant;
-import java.util.ArrayList;
import java.util.Collection;
import java.util.EnumSet;
-import java.util.HashMap;
import java.util.Iterator;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -49,10 +58,12 @@ import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.jackrabbit.guava.common.cache.Cache;
-import org.apache.jackrabbit.guava.common.cache.CacheBuilder;
-import org.apache.jackrabbit.guava.common.collect.AbstractIterator;
+import com.google.common.base.Strings;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.collect.AbstractIterator;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import com.microsoft.azure.storage.AccessCondition;
import com.microsoft.azure.storage.LocationMode;
import com.microsoft.azure.storage.ResultContinuation;
@@ -75,6 +86,9 @@ import org.apache.commons.io.IOUtils;
import org.apache.jackrabbit.core.data.DataIdentifier;
import org.apache.jackrabbit.core.data.DataRecord;
import org.apache.jackrabbit.core.data.DataStoreException;
+import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AbstractAzureBlobStoreBackend;
+import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants;
+import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.Utils;
import org.apache.jackrabbit.oak.commons.PropertiesUtil;
import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordDownloadOptions;
import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUpload;
@@ -88,31 +102,15 @@ import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class AzureBlobStoreBackend extends AbstractSharedBackend {
+public class AzureBlobStoreBackendV8 extends AbstractAzureBlobStoreBackend {
- private static final Logger LOG = LoggerFactory.getLogger(AzureBlobStoreBackend.class);
+ private static final Logger LOG = LoggerFactory.getLogger(AzureBlobStoreBackendV8.class);
private static final Logger LOG_STREAMS_DOWNLOAD = LoggerFactory.getLogger("oak.datastore.download.streams");
private static final Logger LOG_STREAMS_UPLOAD = LoggerFactory.getLogger("oak.datastore.upload.streams");
- private static final String META_DIR_NAME = "META";
- private static final String META_KEY_PREFIX = META_DIR_NAME + "/";
-
- private static final String REF_KEY = "reference.key";
- private static final String LAST_MODIFIED_KEY = "lastModified";
-
- private static final long BUFFERED_STREAM_THRESHOLD = 1024 * 1024;
- static final long MIN_MULTIPART_UPLOAD_PART_SIZE = 1024 * 1024 * 10; // 10MB
- static final long MAX_MULTIPART_UPLOAD_PART_SIZE = 1024 * 1024 * 100; // 100MB
- static final long MAX_SINGLE_PUT_UPLOAD_SIZE = 1024 * 1024 * 256; // 256MB, Azure limit
- static final long MAX_BINARY_UPLOAD_SIZE = (long) Math.floor(1024L * 1024L * 1024L * 1024L * 4.75); // 4.75TB, Azure limit
- private static final int MAX_ALLOWABLE_UPLOAD_URIS = 50000; // Azure limit
- private static final int MAX_UNIQUE_RECORD_TRIES = 10;
- private static final int DEFAULT_CONCURRENT_REQUEST_COUNT = 2;
- private static final int MAX_CONCURRENT_REQUEST_COUNT = 50;
-
private Properties properties;
- private AzureBlobContainerProvider azureBlobContainerProvider;
- private int concurrentRequestCount = DEFAULT_CONCURRENT_REQUEST_COUNT;
+ private AzureBlobContainerProviderV8 azureBlobContainerProvider;
+ private int concurrentRequestCount = AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT;
private RetryPolicy retryPolicy;
private Integer requestTimeout;
private int httpDownloadURIExpirySeconds = 0; // disabled by default
@@ -133,7 +131,7 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
private volatile CloudBlobContainer azureContainer = null;
- protected CloudBlobContainer getAzureContainer() throws DataStoreException {
+ public CloudBlobContainer getAzureContainer() throws DataStoreException {
if (azureContainer == null) {
synchronized (this) {
if (azureContainer == null) {
@@ -170,41 +168,41 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
if (null == properties) {
try {
- properties = Utils.readConfig(Utils.DEFAULT_CONFIG_FILE);
+ properties = Utils.readConfig(UtilsV8.DEFAULT_CONFIG_FILE);
}
catch (IOException e) {
- throw new DataStoreException("Unable to initialize Azure Data Store from " + Utils.DEFAULT_CONFIG_FILE, e);
+ throw new DataStoreException("Unable to initialize Azure Data Store from " + UtilsV8.DEFAULT_CONFIG_FILE, e);
}
}
try {
- Utils.setProxyIfNeeded(properties);
+ UtilsV8.setProxyIfNeeded(properties);
createBlobContainer = PropertiesUtil.toBoolean(
- org.apache.jackrabbit.oak.commons.StringUtils.emptyToNull(properties.getProperty(AzureConstants.AZURE_CREATE_CONTAINER)), true);
+ Strings.emptyToNull(properties.getProperty(AzureConstants.AZURE_CREATE_CONTAINER)), true);
initAzureDSConfig();
concurrentRequestCount = PropertiesUtil.toInteger(
properties.getProperty(AzureConstants.AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION),
- DEFAULT_CONCURRENT_REQUEST_COUNT);
- if (concurrentRequestCount < DEFAULT_CONCURRENT_REQUEST_COUNT) {
+ AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT);
+ if (concurrentRequestCount < AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT) {
LOG.warn("Invalid setting [{}] for concurrentRequestsPerOperation (too low); resetting to {}",
concurrentRequestCount,
- DEFAULT_CONCURRENT_REQUEST_COUNT);
- concurrentRequestCount = DEFAULT_CONCURRENT_REQUEST_COUNT;
- } else if (concurrentRequestCount > MAX_CONCURRENT_REQUEST_COUNT) {
+ AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT);
+ concurrentRequestCount = AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT;
+ } else if (concurrentRequestCount > AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT) {
LOG.warn("Invalid setting [{}] for concurrentRequestsPerOperation (too high); resetting to {}",
concurrentRequestCount,
- MAX_CONCURRENT_REQUEST_COUNT);
- concurrentRequestCount = MAX_CONCURRENT_REQUEST_COUNT;
+ AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT);
+ concurrentRequestCount = AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT;
}
LOG.info("Using concurrentRequestsPerOperation={}", concurrentRequestCount);
- retryPolicy = Utils.getRetryPolicy(properties.getProperty(AzureConstants.AZURE_BLOB_MAX_REQUEST_RETRY));
+ retryPolicy = UtilsV8.getRetryPolicy(properties.getProperty(AzureConstants.AZURE_BLOB_MAX_REQUEST_RETRY));
if (properties.getProperty(AzureConstants.AZURE_BLOB_REQUEST_TIMEOUT) != null) {
requestTimeout = PropertiesUtil.toInteger(properties.getProperty(AzureConstants.AZURE_BLOB_REQUEST_TIMEOUT), RetryPolicy.DEFAULT_CLIENT_RETRY_COUNT);
}
presignedDownloadURIVerifyExists = PropertiesUtil.toBoolean(
- org.apache.jackrabbit.oak.commons.StringUtils.emptyToNull(properties.getProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_VERIFY_EXISTS)), true);
+ Strings.emptyToNull(properties.getProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_VERIFY_EXISTS)), true);
enableSecondaryLocation = PropertiesUtil.toBoolean(
properties.getProperty(AzureConstants.AZURE_BLOB_ENABLE_SECONDARY_LOCATION_NAME),
@@ -242,7 +240,7 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
// Initialize reference key secret
boolean createRefSecretOnInit = PropertiesUtil.toBoolean(
- org.apache.jackrabbit.oak.commons.StringUtils.emptyToNull(properties.getProperty(AzureConstants.AZURE_REF_ON_INIT)), true);
+ Strings.emptyToNull(properties.getProperty(AzureConstants.AZURE_REF_ON_INIT)), true);
if (createRefSecretOnInit) {
getOrCreateReferenceKey();
@@ -258,7 +256,7 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
}
private void initAzureDSConfig() {
- AzureBlobContainerProvider.Builder builder = AzureBlobContainerProvider.Builder.builder(properties.getProperty(AzureConstants.AZURE_BLOB_CONTAINER_NAME))
+ AzureBlobContainerProviderV8.Builder builder = AzureBlobContainerProviderV8.Builder.builder(properties.getProperty(AzureConstants.AZURE_BLOB_CONTAINER_NAME))
.withAzureConnectionString(properties.getProperty(AzureConstants.AZURE_CONNECTION_STRING, ""))
.withAccountName(properties.getProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, ""))
.withBlobEndpoint(properties.getProperty(AzureConstants.AZURE_BLOB_ENDPOINT, ""))
@@ -330,7 +328,7 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
BlobRequestOptions options = new BlobRequestOptions();
options.setConcurrentRequestCount(concurrentRequestCount);
- boolean useBufferedStream = len < BUFFERED_STREAM_THRESHOLD;
+ boolean useBufferedStream = len < AZURE_BLOB_BUFFERED_STREAM_THRESHOLD;
final InputStream in = useBufferedStream ? new BufferedInputStream(new FileInputStream(file)) : new FileInputStream(file);
try {
blob.upload(in, len, null, options, null);
@@ -348,8 +346,8 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
blob.downloadAttributes();
if (blob.getProperties().getLength() != len) {
throw new DataStoreException("Length Collision. identifier=" + key +
- " new length=" + len +
- " old length=" + blob.getProperties().getLength());
+ " new length=" + len +
+ " old length=" + blob.getProperties().getLength());
}
LOG.trace("Blob already exists. identifier={} lastModified={}", key, getLastModified(blob));
@@ -357,7 +355,7 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
blob.uploadMetadata();
LOG.debug("Blob updated. identifier={} lastModified={} duration={}", key,
- getLastModified(blob), (System.currentTimeMillis() - start));
+ getLastModified(blob), (System.currentTimeMillis() - start));
}
catch (StorageException e) {
LOG.info("Error writing blob. identifier={}", key, e);
@@ -388,43 +386,6 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
return status == CopyStatus.SUCCESS;
}
- @Override
- public byte[] getOrCreateReferenceKey() throws DataStoreException {
- try {
- if (secret != null && secret.length != 0) {
- return secret;
- } else {
- byte[] key;
- // Try reading from the metadata folder if it exists
- key = readMetadataBytes(REF_KEY);
- if (key == null) {
- key = super.getOrCreateReferenceKey();
- addMetadataRecord(new ByteArrayInputStream(key), REF_KEY);
- key = readMetadataBytes(REF_KEY);
- }
- secret = key;
- return secret;
- }
- } catch (IOException e) {
- throw new DataStoreException("Unable to get or create key " + e);
- }
- }
-
- private byte[] readMetadataBytes(String name) throws IOException, DataStoreException {
- DataRecord rec = getMetadataRecord(name);
- byte[] key = null;
- if (rec != null) {
- InputStream stream = null;
- try {
- stream = rec.getStream();
- return IOUtils.toByteArray(stream);
- } finally {
- IOUtils.closeQuietly(stream);
- }
- }
- return key;
- }
-
@Override
public DataRecord getRecord(DataIdentifier identifier) throws DataStoreException {
if (null == identifier) {
@@ -445,7 +406,7 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
getLastModified(blob),
blob.getProperties().getLength());
LOG.debug("Data record read for blob. identifier={} duration={} record={}",
- key, (System.currentTimeMillis() - start), record);
+ key, (System.currentTimeMillis() - start), record);
return record;
}
catch (StorageException e) {
@@ -473,8 +434,6 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
input -> new DataIdentifier(getIdentifierName(input.getName())));
}
-
-
@Override
public Iterator<DataRecord> getAllRecords() {
final AbstractSharedBackend backend = this;
@@ -511,7 +470,7 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
}
@Override
- public void close() throws DataStoreException {
+ public void close() {
azureBlobContainerProvider.close();
LOG.info("AzureBlobBackend closed.");
}
@@ -549,7 +508,7 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
if (null == input) {
throw new NullPointerException("input");
}
- if (StringUtils.isEmpty(name)) {
+ if (Strings.isNullOrEmpty(name)) {
throw new IllegalArgumentException("name");
}
long start = System.currentTimeMillis();
@@ -572,7 +531,7 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
if (null == input) {
throw new NullPointerException("input");
}
- if (StringUtils.isEmpty(name)) {
+ if (Strings.isNullOrEmpty(name)) {
throw new IllegalArgumentException("name");
}
long start = System.currentTimeMillis();
@@ -595,7 +554,7 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
private void addMetadataRecordImpl(final InputStream input, String name, long recordLength) throws DataStoreException {
try {
- CloudBlobDirectory metaDir = getAzureContainer().getDirectoryReference(META_DIR_NAME);
+ CloudBlobDirectory metaDir = getAzureContainer().getDirectoryReference(AZUre_BlOB_META_DIR_NAME);
CloudBlockBlob blob = metaDir.getBlockBlobReference(name);
addLastModified(blob);
blob.upload(input, recordLength);
@@ -616,7 +575,7 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
try {
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
- CloudBlobDirectory metaDir = getAzureContainer().getDirectoryReference(META_DIR_NAME);
+ CloudBlobDirectory metaDir = getAzureContainer().getDirectoryReference(AZUre_BlOB_META_DIR_NAME);
CloudBlockBlob blob = metaDir.getBlockBlobReference(name);
if (!blob.exists()) {
LOG.warn("Trying to read missing metadata. metadataName={}", name);
@@ -653,12 +612,12 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
throw new NullPointerException("prefix");
}
long start = System.currentTimeMillis();
- final List<DataRecord> records = new ArrayList<>();
+ final List<DataRecord> records = Lists.newArrayList();
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
try {
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
- CloudBlobDirectory metaDir = getAzureContainer().getDirectoryReference(META_DIR_NAME);
+ CloudBlobDirectory metaDir = getAzureContainer().getDirectoryReference(AZUre_BlOB_META_DIR_NAME);
for (ListBlobItem item : metaDir.listBlobs(prefix)) {
if (item instanceof CloudBlob) {
CloudBlob blob = (CloudBlob) item;
@@ -727,7 +686,7 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
try {
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
- CloudBlobDirectory metaDir = getAzureContainer().getDirectoryReference(META_DIR_NAME);
+ CloudBlobDirectory metaDir = getAzureContainer().getDirectoryReference(AZUre_BlOB_META_DIR_NAME);
int total = 0;
for (ListBlobItem item : metaDir.listBlobs(prefix)) {
if (item instanceof CloudBlob) {
@@ -775,54 +734,53 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
return false;
}
-
/**
* Get key from data identifier. Object is stored with key in ADS.
*/
private static String getKeyName(DataIdentifier identifier) {
String key = identifier.toString();
- return key.substring(0, 4) + Utils.DASH + key.substring(4);
+ return key.substring(0, 4) + UtilsV8.DASH + key.substring(4);
}
/**
* Get data identifier from key.
*/
private static String getIdentifierName(String key) {
- if (!key.contains(Utils.DASH)) {
+ if (!key.contains(UtilsV8.DASH)) {
return null;
- } else if (key.contains(META_KEY_PREFIX)) {
+ } else if (key.contains(AZURE_BLOB_META_KEY_PREFIX)) {
return key;
}
return key.substring(0, 4) + key.substring(5);
}
private static String addMetaKeyPrefix(final String key) {
- return META_KEY_PREFIX + key;
+ return AZURE_BLOB_META_KEY_PREFIX + key;
}
private static String stripMetaKeyPrefix(String name) {
- if (name.startsWith(META_KEY_PREFIX)) {
- return name.substring(META_KEY_PREFIX.length());
+ if (name.startsWith(AZURE_BLOB_META_KEY_PREFIX)) {
+ return name.substring(AZURE_BLOB_META_KEY_PREFIX.length());
}
return name;
}
private static void addLastModified(CloudBlockBlob blob) {
- blob.getMetadata().put(LAST_MODIFIED_KEY, String.valueOf(System.currentTimeMillis()));
+ blob.getMetadata().put(AZURE_BLOB_LAST_MODIFIED_KEY, String.valueOf(System.currentTimeMillis()));
}
private static long getLastModified(CloudBlob blob) {
- if (blob.getMetadata().containsKey(LAST_MODIFIED_KEY)) {
- return Long.parseLong(blob.getMetadata().get(LAST_MODIFIED_KEY));
+ if (blob.getMetadata().containsKey(AZURE_BLOB_LAST_MODIFIED_KEY)) {
+ return Long.parseLong(blob.getMetadata().get(AZURE_BLOB_LAST_MODIFIED_KEY));
}
return blob.getProperties().getLastModified().getTime();
}
- void setHttpDownloadURIExpirySeconds(int seconds) {
+ protected void setHttpDownloadURIExpirySeconds(int seconds) {
httpDownloadURIExpirySeconds = seconds;
}
- void setHttpDownloadURICacheSize(int maxSize) {
+ protected void setHttpDownloadURICacheSize(int maxSize) {
// max size 0 or smaller is used to turn off the cache
if (maxSize > 0) {
LOG.info("presigned GET URI cache enabled, maxSize = {} items, expiry = {} seconds", maxSize, httpDownloadURIExpirySeconds / 2);
@@ -836,7 +794,7 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
}
}
- URI createHttpDownloadURI(@NotNull DataIdentifier identifier,
+ protected URI createHttpDownloadURI(@NotNull DataIdentifier identifier,
@NotNull DataRecordDownloadOptions downloadOptions) {
URI uri = null;
@@ -851,7 +809,7 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
throw new NullPointerException("Could not determine domain for direct download");
}
- String cacheKey = identifier.toString()
+ String cacheKey = identifier
+ domain
+ Objects.toString(downloadOptions.getContentTypeHeader(), "")
+ Objects.toString(downloadOptions.getContentDispositionHeader(), "");
@@ -878,13 +836,13 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
headers.setCacheControl(String.format("private, max-age=%d, immutable", httpDownloadURIExpirySeconds));
String contentType = downloadOptions.getContentTypeHeader();
- if (! StringUtils.isEmpty(contentType)) {
+ if (!Strings.isNullOrEmpty(contentType)) {
headers.setContentType(contentType);
}
String contentDisposition =
downloadOptions.getContentDispositionHeader();
- if (! StringUtils.isEmpty(contentDisposition)) {
+ if (!Strings.isNullOrEmpty(contentDisposition)) {
headers.setContentDisposition(contentDisposition);
}
@@ -901,7 +859,7 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
return uri;
}
- void setHttpUploadURIExpirySeconds(int seconds) { httpUploadURIExpirySeconds = seconds; }
+ protected void setHttpUploadURIExpirySeconds(int seconds) { httpUploadURIExpirySeconds = seconds; }
private DataIdentifier generateSafeRandomIdentifier() {
return new DataIdentifier(
@@ -912,10 +870,10 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
);
}
- DataRecordUpload initiateHttpUpload(long maxUploadSizeInBytes, int maxNumberOfURIs, @NotNull final DataRecordUploadOptions options) {
- List<URI> uploadPartURIs = new ArrayList<>();
- long minPartSize = MIN_MULTIPART_UPLOAD_PART_SIZE;
- long maxPartSize = MAX_MULTIPART_UPLOAD_PART_SIZE;
+ protected DataRecordUpload initiateHttpUpload(long maxUploadSizeInBytes, int maxNumberOfURIs, @NotNull final DataRecordUploadOptions options) {
+ List<URI> uploadPartURIs = Lists.newArrayList();
+ long minPartSize = AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE;
+ long maxPartSize = AZURE_BLOB_MAX_MULTIPART_UPLOAD_PART_SIZE;
if (0L >= maxUploadSizeInBytes) {
throw new IllegalArgumentException("maxUploadSizeInBytes must be > 0");
@@ -926,19 +884,19 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
else if (-1 > maxNumberOfURIs) {
throw new IllegalArgumentException("maxNumberOfURIs must either be > 0 or -1");
}
- else if (maxUploadSizeInBytes > MAX_SINGLE_PUT_UPLOAD_SIZE &&
+ else if (maxUploadSizeInBytes > AZURE_BLOB_MAX_SINGLE_PUT_UPLOAD_SIZE &&
maxNumberOfURIs == 1) {
throw new IllegalArgumentException(
String.format("Cannot do single-put upload with file size %d - exceeds max single-put upload size of %d",
maxUploadSizeInBytes,
- MAX_SINGLE_PUT_UPLOAD_SIZE)
+ AZURE_BLOB_MAX_SINGLE_PUT_UPLOAD_SIZE)
);
}
- else if (maxUploadSizeInBytes > MAX_BINARY_UPLOAD_SIZE) {
+ else if (maxUploadSizeInBytes > AZURE_BLOB_MAX_BINARY_UPLOAD_SIZE) {
throw new IllegalArgumentException(
String.format("Cannot do upload with file size %d - exceeds max upload size of %d",
maxUploadSizeInBytes,
- MAX_BINARY_UPLOAD_SIZE)
+ AZURE_BLOB_MAX_BINARY_UPLOAD_SIZE)
);
}
@@ -973,7 +931,7 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
maxNumberOfURIs,
Math.min(
(long) Math.ceil(((double) maxUploadSizeInBytes) / ((double) minPartSize)),
- MAX_ALLOWABLE_UPLOAD_URIS
+ AZURE_BLOB_MAX_ALLOWABLE_UPLOAD_URIS
)
);
} else {
@@ -983,8 +941,8 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
}
}
else {
- long maximalNumParts = (long) Math.ceil(((double) maxUploadSizeInBytes) / ((double) MIN_MULTIPART_UPLOAD_PART_SIZE));
- numParts = Math.min(maximalNumParts, MAX_ALLOWABLE_UPLOAD_URIS);
+ long maximalNumParts = (long) Math.ceil(((double) maxUploadSizeInBytes) / ((double) AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE));
+ numParts = Math.min(maximalNumParts, AZURE_BLOB_MAX_ALLOWABLE_UPLOAD_URIS);
}
String key = getKeyName(newIdentifier);
@@ -994,7 +952,7 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
}
EnumSet<SharedAccessBlobPermissions> perms = EnumSet.of(SharedAccessBlobPermissions.WRITE);
- Map<String, String> presignedURIRequestParams = new HashMap<>();
+ Map<String, String> presignedURIRequestParams = Maps.newHashMap();
// see https://docs.microsoft.com/en-us/rest/api/storageservices/put-block#uri-parameters
presignedURIRequestParams.put("comp", "block");
for (long blockId = 1; blockId <= numParts; ++blockId) {
@@ -1043,10 +1001,10 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
return null;
}
- DataRecord completeHttpUpload(@NotNull String uploadTokenStr)
+ protected DataRecord completeHttpUpload(@NotNull String uploadTokenStr)
throws DataRecordUploadException, DataStoreException {
- if (StringUtils.isEmpty(uploadTokenStr)) {
+ if (Strings.isNullOrEmpty(uploadTokenStr)) {
throw new IllegalArgumentException("uploadToken required");
}
@@ -1106,7 +1064,7 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
private String getDefaultBlobStorageDomain() {
String accountName = properties.getProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, "");
- if (StringUtils.isEmpty(accountName)) {
+ if (Strings.isNullOrEmpty(accountName)) {
LOG.warn("Can't generate presigned URI - Azure account name not found in properties");
return null;
}
@@ -1117,7 +1075,7 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
String domain = ignoreDomainOverride
? getDefaultBlobStorageDomain()
: downloadDomainOverride;
- if (StringUtils.isEmpty(domain)) {
+ if (Strings.isNullOrEmpty(domain)) {
domain = getDefaultBlobStorageDomain();
}
return domain;
@@ -1127,7 +1085,7 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
String domain = ignoreDomainOverride
? getDefaultBlobStorageDomain()
: uploadDomainOverride;
- if (StringUtils.isEmpty(domain)) {
+ if (Strings.isNullOrEmpty(domain)) {
domain = getDefaultBlobStorageDomain();
}
return domain;
@@ -1138,7 +1096,7 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
int expirySeconds,
SharedAccessBlobHeaders optionalHeaders,
String domain) {
- return createPresignedURI(key, permissions, expirySeconds, new HashMap<>(), optionalHeaders, domain);
+ return createPresignedURI(key, permissions, expirySeconds, Maps.newHashMap(), optionalHeaders, domain);
}
private URI createPresignedURI(String key,
@@ -1155,7 +1113,7 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
Map<String, String> additionalQueryParams,
SharedAccessBlobHeaders optionalHeaders,
String domain) {
- if (StringUtils.isEmpty(domain)) {
+ if (Strings.isNullOrEmpty(domain)) {
LOG.warn("Can't generate presigned URI - no Azure domain provided (is Azure account name configured?)");
return null;
}
@@ -1231,8 +1189,8 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
public static AzureBlobInfo fromCloudBlob(CloudBlob cloudBlob) throws StorageException {
cloudBlob.downloadAttributes();
return new AzureBlobInfo(cloudBlob.getName(),
- AzureBlobStoreBackend.getLastModified(cloudBlob),
- cloudBlob.getProperties().getLength());
+ AzureBlobStoreBackendV8.getLastModified(cloudBlob),
+ cloudBlob.getProperties().getLength());
}
}
@@ -1241,7 +1199,7 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
ResultContinuation resultContinuation;
boolean firstCall = true;
final Function<AzureBlobInfo, T> transformer;
- final Queue<AzureBlobInfo> items = new LinkedList<>();
+ final Queue<AzureBlobInfo> items = Lists.newLinkedList();
public RecordsIterator (Function<AzureBlobInfo, T> transformer) {
this.transformer = transformer;
@@ -1278,7 +1236,7 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
}
}
LOG.debug("Container records batch read. batchSize={} containerName={} duration={}",
- results.getLength(), getContainerName(), (System.currentTimeMillis() - start));
+ results.getLength(), getContainerName(), (System.currentTimeMillis() - start));
return results.getLength() > 0;
}
catch (StorageException e) {
@@ -1296,17 +1254,17 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
}
static class AzureBlobStoreDataRecord extends AbstractDataRecord {
- final AzureBlobContainerProvider azureBlobContainerProvider;
+ final AzureBlobContainerProviderV8 azureBlobContainerProvider;
final long lastModified;
final long length;
final boolean isMeta;
- public AzureBlobStoreDataRecord(AbstractSharedBackend backend, AzureBlobContainerProvider azureBlobContainerProvider,
+ public AzureBlobStoreDataRecord(AbstractSharedBackend backend, AzureBlobContainerProviderV8 azureBlobContainerProvider,
DataIdentifier key, long lastModified, long length) {
this(backend, azureBlobContainerProvider, key, lastModified, length, false);
}
- public AzureBlobStoreDataRecord(AbstractSharedBackend backend, AzureBlobContainerProvider azureBlobContainerProvider,
+ public AzureBlobStoreDataRecord(AbstractSharedBackend backend, AzureBlobContainerProviderV8 azureBlobContainerProvider,
DataIdentifier key, long lastModified, long length, boolean isMeta) {
super(backend, key);
this.azureBlobContainerProvider = azureBlobContainerProvider;
@@ -1316,7 +1274,7 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
}
@Override
- public long getLength() throws DataStoreException {
+ public long getLength() {
return length;
}
@@ -1349,17 +1307,55 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
@Override
public String toString() {
return "AzureBlobStoreDataRecord{" +
- "identifier=" + getIdentifier() +
- ", length=" + length +
- ", lastModified=" + lastModified +
- ", containerName='" + Optional.ofNullable(azureBlobContainerProvider).map(AzureBlobContainerProvider::getContainerName).orElse(null) + '\'' +
- '}';
+ "identifier=" + getIdentifier() +
+ ", length=" + length +
+ ", lastModified=" + lastModified +
+ ", containerName='" + Optional.ofNullable(azureBlobContainerProvider).map(AzureBlobContainerProviderV8::getContainerName).orElse(null) + '\'' +
+ '}';
}
}
private String getContainerName() {
return Optional.ofNullable(this.azureBlobContainerProvider)
- .map(AzureBlobContainerProvider::getContainerName)
+ .map(AzureBlobContainerProviderV8::getContainerName)
.orElse(null);
}
+
+ @Override
+ public byte[] getOrCreateReferenceKey() throws DataStoreException {
+ try {
+ if (secret != null && secret.length != 0) {
+ return secret;
+ } else {
+ byte[] key;
+ // Try reading from the metadata folder if it exists
+ key = readMetadataBytes(AZURE_BLOB_REF_KEY);
+ if (key == null) {
+ key = super.getOrCreateReferenceKey();
+ addMetadataRecord(new ByteArrayInputStream(key), AZURE_BLOB_REF_KEY);
+ key = readMetadataBytes(AZURE_BLOB_REF_KEY);
+ }
+ secret = key;
+ return secret;
+ }
+ } catch (IOException e) {
+ throw new DataStoreException("Unable to get or create key " + e);
+ }
+ }
+
+ protected byte[] readMetadataBytes(String name) throws IOException, DataStoreException {
+ DataRecord rec = getMetadataRecord(name);
+ byte[] key = null;
+ if (rec != null) {
+ InputStream stream = null;
+ try {
+ stream = rec.getStream();
+ return IOUtils.toByteArray(stream);
+ } finally {
+ IOUtils.closeQuietly(stream);
+ }
+ }
+ return key;
+ }
+
}
diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/Utils.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/UtilsV8.java
similarity index 82%
copy from oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/Utils.java
copy to oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/UtilsV8.java
index 185816f7c0..dfcbde2a81 100644
--- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/Utils.java
+++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/UtilsV8.java
@@ -17,18 +17,7 @@
* under the License.
*/
-package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.InetSocketAddress;
-import java.net.Proxy;
-import java.net.SocketAddress;
-import java.net.URISyntaxException;
-import java.security.InvalidKeyException;
-import java.util.Properties;
+package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v8;
import com.microsoft.azure.storage.CloudStorageAccount;
import com.microsoft.azure.storage.OperationContext;
@@ -41,11 +30,20 @@ import com.microsoft.azure.storage.blob.CloudBlobClient;
import com.microsoft.azure.storage.blob.CloudBlobContainer;
import org.apache.commons.lang3.StringUtils;
import org.apache.jackrabbit.core.data.DataStoreException;
+import com.google.common.base.Strings;
+import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants;
import org.apache.jackrabbit.oak.commons.PropertiesUtil;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
-public final class Utils {
+import java.net.InetSocketAddress;
+import java.net.Proxy;
+import java.net.SocketAddress;
+import java.net.URISyntaxException;
+import java.security.InvalidKeyException;
+import java.util.Properties;
+
+public final class UtilsV8 {
public static final String DEFAULT_CONFIG_FILE = "azure.properties";
@@ -54,7 +52,7 @@ public final class Utils {
/**
* private constructor so that class cannot initialized from outside.
*/
- private Utils() {
+ private UtilsV8() {
}
/**
@@ -82,14 +80,15 @@ public final class Utils {
return getBlobContainer(connectionString, containerName, null);
}
+
public static CloudBlobContainer getBlobContainer(@NotNull final String connectionString,
@NotNull final String containerName,
@Nullable final BlobRequestOptions requestOptions) throws DataStoreException {
try {
CloudBlobClient client = (
(null == requestOptions)
- ? Utils.getBlobClient(connectionString)
- : Utils.getBlobClient(connectionString, requestOptions)
+ ? UtilsV8.getBlobClient(connectionString)
+ : UtilsV8.getBlobClient(connectionString, requestOptions)
);
return client.getContainerReference(containerName);
} catch (InvalidKeyException | URISyntaxException | StorageException e) {
@@ -101,8 +100,8 @@ public final class Utils {
String proxyHost = properties.getProperty(AzureConstants.PROXY_HOST);
String proxyPort = properties.getProperty(AzureConstants.PROXY_PORT);
- if (!StringUtils.isEmpty(proxyHost) &&
- StringUtils.isEmpty(proxyPort)) {
+ if (!Strings.isNullOrEmpty(proxyHost) &&
+ Strings.isNullOrEmpty(proxyPort)) {
int port = Integer.parseInt(proxyPort);
SocketAddress proxyAddr = new InetSocketAddress(proxyHost, port);
Proxy proxy = new Proxy(Proxy.Type.HTTP, proxyAddr);
@@ -110,18 +109,6 @@ public final class Utils {
}
}
- public static RetryPolicy getRetryPolicy(final String maxRequestRetry) {
- int retries = PropertiesUtil.toInteger(maxRequestRetry, -1);
- if (retries < 0) {
- return null;
- }
- if (retries == 0) {
- return new RetryNoRetry();
- }
- return new RetryExponentialRetry(RetryPolicy.DEFAULT_CLIENT_BACKOFF, retries);
- }
-
-
public static String getConnectionStringFromProperties(Properties properties) {
String sasUri = properties.getProperty(AzureConstants.AZURE_SAS, "");
@@ -140,7 +127,7 @@ public final class Utils {
return getConnectionString(
accountName,
- accountKey,
+ accountKey,
blobEndpoint);
}
@@ -155,40 +142,26 @@ public final class Utils {
public static String getConnectionString(final String accountName, final String accountKey) {
return getConnectionString(accountName, accountKey, null);
}
-
+
public static String getConnectionString(final String accountName, final String accountKey, String blobEndpoint) {
StringBuilder connString = new StringBuilder("DefaultEndpointsProtocol=https");
connString.append(";AccountName=").append(accountName);
connString.append(";AccountKey=").append(accountKey);
-
- if (!StringUtils.isEmpty(blobEndpoint)) {
+
+ if (!Strings.isNullOrEmpty(blobEndpoint)) {
connString.append(";BlobEndpoint=").append(blobEndpoint);
}
return connString.toString();
}
- /**
- * Read a configuration properties file. If the file name ends with ";burn",
- * the file is deleted after reading.
- *
- * @param fileName the properties file name
- * @return the properties
- * @throws java.io.IOException if the file doesn't exist
- */
- public static Properties readConfig(String fileName) throws IOException {
- if (!new File(fileName).exists()) {
- throw new IOException("Config file not found. fileName=" + fileName);
+ public static RetryPolicy getRetryPolicy(final String maxRequestRetry) {
+ int retries = PropertiesUtil.toInteger(maxRequestRetry, -1);
+ if (retries < 0) {
+ return null;
}
- Properties prop = new Properties();
- InputStream in = null;
- try {
- in = new FileInputStream(fileName);
- prop.load(in);
- } finally {
- if (in != null) {
- in.close();
- }
+ else if (retries == 0) {
+ return new RetryNoRetry();
}
- return prop;
+ return new RetryExponentialRetry(RetryPolicy.DEFAULT_CLIENT_BACKOFF, retries);
}
}
diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendTest.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendTest.java
index 788ca33e9e..5ba8052876 100644
--- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendTest.java
+++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendTest.java
@@ -18,8 +18,10 @@
*/
package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;
-import com.microsoft.azure.storage.StorageException;
-import com.microsoft.azure.storage.blob.CloudBlobContainer;
+import com.azure.core.util.BinaryData;
+import com.azure.storage.blob.BlobContainerClient;
+import com.azure.storage.blob.sas.BlobSasPermission;
+import com.azure.storage.blob.sas.BlobServiceSasSignatureValues;
import com.microsoft.azure.storage.blob.SharedAccessBlobPermissions;
import com.microsoft.azure.storage.blob.SharedAccessBlobPolicy;
import org.apache.jackrabbit.core.data.DataRecord;
@@ -29,10 +31,9 @@ import org.junit.After;
import org.junit.ClassRule;
import org.junit.Test;
-import java.io.IOException;
-import java.net.URISyntaxException;
import java.time.Duration;
import java.time.Instant;
+import java.time.OffsetDateTime;
import java.util.Date;
import java.util.EnumSet;
import java.util.Properties;
@@ -41,12 +42,8 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
-import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.ADD;
-import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.CREATE;
-import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.LIST;
-import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.READ;
-import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.WRITE;
import static java.util.stream.Collectors.toSet;
+import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZUre_BlOB_META_DIR_NAME;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
@@ -62,11 +59,9 @@ public class AzureBlobStoreBackendTest {
public static AzuriteDockerRule azurite = new AzuriteDockerRule();
private static final String CONTAINER_NAME = "blobstore";
- private static final EnumSet<SharedAccessBlobPermissions> READ_ONLY = EnumSet.of(READ, LIST);
- private static final EnumSet<SharedAccessBlobPermissions> READ_WRITE = EnumSet.of(READ, LIST, CREATE, WRITE, ADD);
private static final Set<String> BLOBS = Set.of("blob1", "blob2");
- private CloudBlobContainer container;
+ private BlobContainerClient container;
@After
public void tearDown() throws Exception {
@@ -77,8 +72,14 @@ public class AzureBlobStoreBackendTest {
@Test
public void initWithSharedAccessSignature_readOnly() throws Exception {
- CloudBlobContainer container = createBlobContainer();
- String sasToken = container.generateSharedAccessSignature(policy(READ_ONLY), null);
+ BlobContainerClient container = createBlobContainer();
+ OffsetDateTime expiryTime = OffsetDateTime.now().plusDays(7);
+ BlobSasPermission permissions = new BlobSasPermission().setReadPermission(true)
+ .setWritePermission(false)
+ .setListPermission(true);
+
+ BlobServiceSasSignatureValues sasValues = new BlobServiceSasSignatureValues(expiryTime, permissions);
+ String sasToken = container.generateSas(sasValues);
AzureBlobStoreBackend azureBlobStoreBackend = new AzureBlobStoreBackend();
azureBlobStoreBackend.setProperties(getConfigurationWithSasToken(sasToken));
@@ -91,8 +92,16 @@ public class AzureBlobStoreBackendTest {
@Test
public void initWithSharedAccessSignature_readWrite() throws Exception {
- CloudBlobContainer container = createBlobContainer();
- String sasToken = container.generateSharedAccessSignature(policy(READ_WRITE), null);
+ BlobContainerClient container = createBlobContainer();
+ OffsetDateTime expiryTime = OffsetDateTime.now().plusDays(7);
+ BlobSasPermission permissions = new BlobSasPermission().setReadPermission(true)
+ .setListPermission(true)
+ .setAddPermission(true)
+ .setCreatePermission(true)
+ .setWritePermission(true);
+
+ BlobServiceSasSignatureValues sasValues = new BlobServiceSasSignatureValues(expiryTime, permissions);
+ String sasToken = container.generateSas(sasValues);
AzureBlobStoreBackend azureBlobStoreBackend = new AzureBlobStoreBackend();
azureBlobStoreBackend.setProperties(getConfigurationWithSasToken(sasToken));
@@ -106,9 +115,14 @@ public class AzureBlobStoreBackendTest {
@Test
public void connectWithSharedAccessSignatureURL_expired() throws Exception {
- CloudBlobContainer container = createBlobContainer();
- SharedAccessBlobPolicy expiredPolicy = policy(READ_WRITE, yesterday());
- String sasToken = container.generateSharedAccessSignature(expiredPolicy, null);
+ BlobContainerClient container = createBlobContainer();
+
+ OffsetDateTime expiryTime = OffsetDateTime.now().minusDays(1);
+ BlobSasPermission permissions = new BlobSasPermission().setReadPermission(true)
+ .setWritePermission(true);
+
+ BlobServiceSasSignatureValues sasValues = new BlobServiceSasSignatureValues(expiryTime, permissions);
+ String sasToken = container.generateSas(sasValues);
AzureBlobStoreBackend azureBlobStoreBackend = new AzureBlobStoreBackend();
azureBlobStoreBackend.setProperties(getConfigurationWithSasToken(sasToken));
@@ -188,10 +202,10 @@ public class AzureBlobStoreBackendTest {
return System.getenv(variableName);
}
- private CloudBlobContainer createBlobContainer() throws Exception {
- container = azurite.getContainer("blobstore");
+ private BlobContainerClient createBlobContainer() throws Exception {
+ container = azurite.getContainer("blobstore", getConnectionString());
for (String blob : BLOBS) {
- container.getBlockBlobReference(blob + ".txt").uploadText(blob);
+ container.getBlobClient(blob + ".txt").upload(BinaryData.fromString(blob), true);
}
return container;
}
@@ -241,11 +255,10 @@ public class AzureBlobStoreBackendTest {
}
private static void assertReadAccessGranted(AzureBlobStoreBackend backend, Set<String> expectedBlobs) throws Exception {
- CloudBlobContainer container = backend.getAzureContainer();
+ BlobContainerClient container = backend.getAzureContainer();
Set<String> actualBlobNames = StreamSupport.stream(container.listBlobs().spliterator(), false)
- .map(blob -> blob.getUri().getPath())
- .map(path -> path.substring(path.lastIndexOf('/') + 1))
- .filter(path -> !path.isEmpty())
+ .map(blobItem -> container.getBlobClient(blobItem.getName()).getBlobName())
+ .filter(name -> !name.contains(AZUre_BlOB_META_DIR_NAME))
.collect(toSet());
Set<String> expectedBlobNames = expectedBlobs.stream().map(name -> name + ".txt").collect(toSet());
@@ -255,8 +268,8 @@ public class AzureBlobStoreBackendTest {
Set<String> actualBlobContent = actualBlobNames.stream()
.map(name -> {
try {
- return container.getBlockBlobReference(name).downloadText();
- } catch (StorageException | IOException | URISyntaxException e) {
+ return container.getBlobClient(name).getBlockBlobClient().downloadContent().toString();
+ } catch (Exception e) {
throw new RuntimeException("Error while reading blob " + name, e);
}
})
@@ -266,7 +279,8 @@ public class AzureBlobStoreBackendTest {
private static void assertWriteAccessGranted(AzureBlobStoreBackend backend, String blob) throws Exception {
backend.getAzureContainer()
- .getBlockBlobReference(blob + ".txt").uploadText(blob);
+ .getBlobClient(blob + ".txt")
+ .upload(BinaryData.fromString(blob), true);
}
private static void assertWriteAccessNotGranted(AzureBlobStoreBackend backend) {
@@ -299,10 +313,10 @@ public class AzureBlobStoreBackendTest {
return Utils.getConnectionString(AzuriteDockerRule.ACCOUNT_NAME, AzuriteDockerRule.ACCOUNT_KEY, azurite.getBlobEndpoint());
}
- private static void assertReferenceSecret(AzureBlobStoreBackend azureBlobStoreBackend)
- throws DataStoreException, IOException {
+ private static void assertReferenceSecret(AzureBlobStoreBackend AzureBlobStoreBackend)
+ throws DataStoreException {
// assert secret already created on init
- DataRecord refRec = azureBlobStoreBackend.getMetadataRecord("reference.key");
+ DataRecord refRec = AzureBlobStoreBackend.getMetadataRecord("reference.key");
assertNotNull("Reference data record null", refRec);
assertTrue("reference key is empty", refRec.getLength() > 0);
}
diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataRecordAccessProviderIT.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataRecordAccessProviderIT.java
index 203d783188..66f1aff384 100644
--- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataRecordAccessProviderIT.java
+++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataRecordAccessProviderIT.java
@@ -76,7 +76,7 @@ public class AzureDataRecordAccessProviderIT extends AbstractDataRecordAccessPro
@Override
protected long getProviderMaxPartSize() {
- return AzureBlobStoreBackend.MAX_MULTIPART_UPLOAD_PART_SIZE;
+ return AzureConstants.AZURE_BLOB_MAX_MULTIPART_UPLOAD_PART_SIZE;
}
@Override
diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataRecordAccessProviderTest.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataRecordAccessProviderTest.java
index 85b0ede09f..1a63baee2e 100644
--- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataRecordAccessProviderTest.java
+++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataRecordAccessProviderTest.java
@@ -93,19 +93,19 @@ public class AzureDataRecordAccessProviderTest extends AbstractDataRecordAccessP
@Override
protected long getProviderMinPartSize() {
- return Math.max(0L, AzureBlobStoreBackend.MIN_MULTIPART_UPLOAD_PART_SIZE);
+ return Math.max(0L, AzureConstants.AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE);
}
@Override
protected long getProviderMaxPartSize() {
- return AzureBlobStoreBackend.MAX_MULTIPART_UPLOAD_PART_SIZE;
+ return AzureConstants.AZURE_BLOB_MAX_MULTIPART_UPLOAD_PART_SIZE;
}
@Override
- protected long getProviderMaxSinglePutSize() { return AzureBlobStoreBackend.MAX_SINGLE_PUT_UPLOAD_SIZE; }
+ protected long getProviderMaxSinglePutSize() { return AzureConstants.AZURE_BLOB_MAX_SINGLE_PUT_UPLOAD_SIZE; }
@Override
- protected long getProviderMaxBinaryUploadSize() { return AzureBlobStoreBackend.MAX_BINARY_UPLOAD_SIZE; }
+ protected long getProviderMaxBinaryUploadSize() { return AzureConstants.AZURE_BLOB_MAX_BINARY_UPLOAD_SIZE; }
@Override
protected boolean isSinglePutURI(URI uri) {
diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreUtils.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreUtils.java
index fe9a765192..514d3311e4 100644
--- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreUtils.java
+++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreUtils.java
@@ -33,9 +33,9 @@ import java.util.Properties;
import javax.net.ssl.HttpsURLConnection;
-import org.apache.commons.lang3.StringUtils;
-import com.microsoft.azure.storage.blob.CloudBlobContainer;
+import com.azure.storage.blob.BlobContainerClient;
import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.jackrabbit.core.data.DataStore;
import org.apache.jackrabbit.oak.commons.PropertiesUtil;
import org.apache.jackrabbit.oak.commons.collections.MapUtils;
@@ -114,6 +114,8 @@ public class AzureDataStoreUtils extends DataStoreUtils {
props = new Properties();
props.putAll(filtered);
}
+
+ props.setProperty("blob.azure.v12.enabled", "true");
return props;
}
@@ -142,12 +144,12 @@ public class AzureDataStoreUtils extends DataStoreUtils {
@Nullable final Properties overrideProperties)
throws Exception {
assumeTrue(isAzureConfigured());
- DataStore ds = (T) getAzureDataStore(getDirectAccessDataStoreProperties(overrideProperties), homeDir.newFolder().getAbsolutePath());
+ T ds = (T) getAzureDataStore(getDirectAccessDataStoreProperties(overrideProperties), homeDir.newFolder().getAbsolutePath());
if (ds instanceof ConfigurableDataRecordAccessProvider) {
((ConfigurableDataRecordAccessProvider) ds).setDirectDownloadURIExpirySeconds(directDownloadExpirySeconds);
((ConfigurableDataRecordAccessProvider) ds).setDirectUploadURIExpirySeconds(directUploadExpirySeconds);
}
- return (T) ds;
+ return ds;
}
public static Properties getDirectAccessDataStoreProperties() {
@@ -160,7 +162,6 @@ public class AzureDataStoreUtils extends DataStoreUtils {
if (null != overrideProperties) {
mergedProperties.putAll(overrideProperties);
}
-
// set properties needed for direct access testing
if (null == mergedProperties.getProperty("cacheSize", null)) {
mergedProperties.put("cacheSize", "0");
@@ -179,7 +180,7 @@ public class AzureDataStoreUtils extends DataStoreUtils {
try (AzureBlobContainerProvider azureBlobContainerProvider = AzureBlobContainerProvider.Builder.builder(containerName).initializeWithProperties(props)
.build()) {
- CloudBlobContainer container = azureBlobContainerProvider.getBlobContainer();
+ BlobContainerClient container = azureBlobContainerProvider.getBlobContainer();
boolean result = container.deleteIfExists();
log.info("Container deleted. containerName={} existed={}", containerName, result);
}
diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzuriteDockerRule.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzuriteDockerRule.java
index cb709aca29..04003156c4 100644
--- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzuriteDockerRule.java
+++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzuriteDockerRule.java
@@ -16,6 +16,9 @@
*/
package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;
+import com.azure.storage.blob.BlobContainerClient;
+import com.azure.storage.blob.BlobServiceClient;
+import com.azure.storage.blob.BlobServiceClientBuilder;
import com.microsoft.azure.storage.CloudStorageAccount;
import com.microsoft.azure.storage.StorageException;
import com.microsoft.azure.storage.blob.CloudBlobClient;
@@ -38,7 +41,7 @@ import java.util.concurrent.atomic.AtomicReference;
public class AzuriteDockerRule extends ExternalResource {
- private static final DockerImageName DOCKER_IMAGE_NAME = DockerImageName.parse("mcr.microsoft.com/azure-storage/azurite:3.29.0");
+ private static final DockerImageName DOCKER_IMAGE_NAME = DockerImageName.parse("mcr.microsoft.com/azure-storage/azurite:3.31.0");
public static final String ACCOUNT_KEY = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==";
public static final String ACCOUNT_NAME = "devstoreaccount1";
private static final AtomicReference<Exception> STARTUP_EXCEPTION = new AtomicReference<>();
@@ -109,6 +112,17 @@ public class AzuriteDockerRule extends ExternalResource {
return container;
}
+ public BlobContainerClient getContainer(String containerName, String connectionString) {
+ BlobServiceClient blobServiceClient = new BlobServiceClientBuilder()
+ .connectionString(connectionString)
+ .buildClient();
+
+ BlobContainerClient blobContainerClient = blobServiceClient.getBlobContainerClient(containerName);
+ blobContainerClient.deleteIfExists();
+ blobContainerClient.create();
+ return blobContainerClient;
+ }
+
public CloudStorageAccount getCloudStorageAccount() throws URISyntaxException, InvalidKeyException {
String blobEndpoint = "BlobEndpoint=" + getBlobEndpoint();
String accountName = "AccountName=" + ACCOUNT_NAME;
diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDS.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDS.java
index 14eca1b94a..2405ecc97c 100644
--- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDS.java
+++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDS.java
@@ -27,8 +27,6 @@ import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.slf4j.event.Level;
import java.util.Properties;
@@ -44,7 +42,6 @@ import javax.jcr.RepositoryException;
*/
public class TestAzureDS extends AbstractDataStoreTest {
- protected static final Logger LOG = LoggerFactory.getLogger(TestAzureDS.class);
protected Properties props = new Properties();
protected String container;
@@ -57,7 +54,7 @@ public class TestAzureDS extends AbstractDataStoreTest {
@Before
public void setUp() throws Exception {
props.putAll(AzureDataStoreUtils.getAzureConfig());
- container = String.valueOf(randomGen.nextInt(9999)) + "-" + String.valueOf(randomGen.nextInt(9999))
+ container = randomGen.nextInt(9999) + "-" + randomGen.nextInt(9999)
+ "-test";
props.setProperty(AzureConstants.AZURE_BLOB_CONTAINER_NAME, container);
props.setProperty("secret", "123456");
diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDSWithSmallCache.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDSWithSmallCache.java
index 79072ddd75..159162dec1 100644
--- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDSWithSmallCache.java
+++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDSWithSmallCache.java
@@ -26,7 +26,7 @@ import org.slf4j.LoggerFactory;
* Test {@link CachingDataStore} with AzureBlobStoreBackend and with very small size (@link
* {@link LocalCache}.
* It requires to pass azure config file via system property or system properties by prefixing with 'ds.'.
- * See details @ {@link AzureDataStoreUtils}.
+ * See details @ {@link TestAzureDataStoreUtils}.
* For e.g. -Dconfig=/opt/cq/azure.properties. Sample azure properties located at
* src/test/resources/azure.properties
diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDsCacheOff.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDsCacheOff.java
index be31b57823..8396f6d41d 100644
--- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDsCacheOff.java
+++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDsCacheOff.java
@@ -24,7 +24,7 @@ import org.slf4j.LoggerFactory;
* Test {@link org.apache.jackrabbit.core.data.CachingDataStore} with AzureBlobStoreBackend
* and local cache Off.
* It requires to pass azure config file via system property or system properties by prefixing with 'ds.'.
- * See details @ {@link AzureDataStoreUtils}.
+ * See details @ {@link TestAzureDataStoreUtils}.
* For e.g. -Dconfig=/opt/cq/azure.properties. Sample azure properties located at
* src/test/resources/azure.properties
diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/UtilsTest.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/UtilsTest.java
index 5776fbbd37..695a4e2ab0 100644
--- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/UtilsTest.java
+++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/UtilsTest.java
@@ -16,14 +16,30 @@
*/
package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;
+import com.azure.storage.blob.BlobContainerClient;
+import com.azure.storage.common.policy.RequestRetryOptions;
+import org.apache.jackrabbit.core.data.DataStoreException;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileWriter;
+import java.io.IOException;
import java.util.Properties;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
public class UtilsTest {
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder();
+
@Test
public void testConnectionStringIsBasedOnProperty() {
Properties properties = new Properties();
@@ -77,5 +93,60 @@ public class UtilsTest {
String.format("BlobEndpoint=%s;SharedAccessSignature=%s", "endpoint", "sas"));
}
+ @Test
+ public void testReadConfig() throws IOException {
+ File tempFile = folder.newFile("test.properties");
+ try(FileWriter writer = new FileWriter(tempFile)) {
+ writer.write("key1=value1\n");
+ writer.write("key2=value2\n");
+ }
+
+ Properties properties = Utils.readConfig(tempFile.getAbsolutePath());
+ assertEquals("value1", properties.getProperty("key1"));
+ assertEquals("value2", properties.getProperty("key2"));
+ }
+
+ @Test
+ public void testReadConfig_exception() {
+ assertThrows(IOException.class, () -> Utils.readConfig("non-existent-file"));
+ }
+
+ @Test
+ public void testGetBlobContainer() throws IOException, DataStoreException {
+ File tempFile = folder.newFile("azure.properties");
+ try (FileWriter writer = new FileWriter(tempFile)) {
+ writer.write("proxyHost=127.0.0.1\n");
+ writer.write("proxyPort=8888\n");
+ }
+
+ Properties properties = new Properties();
+ properties.load(new FileInputStream(tempFile));
+
+ String connectionString = Utils.getConnectionString(AzuriteDockerRule.ACCOUNT_NAME, AzuriteDockerRule.ACCOUNT_KEY, "http://127.0.0.1:10000/devstoreaccount1" );
+ String containerName = "test-container";
+ RequestRetryOptions retryOptions = Utils.getRetryOptions("3", 3, null);
+
+ BlobContainerClient containerClient = Utils.getBlobContainer(connectionString, containerName, retryOptions, properties);
+ assertNotNull(containerClient);
+ }
+
+ @Test
+ public void testGetRetryOptions() {
+ RequestRetryOptions retryOptions = Utils.getRetryOptions("3", 3, null);
+ assertNotNull(retryOptions);
+ assertEquals(3, retryOptions.getMaxTries());
+ }
+ @Test
+ public void testGetRetryOptionsNoRetry() {
+ RequestRetryOptions retryOptions = Utils.getRetryOptions("0",3, null);
+ assertNotNull(retryOptions);
+ assertEquals(1, retryOptions.getMaxTries());
+ }
+
+ @Test
+ public void testGetRetryOptionsInvalid() {
+ RequestRetryOptions retryOptions = Utils.getRetryOptions("-1", 3, null);
+ assertNull(retryOptions);
+ }
}
diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendTest.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobContainerProviderV8Test.java
similarity index 89%
copy from oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendTest.java
copy to oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobContainerProviderV8Test.java
index 788ca33e9e..f5b15ee93e 100644
--- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendTest.java
+++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobContainerProviderV8Test.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;
+package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v8;
import com.microsoft.azure.storage.StorageException;
import com.microsoft.azure.storage.blob.CloudBlobContainer;
@@ -24,10 +24,13 @@ import com.microsoft.azure.storage.blob.SharedAccessBlobPermissions;
import com.microsoft.azure.storage.blob.SharedAccessBlobPolicy;
import org.apache.jackrabbit.core.data.DataRecord;
import org.apache.jackrabbit.core.data.DataStoreException;
+import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants;
+import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzuriteDockerRule;
import org.jetbrains.annotations.NotNull;
import org.junit.After;
import org.junit.ClassRule;
import org.junit.Test;
+import org.testcontainers.shaded.com.google.common.collect.ImmutableSet;
import java.io.IOException;
import java.net.URISyntaxException;
@@ -37,8 +40,6 @@ import java.util.Date;
import java.util.EnumSet;
import java.util.Properties;
import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.ADD;
@@ -53,7 +54,8 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.junit.Assume.assumeNotNull;
-public class AzureBlobStoreBackendTest {
+public class AzureBlobContainerProviderV8Test {
+
private static final String AZURE_ACCOUNT_NAME = "AZURE_ACCOUNT_NAME";
private static final String AZURE_TENANT_ID = "AZURE_TENANT_ID";
private static final String AZURE_CLIENT_ID = "AZURE_CLIENT_ID";
@@ -80,7 +82,7 @@ public class AzureBlobStoreBackendTest {
CloudBlobContainer container = createBlobContainer();
String sasToken = container.generateSharedAccessSignature(policy(READ_ONLY), null);
- AzureBlobStoreBackend azureBlobStoreBackend = new AzureBlobStoreBackend();
+ AzureBlobStoreBackendV8 azureBlobStoreBackend = new AzureBlobStoreBackendV8();
azureBlobStoreBackend.setProperties(getConfigurationWithSasToken(sasToken));
azureBlobStoreBackend.init();
@@ -94,14 +96,13 @@ public class AzureBlobStoreBackendTest {
CloudBlobContainer container = createBlobContainer();
String sasToken = container.generateSharedAccessSignature(policy(READ_WRITE), null);
- AzureBlobStoreBackend azureBlobStoreBackend = new AzureBlobStoreBackend();
+ AzureBlobStoreBackendV8 azureBlobStoreBackend = new AzureBlobStoreBackendV8();
azureBlobStoreBackend.setProperties(getConfigurationWithSasToken(sasToken));
azureBlobStoreBackend.init();
assertWriteAccessGranted(azureBlobStoreBackend, "file");
- assertReadAccessGranted(azureBlobStoreBackend,
- concat(BLOBS, "file"));
+ assertReadAccessGranted(azureBlobStoreBackend, concat(BLOBS, "file"));
}
@Test
@@ -110,7 +111,7 @@ public class AzureBlobStoreBackendTest {
SharedAccessBlobPolicy expiredPolicy = policy(READ_WRITE, yesterday());
String sasToken = container.generateSharedAccessSignature(expiredPolicy, null);
- AzureBlobStoreBackend azureBlobStoreBackend = new AzureBlobStoreBackend();
+ AzureBlobStoreBackendV8 azureBlobStoreBackend = new AzureBlobStoreBackendV8();
azureBlobStoreBackend.setProperties(getConfigurationWithSasToken(sasToken));
azureBlobStoreBackend.init();
@@ -121,7 +122,7 @@ public class AzureBlobStoreBackendTest {
@Test
public void initWithAccessKey() throws Exception {
- AzureBlobStoreBackend azureBlobStoreBackend = new AzureBlobStoreBackend();
+ AzureBlobStoreBackendV8 azureBlobStoreBackend = new AzureBlobStoreBackendV8();
azureBlobStoreBackend.setProperties(getConfigurationWithAccessKey());
azureBlobStoreBackend.init();
@@ -132,7 +133,7 @@ public class AzureBlobStoreBackendTest {
@Test
public void initWithConnectionURL() throws Exception {
- AzureBlobStoreBackend azureBlobStoreBackend = new AzureBlobStoreBackend();
+ AzureBlobStoreBackendV8 azureBlobStoreBackend = new AzureBlobStoreBackendV8();
azureBlobStoreBackend.setProperties(getConfigurationWithConnectionString());
azureBlobStoreBackend.init();
@@ -143,7 +144,7 @@ public class AzureBlobStoreBackendTest {
@Test
public void initSecret() throws Exception {
- AzureBlobStoreBackend azureBlobStoreBackend = new AzureBlobStoreBackend();
+ AzureBlobStoreBackendV8 azureBlobStoreBackend = new AzureBlobStoreBackendV8();
azureBlobStoreBackend.setProperties(getConfigurationWithConnectionString());
azureBlobStoreBackend.init();
@@ -160,7 +161,7 @@ public class AzureBlobStoreBackendTest {
assumeNotNull(getEnvironmentVariable(AZURE_CLIENT_ID));
assumeNotNull(getEnvironmentVariable(AZURE_CLIENT_SECRET));
- AzureBlobStoreBackend azureBlobStoreBackend = new AzureBlobStoreBackend();
+ AzureBlobStoreBackendV8 azureBlobStoreBackend = new AzureBlobStoreBackendV8();
azureBlobStoreBackend.setProperties(getPropertiesWithServicePrincipals());
azureBlobStoreBackend.init();
@@ -240,7 +241,7 @@ public class AzureBlobStoreBackendTest {
return policy(permissions, Instant.now().plus(Duration.ofDays(7)));
}
- private static void assertReadAccessGranted(AzureBlobStoreBackend backend, Set<String> expectedBlobs) throws Exception {
+ private static void assertReadAccessGranted(AzureBlobStoreBackendV8 backend, Set<String> expectedBlobs) throws Exception {
CloudBlobContainer container = backend.getAzureContainer();
Set<String> actualBlobNames = StreamSupport.stream(container.listBlobs().spliterator(), false)
.map(blob -> blob.getUri().getPath())
@@ -264,12 +265,12 @@ public class AzureBlobStoreBackendTest {
assertEquals(expectedBlobs, actualBlobContent);
}
- private static void assertWriteAccessGranted(AzureBlobStoreBackend backend, String blob) throws Exception {
+ private static void assertWriteAccessGranted(AzureBlobStoreBackendV8 backend, String blob) throws Exception {
backend.getAzureContainer()
.getBlockBlobReference(blob + ".txt").uploadText(blob);
}
- private static void assertWriteAccessNotGranted(AzureBlobStoreBackend backend) {
+ private static void assertWriteAccessNotGranted(AzureBlobStoreBackendV8 backend) {
try {
assertWriteAccessGranted(backend, "test.txt");
fail("Write access should not be granted, but writing to the storage succeeded.");
@@ -278,7 +279,7 @@ public class AzureBlobStoreBackendTest {
}
}
- private static void assertReadAccessNotGranted(AzureBlobStoreBackend backend) {
+ private static void assertReadAccessNotGranted(AzureBlobStoreBackendV8 backend) {
try {
assertReadAccessGranted(backend, BLOBS);
fail("Read access should not be granted, but reading from the storage succeeded.");
@@ -292,18 +293,19 @@ public class AzureBlobStoreBackendTest {
}
private static Set<String> concat(Set<String> set, String element) {
- return Stream.concat(set.stream(), Stream.of(element)).collect(Collectors.toSet());
+ return ImmutableSet.<String>builder().addAll(set).add(element).build();
}
private static String getConnectionString() {
- return Utils.getConnectionString(AzuriteDockerRule.ACCOUNT_NAME, AzuriteDockerRule.ACCOUNT_KEY, azurite.getBlobEndpoint());
+ return UtilsV8.getConnectionString(AzuriteDockerRule.ACCOUNT_NAME, AzuriteDockerRule.ACCOUNT_KEY, azurite.getBlobEndpoint());
}
- private static void assertReferenceSecret(AzureBlobStoreBackend azureBlobStoreBackend)
+ private static void assertReferenceSecret(AzureBlobStoreBackendV8 azureBlobStoreBackend)
throws DataStoreException, IOException {
// assert secret already created on init
DataRecord refRec = azureBlobStoreBackend.getMetadataRecord("reference.key");
assertNotNull("Reference data record null", refRec);
assertTrue("reference key is empty", refRec.getLength() > 0);
}
-}
+
+}
\ No newline at end of file
diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendTest.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobStoreBackendV8Test.java
similarity index 91%
copy from oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendTest.java
copy to oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobStoreBackendV8Test.java
index 788ca33e9e..51d036328a 100644
--- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendTest.java
+++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobStoreBackendV8Test.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;
+package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v8;
import com.microsoft.azure.storage.StorageException;
import com.microsoft.azure.storage.blob.CloudBlobContainer;
@@ -24,6 +24,8 @@ import com.microsoft.azure.storage.blob.SharedAccessBlobPermissions;
import com.microsoft.azure.storage.blob.SharedAccessBlobPolicy;
import org.apache.jackrabbit.core.data.DataRecord;
import org.apache.jackrabbit.core.data.DataStoreException;
+import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants;
+import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzuriteDockerRule;
import org.jetbrains.annotations.NotNull;
import org.junit.After;
import org.junit.ClassRule;
@@ -53,7 +55,7 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.junit.Assume.assumeNotNull;
-public class AzureBlobStoreBackendTest {
+public class AzureBlobStoreBackendV8Test {
private static final String AZURE_ACCOUNT_NAME = "AZURE_ACCOUNT_NAME";
private static final String AZURE_TENANT_ID = "AZURE_TENANT_ID";
private static final String AZURE_CLIENT_ID = "AZURE_CLIENT_ID";
@@ -80,7 +82,7 @@ public class AzureBlobStoreBackendTest {
CloudBlobContainer container = createBlobContainer();
String sasToken = container.generateSharedAccessSignature(policy(READ_ONLY), null);
- AzureBlobStoreBackend azureBlobStoreBackend = new AzureBlobStoreBackend();
+ AzureBlobStoreBackendV8 azureBlobStoreBackend = new AzureBlobStoreBackendV8();
azureBlobStoreBackend.setProperties(getConfigurationWithSasToken(sasToken));
azureBlobStoreBackend.init();
@@ -94,7 +96,7 @@ public class AzureBlobStoreBackendTest {
CloudBlobContainer container = createBlobContainer();
String sasToken = container.generateSharedAccessSignature(policy(READ_WRITE), null);
- AzureBlobStoreBackend azureBlobStoreBackend = new AzureBlobStoreBackend();
+ AzureBlobStoreBackendV8 azureBlobStoreBackend = new AzureBlobStoreBackendV8();
azureBlobStoreBackend.setProperties(getConfigurationWithSasToken(sasToken));
azureBlobStoreBackend.init();
@@ -110,7 +112,7 @@ public class AzureBlobStoreBackendTest {
SharedAccessBlobPolicy expiredPolicy = policy(READ_WRITE, yesterday());
String sasToken = container.generateSharedAccessSignature(expiredPolicy, null);
- AzureBlobStoreBackend azureBlobStoreBackend = new AzureBlobStoreBackend();
+ AzureBlobStoreBackendV8 azureBlobStoreBackend = new AzureBlobStoreBackendV8();
azureBlobStoreBackend.setProperties(getConfigurationWithSasToken(sasToken));
azureBlobStoreBackend.init();
@@ -121,7 +123,7 @@ public class AzureBlobStoreBackendTest {
@Test
public void initWithAccessKey() throws Exception {
- AzureBlobStoreBackend azureBlobStoreBackend = new AzureBlobStoreBackend();
+ AzureBlobStoreBackendV8 azureBlobStoreBackend = new AzureBlobStoreBackendV8();
azureBlobStoreBackend.setProperties(getConfigurationWithAccessKey());
azureBlobStoreBackend.init();
@@ -132,7 +134,7 @@ public class AzureBlobStoreBackendTest {
@Test
public void initWithConnectionURL() throws Exception {
- AzureBlobStoreBackend azureBlobStoreBackend = new AzureBlobStoreBackend();
+ AzureBlobStoreBackendV8 azureBlobStoreBackend = new AzureBlobStoreBackendV8();
azureBlobStoreBackend.setProperties(getConfigurationWithConnectionString());
azureBlobStoreBackend.init();
@@ -143,7 +145,7 @@ public class AzureBlobStoreBackendTest {
@Test
public void initSecret() throws Exception {
- AzureBlobStoreBackend azureBlobStoreBackend = new AzureBlobStoreBackend();
+ AzureBlobStoreBackendV8 azureBlobStoreBackend = new AzureBlobStoreBackendV8();
azureBlobStoreBackend.setProperties(getConfigurationWithConnectionString());
azureBlobStoreBackend.init();
@@ -160,7 +162,7 @@ public class AzureBlobStoreBackendTest {
assumeNotNull(getEnvironmentVariable(AZURE_CLIENT_ID));
assumeNotNull(getEnvironmentVariable(AZURE_CLIENT_SECRET));
- AzureBlobStoreBackend azureBlobStoreBackend = new AzureBlobStoreBackend();
+ AzureBlobStoreBackendV8 azureBlobStoreBackend = new AzureBlobStoreBackendV8();
azureBlobStoreBackend.setProperties(getPropertiesWithServicePrincipals());
azureBlobStoreBackend.init();
@@ -240,7 +242,7 @@ public class AzureBlobStoreBackendTest {
return policy(permissions, Instant.now().plus(Duration.ofDays(7)));
}
- private static void assertReadAccessGranted(AzureBlobStoreBackend backend, Set<String> expectedBlobs) throws Exception {
+ private static void assertReadAccessGranted(AzureBlobStoreBackendV8 backend, Set<String> expectedBlobs) throws Exception {
CloudBlobContainer container = backend.getAzureContainer();
Set<String> actualBlobNames = StreamSupport.stream(container.listBlobs().spliterator(), false)
.map(blob -> blob.getUri().getPath())
@@ -264,12 +266,12 @@ public class AzureBlobStoreBackendTest {
assertEquals(expectedBlobs, actualBlobContent);
}
- private static void assertWriteAccessGranted(AzureBlobStoreBackend backend, String blob) throws Exception {
+ private static void assertWriteAccessGranted(AzureBlobStoreBackendV8 backend, String blob) throws Exception {
backend.getAzureContainer()
.getBlockBlobReference(blob + ".txt").uploadText(blob);
}
- private static void assertWriteAccessNotGranted(AzureBlobStoreBackend backend) {
+ private static void assertWriteAccessNotGranted(AzureBlobStoreBackendV8 backend) {
try {
assertWriteAccessGranted(backend, "test.txt");
fail("Write access should not be granted, but writing to the storage succeeded.");
@@ -278,7 +280,7 @@ public class AzureBlobStoreBackendTest {
}
}
- private static void assertReadAccessNotGranted(AzureBlobStoreBackend backend) {
+ private static void assertReadAccessNotGranted(AzureBlobStoreBackendV8 backend) {
try {
assertReadAccessGranted(backend, BLOBS);
fail("Read access should not be granted, but reading from the storage succeeded.");
@@ -296,10 +298,10 @@ public class AzureBlobStoreBackendTest {
}
private static String getConnectionString() {
- return Utils.getConnectionString(AzuriteDockerRule.ACCOUNT_NAME, AzuriteDockerRule.ACCOUNT_KEY, azurite.getBlobEndpoint());
+ return UtilsV8.getConnectionString(AzuriteDockerRule.ACCOUNT_NAME, AzuriteDockerRule.ACCOUNT_KEY, azurite.getBlobEndpoint());
}
- private static void assertReferenceSecret(AzureBlobStoreBackend azureBlobStoreBackend)
+ private static void assertReferenceSecret(AzureBlobStoreBackendV8 azureBlobStoreBackend)
throws DataStoreException, IOException {
// assert secret already created on init
DataRecord refRec = azureBlobStoreBackend.getMetadataRecord("reference.key");
diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/UtilsTest.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/UtilsV8Test.java
similarity index 61%
copy from oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/UtilsTest.java
copy to oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/UtilsV8Test.java
index 5776fbbd37..c02a7f05c4 100644
--- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/UtilsTest.java
+++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/UtilsV8Test.java
@@ -1,34 +1,37 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
*/
-package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;
+package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v8;
+import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants;
import org.junit.Test;
import java.util.Properties;
import static org.junit.Assert.assertEquals;
-public class UtilsTest {
+public class UtilsV8Test {
@Test
public void testConnectionStringIsBasedOnProperty() {
Properties properties = new Properties();
properties.put(AzureConstants.AZURE_CONNECTION_STRING, "DefaultEndpointsProtocol=https;AccountName=accountName;AccountKey=accountKey");
- String connectionString = Utils.getConnectionStringFromProperties(properties);
+ String connectionString = UtilsV8.getConnectionStringFromProperties(properties);
assertEquals(connectionString,"DefaultEndpointsProtocol=https;AccountName=accountName;AccountKey=accountKey");
}
@@ -37,7 +40,7 @@ public class UtilsTest {
Properties properties = new Properties();
properties.put(AzureConstants.AZURE_SAS, "sas");
properties.put(AzureConstants.AZURE_BLOB_ENDPOINT, "endpoint");
- String connectionString = Utils.getConnectionStringFromProperties(properties);
+ String connectionString = UtilsV8.getConnectionStringFromProperties(properties);
assertEquals(connectionString,
String.format("BlobEndpoint=%s;SharedAccessSignature=%s", "endpoint", "sas"));
}
@@ -47,7 +50,7 @@ public class UtilsTest {
Properties properties = new Properties();
properties.put(AzureConstants.AZURE_SAS, "sas");
properties.put(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, "account");
- String connectionString = Utils.getConnectionStringFromProperties(properties);
+ String connectionString = UtilsV8.getConnectionStringFromProperties(properties);
assertEquals(connectionString,
String.format("AccountName=%s;SharedAccessSignature=%s", "account", "sas"));
}
@@ -58,7 +61,7 @@ public class UtilsTest {
properties.put(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, "accessKey");
properties.put(AzureConstants.AZURE_STORAGE_ACCOUNT_KEY, "secretKey");
- String connectionString = Utils.getConnectionStringFromProperties(properties);
+ String connectionString = UtilsV8.getConnectionStringFromProperties(properties);
assertEquals(connectionString,
String.format("DefaultEndpointsProtocol=https;AccountName=%s;AccountKey=%s","accessKey","secretKey"));
}
@@ -72,10 +75,9 @@ public class UtilsTest {
properties.put(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, "accessKey");
properties.put(AzureConstants.AZURE_STORAGE_ACCOUNT_KEY, "secretKey");
- String connectionString = Utils.getConnectionStringFromProperties(properties);
+ String connectionString = UtilsV8.getConnectionStringFromProperties(properties);
assertEquals(connectionString,
String.format("BlobEndpoint=%s;SharedAccessSignature=%s", "endpoint", "sas"));
}
-
-}
+}
\ No newline at end of file
diff --git a/oak-blob-cloud-azure/src/test/resources/azure.properties b/oak-blob-cloud-azure/src/test/resources/azure.properties
index 5de17bf15e..c0b9f40d0f 100644
--- a/oak-blob-cloud-azure/src/test/resources/azure.properties
+++ b/oak-blob-cloud-azure/src/test/resources/azure.properties
@@ -46,4 +46,6 @@ proxyPort=
# service principals
tenantId=
clientId=
-clientSecret=
\ No newline at end of file
+clientSecret=
+
+azure.sdk.12.enabled=true
\ No newline at end of file
diff --git a/oak-it-osgi/src/test/config/org.apache.jackrabbit.oak.jcr.osgi.RepositoryManager.cfg b/oak-it-osgi/src/test/config/org.apache.jackrabbit.oak.jcr.osgi.RepositoryManager.cfg
index f8181f92fb..9de8f50d9c 100644
--- a/oak-it-osgi/src/test/config/org.apache.jackrabbit.oak.jcr.osgi.RepositoryManager.cfg
+++ b/oak-it-osgi/src/test/config/org.apache.jackrabbit.oak.jcr.osgi.RepositoryManager.cfg
@@ -1,15 +1,15 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#Empty config to trigger default setup
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#Empty config to trigger default setup
diff --git a/oak-it-osgi/src/test/config/org.apache.jackrabbit.oak.segment.SegmentNodeStoreService.cfg b/oak-it-osgi/src/test/config/org.apache.jackrabbit.oak.segment.SegmentNodeStoreService.cfg
index ccc6a4fde9..d05269419a 100644
--- a/oak-it-osgi/src/test/config/org.apache.jackrabbit.oak.segment.SegmentNodeStoreService.cfg
+++ b/oak-it-osgi/src/test/config/org.apache.jackrabbit.oak.segment.SegmentNodeStoreService.cfg
@@ -1,16 +1,16 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-name=Oak
-repository.home=target/tarmk
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+name=Oak
+repository.home=target/tarmk
diff --git a/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/datastore/AzureDataStoreFixture.java b/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/datastore/AzureDataStoreFixture.java
index 84bf6f2c82..800bbd5e64 100644
--- a/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/datastore/AzureDataStoreFixture.java
+++ b/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/datastore/AzureDataStoreFixture.java
@@ -28,6 +28,7 @@ import org.apache.jackrabbit.core.data.DataStoreException;
import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants;
import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureDataStore;
import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.Utils;
+import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v8.UtilsV8;
import org.apache.jackrabbit.oak.fixture.NodeStoreFixture;
import org.apache.jackrabbit.oak.jcr.binary.fixtures.nodestore.FixtureUtils;
import org.jetbrains.annotations.NotNull;
@@ -37,6 +38,7 @@ import org.slf4j.LoggerFactory;
import com.microsoft.azure.storage.StorageException;
import com.microsoft.azure.storage.blob.CloudBlobContainer;
+import com.azure.storage.blob.BlobContainerClient;
/**
* Fixture for AzureDataStore based on an azure.properties config file. It creates
@@ -64,7 +66,8 @@ public class AzureDataStoreFixture implements DataStoreFixture {
@Nullable
private final Properties azProps;
- private Map<DataStore, CloudBlobContainer> containers = new HashMap<>();
+ private Map<DataStore, Object> containers = new HashMap<>();
+ private static final String AZURE_SDK_12_ENABLED = "azure.sdk.12.enabled";
public AzureDataStoreFixture() {
azProps = FixtureUtils.loadDataStoreProperties("azure.config", "azure.properties", ".azure");
@@ -94,12 +97,22 @@ public class AzureDataStoreFixture implements DataStoreFixture {
String connectionString = Utils.getConnectionStringFromProperties(azProps);
try {
- CloudBlobContainer container = Utils.getBlobContainer(connectionString, containerName);
- container.createIfNotExists();
+ boolean useSDK12 = Boolean.parseBoolean(azProps.getProperty(AZURE_SDK_12_ENABLED, "false"));
+ Object container;
+
+ if (useSDK12) {
+ BlobContainerClient containerClient = Utils.getBlobContainer(connectionString, containerName, null, azProps);
+ containerClient.createIfNotExists();
+ container = containerClient;
+ } else {
+ CloudBlobContainer blobContainer = UtilsV8.getBlobContainer(connectionString, containerName);
+ blobContainer.createIfNotExists();
+ container = blobContainer;
+ }
// create new properties since azProps is shared for all created DataStores
Properties clonedAzProps = new Properties(azProps);
- clonedAzProps.setProperty(AzureConstants.AZURE_BLOB_CONTAINER_NAME, container.getName());
+ clonedAzProps.setProperty(AzureConstants.AZURE_BLOB_CONTAINER_NAME, containerName);
// setup Oak DS
AzureDataStore dataStore = new AzureDataStore();
@@ -126,15 +139,20 @@ public class AzureDataStoreFixture implements DataStoreFixture {
log.warn("Issue while disposing DataStore", e);
}
- CloudBlobContainer container = containers.get(dataStore);
+ Object container = containers.get(dataStore);
if (container != null) {
- log.info("Removing Azure test blob container {}", container.getName());
try {
- // For Azure, you can just delete the container and all
- // blobs it in will also be deleted
- container.delete();
- } catch (StorageException e) {
- log.warn("Unable to delete Azure Blob container {}", container.getName());
+ if (container instanceof CloudBlobContainer) {
+ CloudBlobContainer blobContainer = (CloudBlobContainer) container;
+ log.info("Removing Azure test blob container {}", blobContainer.getName());
+ blobContainer.delete();
+ } else if (container instanceof BlobContainerClient) {
+ BlobContainerClient containerClient = (BlobContainerClient) container;
+ log.info("Removing Azure test blob container {}", containerClient.getBlobContainerName());
+ containerClient.delete();
+ }
+ } catch (Exception e) {
+ log.warn("Unable to delete Azure Blob container", e);
}
containers.remove(dataStore);
diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/DataStoreUtils.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/DataStoreUtils.java
index 56502ea90f..c42992770f 100644
--- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/DataStoreUtils.java
+++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/DataStoreUtils.java
@@ -23,7 +23,7 @@ import com.amazonaws.services.s3.model.DeleteObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.amazonaws.services.s3.transfer.TransferManager;
-import com.microsoft.azure.storage.blob.CloudBlobContainer;
+import com.azure.storage.blob.BlobContainerClient;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.jackrabbit.core.data.DataStore;
@@ -157,7 +157,7 @@ public class DataStoreUtils {
log.warn("container name is null or blank, cannot initialize blob container");
return;
}
- CloudBlobContainer container = getCloudBlobContainer(config, containerName);
+ BlobContainerClient container = getBlobContainerClient(config, containerName);
if (container == null) {
log.warn("cannot delete the container as it is not initialized");
return;
@@ -171,8 +171,8 @@ public class DataStoreUtils {
}
@Nullable
- private static CloudBlobContainer getCloudBlobContainer(@NotNull Map<String, ?> config,
- @NotNull String containerName) throws DataStoreException {
+ private static BlobContainerClient getBlobContainerClient(@NotNull Map<String, ?> config,
+ @NotNull String containerName) throws DataStoreException {
final String azureConnectionString = (String) config.get(AzureConstants.AZURE_CONNECTION_STRING);
final String clientId = (String) config.get(AzureConstants.AZURE_CLIENT_ID);
final String clientSecret = (String) config.get(AzureConstants.AZURE_CLIENT_SECRET);
diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/fixture/DataStoreUtilsTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/fixture/DataStoreUtilsTest.java
index 56e6d1f17f..32e4f8ca3a 100644
--- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/fixture/DataStoreUtilsTest.java
+++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/fixture/DataStoreUtilsTest.java
@@ -23,8 +23,7 @@ import ch.qos.logback.classic.LoggerContext;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.Appender;
import ch.qos.logback.core.read.ListAppender;
-import com.microsoft.azure.storage.StorageException;
-import com.microsoft.azure.storage.blob.CloudBlobContainer;
+import com.azure.storage.blob.BlobContainerClient;
import org.apache.commons.lang3.StringUtils;
import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureBlobContainerProvider;
import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants;
@@ -39,6 +38,7 @@ import org.slf4j.LoggerFactory;
import java.net.URISyntaxException;
import java.security.InvalidKeyException;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
@@ -67,20 +67,20 @@ public class DataStoreUtilsTest {
private static final String AUTHENTICATE_VIA_ACCESS_KEY_LOG = "connecting to azure blob storage via access key";
private static final String AUTHENTICATE_VIA_SERVICE_PRINCIPALS_LOG = "connecting to azure blob storage via service principal credentials";
private static final String AUTHENTICATE_VIA_SAS_TOKEN_LOG = "connecting to azure blob storage via sas token";
- private static final String REFRESH_TOKEN_EXECUTOR_SHUTDOWN_LOG = "Refresh token executor service shutdown completed";
private static final String CONTAINER_DOES_NOT_EXIST_MESSAGE = "container [%s] doesn't exists";
private static final String CONTAINER_DELETED_MESSAGE = "container [%s] deleted";
+ private static final String DELETING_CONTAINER_MESSAGE = "deleting container [%s]";
- private CloudBlobContainer container;
+ private BlobContainerClient container;
@Before
- public void init() throws URISyntaxException, InvalidKeyException, StorageException {
- container = azuriteDockerRule.getContainer(CONTAINER_NAME);
+ public void init() throws URISyntaxException, InvalidKeyException {
+ container = azuriteDockerRule.getContainer(CONTAINER_NAME, String.format(AZURE_CONNECTION_STRING, AzuriteDockerRule.ACCOUNT_NAME, AzuriteDockerRule.ACCOUNT_KEY, azuriteDockerRule.getBlobEndpoint()));
assertTrue(container.exists());
}
@After
- public void cleanup() throws StorageException {
+ public void cleanup() {
if (container != null) {
container.deleteIfExists();
}
@@ -94,7 +94,8 @@ public class DataStoreUtilsTest {
DataStoreUtils.deleteAzureContainer(getConfigMap(azureConnectionString, null, null, null, null, null, null, null),
newContainerName);
validate(Arrays.asList(AUTHENTICATE_VIA_AZURE_CONNECTION_STRING_LOG,
- REFRESH_TOKEN_EXECUTOR_SHUTDOWN_LOG, String.format(CONTAINER_DOES_NOT_EXIST_MESSAGE, newContainerName)),
+ String.format(DELETING_CONTAINER_MESSAGE, newContainerName),
+ String.format(CONTAINER_DOES_NOT_EXIST_MESSAGE, newContainerName)),
Arrays.asList(AUTHENTICATE_VIA_SERVICE_PRINCIPALS_LOG, AUTHENTICATE_VIA_SAS_TOKEN_LOG, AUTHENTICATE_VIA_ACCESS_KEY_LOG),
getLogMessages(logAppender));
unsubscribe(logAppender);
@@ -107,7 +108,9 @@ public class DataStoreUtilsTest {
DataStoreUtils.deleteAzureContainer(getConfigMap(azureConnectionString, null, null, null, null, null, null, null),
CONTAINER_NAME);
- validate(Arrays.asList(AUTHENTICATE_VIA_AZURE_CONNECTION_STRING_LOG, REFRESH_TOKEN_EXECUTOR_SHUTDOWN_LOG,
+
+
+ validate(Arrays.asList(AUTHENTICATE_VIA_AZURE_CONNECTION_STRING_LOG,
String.format(CONTAINER_DELETED_MESSAGE, CONTAINER_NAME)),
Arrays.asList(AUTHENTICATE_VIA_SERVICE_PRINCIPALS_LOG, AUTHENTICATE_VIA_SAS_TOKEN_LOG, AUTHENTICATE_VIA_ACCESS_KEY_LOG),
getLogMessages(logAppender));
@@ -131,13 +134,14 @@ public class DataStoreUtilsTest {
final String newContainerName = getNewContainerName();
DataStoreUtils.deleteAzureContainer(getConfigMap(null, accountName, null, null, null, clientId, clientSecret, tenantId), newContainerName);
- validate(Arrays.asList(AUTHENTICATE_VIA_SERVICE_PRINCIPALS_LOG, REFRESH_TOKEN_EXECUTOR_SHUTDOWN_LOG, String.format(CONTAINER_DOES_NOT_EXIST_MESSAGE, newContainerName)),
+ validate(Arrays.asList(AUTHENTICATE_VIA_SERVICE_PRINCIPALS_LOG,
+ String.format(DELETING_CONTAINER_MESSAGE, newContainerName),
+ String.format(CONTAINER_DOES_NOT_EXIST_MESSAGE, newContainerName)),
Arrays.asList(AUTHENTICATE_VIA_AZURE_CONNECTION_STRING_LOG, AUTHENTICATE_VIA_SAS_TOKEN_LOG, AUTHENTICATE_VIA_ACCESS_KEY_LOG),
getLogMessages(logAppender));
unsubscribe(logAppender);
}
-
@Test
public void delete_container_service_principal() throws Exception {
final String accountName = getEnvironmentVariable(AZURE_ACCOUNT_NAME);
@@ -150,7 +154,7 @@ public class DataStoreUtilsTest {
Assume.assumeNotNull(clientSecret);
Assume.assumeNotNull(tenantId);
- CloudBlobContainer container;
+ BlobContainerClient container;
try (AzureBlobContainerProvider azureBlobContainerProvider = AzureBlobContainerProvider.Builder.builder(CONTAINER_NAME)
.withAccountName(accountName)
.withClientId(clientId)
@@ -165,7 +169,8 @@ public class DataStoreUtilsTest {
ListAppender<ILoggingEvent> logAppender = subscribeAppender();
DataStoreUtils.deleteAzureContainer(getConfigMap(null, accountName, null, null, null, clientId, clientSecret, tenantId), CONTAINER_NAME);
- validate(Arrays.asList(AUTHENTICATE_VIA_SERVICE_PRINCIPALS_LOG, REFRESH_TOKEN_EXECUTOR_SHUTDOWN_LOG,
+ validate(Arrays.asList(AUTHENTICATE_VIA_SERVICE_PRINCIPALS_LOG,
+ String.format(DELETING_CONTAINER_MESSAGE, CONTAINER_NAME),
String.format(CONTAINER_DELETED_MESSAGE, CONTAINER_NAME)),
Arrays.asList(AUTHENTICATE_VIA_AZURE_CONNECTION_STRING_LOG, AUTHENTICATE_VIA_SAS_TOKEN_LOG, AUTHENTICATE_VIA_ACCESS_KEY_LOG),
getLogMessages(logAppender));
@@ -180,7 +185,8 @@ public class DataStoreUtilsTest {
DataStoreUtils.deleteAzureContainer(getConfigMap(null, AzuriteDockerRule.ACCOUNT_NAME, AzuriteDockerRule.ACCOUNT_KEY, null, azuriteDockerRule.getBlobEndpoint(), null, null, null),
newContainerName);
- validate(Arrays.asList(AUTHENTICATE_VIA_ACCESS_KEY_LOG, REFRESH_TOKEN_EXECUTOR_SHUTDOWN_LOG,
+ validate(Arrays.asList(AUTHENTICATE_VIA_ACCESS_KEY_LOG,
+ String.format(DELETING_CONTAINER_MESSAGE, newContainerName),
String.format(CONTAINER_DOES_NOT_EXIST_MESSAGE, newContainerName)),
Arrays.asList(AUTHENTICATE_VIA_SERVICE_PRINCIPALS_LOG, AUTHENTICATE_VIA_SAS_TOKEN_LOG, AUTHENTICATE_VIA_SERVICE_PRINCIPALS_LOG),
getLogMessages(logAppender));
@@ -192,7 +198,8 @@ public class DataStoreUtilsTest {
ListAppender<ILoggingEvent> logAppender = subscribeAppender();
DataStoreUtils.deleteAzureContainer(getConfigMap(null, AzuriteDockerRule.ACCOUNT_NAME, AzuriteDockerRule.ACCOUNT_KEY, null, azuriteDockerRule.getBlobEndpoint(), null, null, null), CONTAINER_NAME);
- validate(Arrays.asList(AUTHENTICATE_VIA_ACCESS_KEY_LOG, REFRESH_TOKEN_EXECUTOR_SHUTDOWN_LOG,
+ validate(Arrays.asList(AUTHENTICATE_VIA_ACCESS_KEY_LOG,
+ String.format(DELETING_CONTAINER_MESSAGE, CONTAINER_NAME),
String.format(CONTAINER_DELETED_MESSAGE, CONTAINER_NAME)),
Arrays.asList(AUTHENTICATE_VIA_SERVICE_PRINCIPALS_LOG, AUTHENTICATE_VIA_SAS_TOKEN_LOG, AUTHENTICATE_VIA_SERVICE_PRINCIPALS_LOG),
getLogMessages(logAppender));
@@ -201,14 +208,23 @@ public class DataStoreUtilsTest {
}
private void validate(List<String> includedLogs, List<String> excludedLogs, Set<String> allLogs) {
- includedLogs.forEach(log -> assertTrue(allLogs.contains(log)));
- excludedLogs.forEach(log -> assertFalse(allLogs.contains(log)));
+
+ for (String log : includedLogs) {
+ if (!allLogs.contains(log)) {
+ System.out.println("Missing expected log: " + log);
+ }
+ }
+
+ includedLogs.forEach(log -> assertTrue("Missing expected log: " + log, allLogs.contains(log)));
+ excludedLogs.forEach(log -> assertFalse("Found unexpected log: " + log, allLogs.contains(log)));
}
private Set<String> getLogMessages(ListAppender<ILoggingEvent> logAppender) {
- return Optional.ofNullable(logAppender.list)
- .orElse(Collections.emptyList())
- .stream()
+ List<ILoggingEvent> events = Optional.ofNullable(logAppender.list)
+ .orElse(Collections.emptyList());
+ // Create a copy of the list to avoid concurrent modification
+ List<ILoggingEvent> eventsCopy = new ArrayList<>(events);
+ return eventsCopy.stream()
.map(ILoggingEvent::getFormattedMessage)
.filter(StringUtils::isNotBlank)
.collect(Collectors.toSet());
diff --git a/oak-run-elastic/pom.xml b/oak-run-elastic/pom.xml
index f35931caf9..08ecafa052 100644
--- a/oak-run-elastic/pom.xml
+++ b/oak-run-elastic/pom.xml
@@ -34,6 +34,7 @@
<jetty.version>8.2.0.v20160908</jetty.version>
<!--
Size History:
+ 120 MB : Added Azure SDK v12 with embeddings in oak-cloud-blob-azure
105 MB : Setting constraint to default oak-run jar post adding the build plugin to rename the fat jar with embedded dependencies as the default jar.
121 MB : add Elasticsearch Java client along with RHLC: the latter can be removed when the code can be fully migrated to use the new client
125 MB : shaded Guava
@@ -42,7 +43,7 @@
105 MB: Azure updates
107 MB: RDB/Tomcat (OAK-10752)
-->
- <max.jar.size>119000000</max.jar.size>
+ <max.jar.size>120000000</max.jar.size>
</properties>