You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by re...@apache.org on 2026/04/15 07:48:59 UTC
(jackrabbit-oak) branch OAK-12184 created (now 4bad83df94)
This is an automated email from the ASF dual-hosted git repository.
reschke pushed a change to branch OAK-12184
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
at 4bad83df94 OAK-12184: revert oak-blob-cloud-azure to version as of 1.90.0
This branch includes the following new commits:
new 4bad83df94 OAK-12184: revert oak-blob-cloud-azure to version as of 1.90.0
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(jackrabbit-oak) 01/01: OAK-12184: revert oak-blob-cloud-azure to version as of 1.90.0
Posted by re...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
reschke pushed a commit to branch OAK-12184
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit 4bad83df946847eed5ad6f95d227d2e5cc583889
Author: Julian Reschke <ju...@gmx.de>
AuthorDate: Wed Apr 15 08:48:28 2026 +0100
OAK-12184: revert oak-blob-cloud-azure to version as of 1.90.0
---
oak-blob-cloud-azure/pom.xml | 116 +-
.../blobstorage/AbstractAzureBlobStoreBackend.java | 45 -
.../blobstorage/AbstractAzureDataStoreService.java | 4 +-
.../blobstorage/AzureBlobContainerProvider.java | 285 ++-
.../azure/blobstorage/AzureBlobStoreBackend.java | 1063 ++++----
.../cloud/azure/blobstorage/AzureConstants.java | 82 +-
.../cloud/azure/blobstorage/AzureDataStore.java | 48 +-
.../azure/blobstorage/AzureDataStoreService.java | 3 +-
.../blobstorage/AzureHttpRequestLoggingPolicy.java | 68 -
.../cloud/azure/blobstorage/BlobSasHeaders.java | 211 --
.../oak/blob/cloud/azure/blobstorage/Utils.java | 137 +-
.../v8/AzureBlobContainerProviderV8.java | 346 ---
.../blobstorage/v8/AzureBlobStoreBackendV8.java | 1280 ----------
.../blob/cloud/azure/blobstorage/v8/UtilsV8.java | 167 --
.../AzureBlobContainerProviderTest.java | 889 -------
.../AzureBlobStoreBackendCompatibilityTest.java | 102 -
.../blobstorage/AzureBlobStoreBackendTest.java | 2413 +-----------------
.../azure/blobstorage/AzureConstantsTest.java | 243 --
.../AzureDataRecordAccessProviderCDNTest.java | 2 +-
.../AzureDataRecordAccessProviderIT.java | 10 +-
.../AzureDataRecordAccessProviderTest.java | 28 +-
.../azure/blobstorage/AzureDataStoreTest.java | 1529 ++++--------
.../azure/blobstorage/AzureDataStoreUtils.java | 29 +-
.../AzureHttpRequestLoggingPolicyTest.java | 390 ---
.../cloud/azure/blobstorage/AzuriteDockerRule.java | 20 +-
.../blob/cloud/azure/blobstorage/TestAzureDS.java | 11 +-
.../blobstorage/TestAzureDSWithSmallCache.java | 10 +-
.../azure/blobstorage/TestAzureDsCacheOff.java | 9 +-
.../blob/cloud/azure/blobstorage/UtilsTest.java | 224 +-
...eBlobContainerProviderV8AuthenticationTest.java | 305 ---
.../AzureBlobContainerProviderV8BuilderTest.java | 227 --
...reBlobContainerProviderV8ComprehensiveTest.java | 531 ----
...ContainerProviderV8ContainerOperationsTest.java | 372 ---
...BlobContainerProviderV8ErrorConditionsTest.java | 326 ---
...lobContainerProviderV8HeaderManagementTest.java | 304 ---
...reBlobContainerProviderV8SasGenerationTest.java | 308 ---
.../v8/AzureBlobContainerProviderV8Test.java | 905 -------
...BlobContainerProviderV8TokenManagementTest.java | 361 ---
.../v8/AzureBlobStoreBackendV8Test.java | 2554 --------------------
.../cloud/azure/blobstorage/v8/UtilsV8Test.java | 529 ----
.../src/test/resources/azure.properties | 4 +-
41 files changed, 1515 insertions(+), 14975 deletions(-)
diff --git a/oak-blob-cloud-azure/pom.xml b/oak-blob-cloud-azure/pom.xml
index 816de46fc7..137b78456f 100644
--- a/oak-blob-cloud-azure/pom.xml
+++ b/oak-blob-cloud-azure/pom.xml
@@ -38,51 +38,29 @@
<configuration>
<instructions combine.self="override">
<Import-Package>
- <!-- Optional system packages -->
+ com.fasterxml.jackson.annotation;resolution:=optional,
+ com.fasterxml.jackson.databind*;resolution:=optional,
+ com.fasterxml.jackson.dataformat.xml;resolution:=optional,
+ com.fasterxml.jackson.datatype*;resolution:=optional,
+ com.azure.identity.broker.implementation;resolution:=optional,
+ com.azure.xml;resolution:=optional,
+ com.microsoft.aad.msal4jextensions*;resolution:=optional,
com.sun.net.httpserver;resolution:=optional,
sun.misc;resolution:=optional,
net.jcip.annotations;resolution:=optional,
org.objectweb.asm;resolution:=optional,
+ com.nimbusds.langtag;resolution:=optional,
com.sun.jna*;resolution:=optional,
-
- <!-- Packages not needed (truly excluded) -->
+ io.netty.channel.epoll;resolution:=optional,
+ io.netty.handler.codec.*;resolution:=optional,
!com.google.*,
+ !org.checkerframework.*,
!android.os,
- !com.microsoft.aad.*,
-
- <!-- Exclude all embedded Azure packages -->
- !com.azure.*,
- !com.microsoft.azure.*,
-
- <!-- Exclude all embedded Netty packages -->
- !io.netty.*,
-
- <!-- Exclude all embedded Reactor packages -->
- !reactor.*,
- !org.reactivestreams,
-
- <!-- Exclude all embedded Nimbusds/OAuth packages -->
- !com.nimbusds.*,
-
- <!-- Exclude all embedded JSON packages -->
- !net.minidev.*,
-
- <!-- Exclude Stax2 (embedded) -->
- !org.codehaus.stax2.*,
-
- <!-- Exclude all embedded Jackson packages -->
- !com.fasterxml.jackson.*,
-
- <!-- Import everything else (Oak, Guava, Commons, etc.) -->
*
</Import-Package>
<Export-Package>
- !*
+ org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage
</Export-Package>
- <Private-Package>
- org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.*,
- io.netty.util.internal.shaded.*
- </Private-Package>
<DynamicImport-Package>sun.io</DynamicImport-Package>
<Embed-Dependency>
azure-storage,
@@ -90,10 +68,7 @@
azure-core,
azure-identity,
azure-json,
- azure-xml,
- azure-storage-blob,
- azure-storage-common,
- azure-storage-internal-avro,
+ guava,
jsr305,
reactive-streams,
msal4j,
@@ -106,9 +81,7 @@
json-smart,
content-type,
accessors-smart,
- nimbus-jose-jwt,
- jackson-*,
- stax2-api
+ nimbus-jose-jwt
</Embed-Dependency>
<!-- needed to override value from oak-parent; can be removed when OAK-6741 is resolved -->
<_plugin />
@@ -154,12 +127,18 @@
<dependency>
<groupId>javax.jcr</groupId>
<artifactId>jcr</artifactId>
+ <version>2.0</version>
</dependency>
<dependency>
<groupId>org.apache.jackrabbit</groupId>
<artifactId>jackrabbit-jcr-commons</artifactId>
<version>${jackrabbit.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>jackrabbit-data</artifactId>
+ <version>${jackrabbit.version}</version>
+ </dependency>
<!-- General utility libraries -->
<dependency>
@@ -174,11 +153,6 @@
<artifactId>oak-commons</artifactId>
<version>${project.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.jackrabbit</groupId>
- <artifactId>oak-core-spi</artifactId>
- <version>${project.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.jackrabbit</groupId>
@@ -196,14 +170,6 @@
<groupId>com.microsoft.azure</groupId>
<artifactId>azure-storage</artifactId>
</dependency>
- <dependency>
- <groupId>com.azure</groupId>
- <artifactId>azure-storage-blob</artifactId>
- </dependency>
- <dependency>
- <groupId>com.azure</groupId>
- <artifactId>azure-storage-common</artifactId>
- </dependency>
<dependency>
<groupId>com.microsoft.azure</groupId>
<artifactId>azure-keyvault-core</artifactId>
@@ -222,16 +188,6 @@
<groupId>com.azure</groupId>
<artifactId>azure-json</artifactId>
</dependency>
- <dependency>
- <groupId>com.azure</groupId>
- <artifactId>azure-xml</artifactId>
- </dependency>
- <!-- Transitive dependency of jackson-dataformat-xml, required for OSGi bundle resolution -->
- <dependency>
- <groupId>org.codehaus.woodstox</groupId>
- <artifactId>stax2-api</artifactId>
- <version>4.2.2</version>
- </dependency>
<dependency>
<groupId>org.reactivestreams</groupId>
<artifactId>reactive-streams</artifactId>
@@ -332,34 +288,26 @@
<artifactId>nimbus-jose-jwt</artifactId>
</dependency>
- <!-- Jackson dependencies - explicitly use version 2.17.3 -->
+ <!-- Azure Guava dependency -->
<dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>33.1.0-jre</version>
</dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.dataformat</groupId>
- <artifactId>jackson-dataformat-xml</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.datatype</groupId>
- <artifactId>jackson-datatype-jsr310</artifactId>
- </dependency>
-
<dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
+ <version>3.0.2</version>
</dependency>
<!-- Test dependencies -->
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>jackrabbit-data</artifactId>
+ <version>${jackrabbit.version}</version>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.apache.jackrabbit</groupId>
<artifactId>oak-blob-plugins</artifactId>
diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AbstractAzureBlobStoreBackend.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AbstractAzureBlobStoreBackend.java
deleted file mode 100644
index 0796434342..0000000000
--- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AbstractAzureBlobStoreBackend.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;
-
-import org.apache.jackrabbit.oak.spi.blob.data.DataIdentifier;
-import org.apache.jackrabbit.oak.spi.blob.data.DataRecord;
-import org.apache.jackrabbit.oak.spi.blob.data.DataStoreException;
-import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordDownloadOptions;
-import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUpload;
-import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadException;
-import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadOptions;
-import org.apache.jackrabbit.oak.spi.blob.AbstractSharedBackend;
-import org.jetbrains.annotations.NotNull;
-
-import java.net.URI;
-import java.util.Properties;
-
-
-public abstract class AbstractAzureBlobStoreBackend extends AbstractSharedBackend {
-
- protected abstract DataRecordUpload initiateHttpUpload(long maxUploadSizeInBytes, int maxNumberOfURIs, @NotNull final DataRecordUploadOptions options);
- protected abstract DataRecord completeHttpUpload(@NotNull String uploadTokenStr) throws DataRecordUploadException, DataStoreException;
- protected abstract void setHttpDownloadURIExpirySeconds(int seconds);
- protected abstract void setHttpUploadURIExpirySeconds(int seconds);
- protected abstract void setHttpDownloadURICacheSize(int maxSize);
- protected abstract URI createHttpDownloadURI(@NotNull DataIdentifier identifier, @NotNull DataRecordDownloadOptions downloadOptions);
- public abstract void setProperties(final Properties properties);
-
-}
diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AbstractAzureDataStoreService.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AbstractAzureDataStoreService.java
index d15f7505c6..dce503f979 100644
--- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AbstractAzureDataStoreService.java
+++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AbstractAzureDataStoreService.java
@@ -24,8 +24,8 @@ import java.util.Hashtable;
import java.util.Map;
import java.util.Properties;
-import org.apache.jackrabbit.oak.spi.blob.data.DataStore;
-import org.apache.jackrabbit.oak.spi.blob.data.DataStoreException;
+import org.apache.jackrabbit.core.data.DataStore;
+import org.apache.jackrabbit.core.data.DataStoreException;
import org.apache.jackrabbit.oak.plugins.blob.AbstractSharedCachingDataStore;
import org.apache.jackrabbit.oak.plugins.blob.datastore.AbstractDataStoreService;
import org.osgi.framework.Constants;
diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobContainerProvider.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobContainerProvider.java
index 2522747811..01522248b0 100644
--- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobContainerProvider.java
+++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobContainerProvider.java
@@ -18,33 +18,49 @@
*/
package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;
+import com.azure.core.credential.AccessToken;
+import com.azure.core.credential.TokenRequestContext;
import com.azure.identity.ClientSecretCredential;
import com.azure.identity.ClientSecretCredentialBuilder;
-import com.azure.storage.blob.BlobContainerClient;
-import com.azure.storage.blob.BlobContainerClientBuilder;
-import com.azure.storage.blob.BlobServiceClient;
-import com.azure.storage.blob.BlobServiceClientBuilder;
-import com.azure.storage.blob.models.UserDelegationKey;
-import com.azure.storage.blob.sas.BlobSasPermission;
-import com.azure.storage.blob.sas.BlobServiceSasSignatureValues;
-import com.azure.storage.blob.specialized.BlockBlobClient;
-import com.azure.storage.common.policy.RequestRetryOptions;
+import com.microsoft.azure.storage.CloudStorageAccount;
+import com.microsoft.azure.storage.StorageCredentialsToken;
+import com.microsoft.azure.storage.StorageException;
+import com.microsoft.azure.storage.UserDelegationKey;
+import com.microsoft.azure.storage.blob.BlobRequestOptions;
+import com.microsoft.azure.storage.blob.CloudBlobClient;
+import com.microsoft.azure.storage.blob.CloudBlobContainer;
+import com.microsoft.azure.storage.blob.CloudBlockBlob;
+import com.microsoft.azure.storage.blob.SharedAccessBlobHeaders;
+import com.microsoft.azure.storage.blob.SharedAccessBlobPermissions;
+import com.microsoft.azure.storage.blob.SharedAccessBlobPolicy;
import org.apache.commons.lang3.StringUtils;
-import org.apache.jackrabbit.oak.spi.blob.data.DataStoreException;
+import org.apache.jackrabbit.core.data.DataStoreException;
+import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.Closeable;
import java.net.URISyntaxException;
import java.security.InvalidKeyException;
+import java.time.Instant;
+import java.time.LocalDateTime;
import java.time.OffsetDateTime;
-import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.Date;
+import java.util.EnumSet;
+import java.util.Objects;
+import java.util.Optional;
import java.util.Properties;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
-public class AzureBlobContainerProvider {
+public class AzureBlobContainerProvider implements Closeable {
private static final Logger log = LoggerFactory.getLogger(AzureBlobContainerProvider.class);
private static final String DEFAULT_ENDPOINT_SUFFIX = "core.windows.net";
+ private static final String AZURE_DEFAULT_SCOPE = "https://storage.azure.com/.default";
private final String azureConnectionString;
private final String accountName;
private final String containerName;
@@ -54,6 +70,12 @@ public class AzureBlobContainerProvider {
private final String tenantId;
private final String clientId;
private final String clientSecret;
+ private ClientSecretCredential clientSecretCredential;
+ private AccessToken accessToken;
+ private StorageCredentialsToken storageCredentialsToken;
+ private static final long TOKEN_REFRESHER_INITIAL_DELAY = 45L;
+ private static final long TOKEN_REFRESHER_DELAY = 1L;
+ private final ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
private AzureBlobContainerProvider(Builder builder) {
this.azureConnectionString = builder.azureConnectionString;
@@ -67,6 +89,7 @@ public class AzureBlobContainerProvider {
this.clientSecret = builder.clientSecret;
}
+
public static class Builder {
private final String containerName;
@@ -148,154 +171,176 @@ public class AzureBlobContainerProvider {
return containerName;
}
- public String getAzureConnectionString() {
- return azureConnectionString;
- }
-
@NotNull
- public BlobContainerClient getBlobContainer() throws DataStoreException {
- return this.getBlobContainer(null, new Properties());
+ public CloudBlobContainer getBlobContainer() throws DataStoreException {
+ return this.getBlobContainer(null);
}
@NotNull
- public BlobContainerClient getBlobContainer(@Nullable RequestRetryOptions retryOptions, Properties properties) throws DataStoreException {
+ public CloudBlobContainer getBlobContainer(@Nullable BlobRequestOptions blobRequestOptions) throws DataStoreException {
// connection string will be given preference over service principals / sas / account key
if (StringUtils.isNotBlank(azureConnectionString)) {
log.debug("connecting to azure blob storage via azureConnectionString");
- return Utils.getBlobContainerFromConnectionString(getAzureConnectionString(), containerName);
+ return Utils.getBlobContainer(azureConnectionString, containerName, blobRequestOptions);
} else if (authenticateViaServicePrincipal()) {
log.debug("connecting to azure blob storage via service principal credentials");
- return getBlobContainerFromServicePrincipals(accountName, retryOptions);
+ return getBlobContainerFromServicePrincipals(blobRequestOptions);
} else if (StringUtils.isNotBlank(sasToken)) {
log.debug("connecting to azure blob storage via sas token");
final String connectionStringWithSasToken = Utils.getConnectionStringForSas(sasToken, blobEndpoint, accountName);
- return Utils.getBlobContainer(connectionStringWithSasToken, containerName, retryOptions, properties);
+ return Utils.getBlobContainer(connectionStringWithSasToken, containerName, blobRequestOptions);
}
log.debug("connecting to azure blob storage via access key");
final String connectionStringWithAccountKey = Utils.getConnectionString(accountName, accountKey, blobEndpoint);
- return Utils.getBlobContainer(connectionStringWithAccountKey, containerName, retryOptions, properties);
+ return Utils.getBlobContainer(connectionStringWithAccountKey, containerName, blobRequestOptions);
}
@NotNull
- public String generateSharedAccessSignature(RequestRetryOptions retryOptions,
- String key,
- BlobSasPermission blobSasPermissions,
- int expirySeconds,
- Properties properties) throws DataStoreException, URISyntaxException, InvalidKeyException {
- return generateSharedAccessSignature(retryOptions, key, blobSasPermissions, expirySeconds, properties, null);
+ private CloudBlobContainer getBlobContainerFromServicePrincipals(@Nullable BlobRequestOptions blobRequestOptions) throws DataStoreException {
+ StorageCredentialsToken storageCredentialsToken = getStorageCredentials();
+ try {
+ CloudStorageAccount cloud = new CloudStorageAccount(storageCredentialsToken, true, DEFAULT_ENDPOINT_SUFFIX, accountName);
+ CloudBlobClient cloudBlobClient = cloud.createCloudBlobClient();
+ if (blobRequestOptions != null) {
+ cloudBlobClient.setDefaultRequestOptions(blobRequestOptions);
+ }
+ return cloudBlobClient.getContainerReference(containerName);
+ } catch (URISyntaxException | StorageException e) {
+ throw new DataStoreException(e);
+ }
}
- /**
- * Generates a shared access signature (SAS) for the specified blob with optional headers.
- * This is the Azure SDK 12 equivalent of the V8 method that accepted {@code SharedAccessBlobHeaders}.
- *
- * @param retryOptions retry options for the request
- * @param key the blob key
- * @param blobSasPermissions the permissions for the SAS
- * @param expirySeconds the number of seconds until the SAS expires
- * @param properties additional properties
- * @param optionalHeaders optional headers to include in the SAS (can be null)
- * @return the SAS query string
- * @throws DataStoreException if an error occurs
- * @throws URISyntaxException if the URI is invalid
- * @throws InvalidKeyException if the key is invalid
- */
@NotNull
- public String generateSharedAccessSignature(RequestRetryOptions retryOptions,
- String key,
- BlobSasPermission blobSasPermissions,
- int expirySeconds,
- Properties properties,
- @Nullable BlobSasHeaders optionalHeaders) throws DataStoreException, URISyntaxException, InvalidKeyException {
+ private StorageCredentialsToken getStorageCredentials() {
+ boolean isAccessTokenGenerated = false;
+ /* generate access token, the same token will be used for subsequent access
+ * generated token is valid for 1 hour only and will be refreshed in background
+ * */
+ if (accessToken == null) {
+ clientSecretCredential = new ClientSecretCredentialBuilder()
+ .clientId(clientId)
+ .clientSecret(clientSecret)
+ .tenantId(tenantId)
+ .build();
+ accessToken = clientSecretCredential.getTokenSync(new TokenRequestContext().addScopes(AZURE_DEFAULT_SCOPE));
+ if (accessToken == null || StringUtils.isBlank(accessToken.getToken())) {
+ log.error("Access token is null or empty");
+ throw new IllegalArgumentException("Could not connect to azure storage, access token is null or empty");
+ }
+ storageCredentialsToken = new StorageCredentialsToken(accountName, accessToken.getToken());
+ isAccessTokenGenerated = true;
+ }
- OffsetDateTime expiry = OffsetDateTime.now().plusSeconds(expirySeconds);
- BlobServiceSasSignatureValues serviceSasSignatureValues = new BlobServiceSasSignatureValues(expiry, blobSasPermissions);
+ Objects.requireNonNull(storageCredentialsToken, "storage credentials token cannot be null");
- // Apply headers if provided
- if (optionalHeaders != null) {
- optionalHeaders.applyTo(serviceSasSignatureValues);
+ // start refresh token executor only when the access token is first generated
+ if (isAccessTokenGenerated) {
+ log.info("starting refresh token task at: {}", OffsetDateTime.now());
+ TokenRefresher tokenRefresher = new TokenRefresher();
+ executorService.scheduleWithFixedDelay(tokenRefresher, TOKEN_REFRESHER_INITIAL_DELAY, TOKEN_REFRESHER_DELAY, TimeUnit.MINUTES);
}
+ return storageCredentialsToken;
+ }
+
+ @NotNull
+ public String generateSharedAccessSignature(BlobRequestOptions requestOptions,
+ String key,
+ EnumSet<SharedAccessBlobPermissions> permissions,
+ int expirySeconds,
+ SharedAccessBlobHeaders optionalHeaders) throws DataStoreException, URISyntaxException, StorageException, InvalidKeyException {
+ SharedAccessBlobPolicy policy = new SharedAccessBlobPolicy();
+ Date expiry = Date.from(Instant.now().plusSeconds(expirySeconds));
+ policy.setSharedAccessExpiryTime(expiry);
+ policy.setPermissions(permissions);
- BlockBlobClient blob = getBlobContainer(retryOptions, properties).getBlobClient(key).getBlockBlobClient();
+ CloudBlockBlob blob = getBlobContainer(requestOptions).getBlockBlobReference(key);
if (authenticateViaServicePrincipal()) {
- return generateUserDelegationKeySignedSas(blob, serviceSasSignatureValues, expiry);
+ return generateUserDelegationKeySignedSas(blob, policy, optionalHeaders, expiry);
}
- return generateSas(blob, serviceSasSignatureValues);
+ return generateSas(blob, policy, optionalHeaders);
}
@NotNull
- public String generateUserDelegationKeySignedSas(BlockBlobClient blobClient,
- BlobServiceSasSignatureValues serviceSasSignatureValues,
- OffsetDateTime expiryTime) {
-
- AzureHttpRequestLoggingPolicy loggingPolicy = new AzureHttpRequestLoggingPolicy();
-
- String endpoint = getEndpointUrl(accountName, blobEndpoint);
- BlobServiceClient blobServiceClient = new BlobServiceClientBuilder()
- .endpoint(endpoint)
- .credential(getClientSecretCredential())
- .addPolicy(loggingPolicy)
- .buildClient();
- OffsetDateTime startTime = OffsetDateTime.now(ZoneOffset.UTC);
- UserDelegationKey userDelegationKey = blobServiceClient.getUserDelegationKey(startTime, expiryTime);
- return blobClient.generateUserDelegationSas(serviceSasSignatureValues, userDelegationKey);
- }
-
- private boolean authenticateViaServicePrincipal() {
- return StringUtils.isBlank(azureConnectionString) &&
- StringUtils.isNoneBlank(accountName, tenantId, clientId, clientSecret);
+ private String generateUserDelegationKeySignedSas(CloudBlockBlob blob,
+ SharedAccessBlobPolicy policy,
+ SharedAccessBlobHeaders optionalHeaders,
+ Date expiry) throws StorageException {
+ fillEmptyHeaders(optionalHeaders);
+ UserDelegationKey userDelegationKey = blob.getServiceClient().getUserDelegationKey(Date.from(Instant.now().minusSeconds(900)),
+ expiry);
+ return optionalHeaders == null ? blob.generateUserDelegationSharedAccessSignature(userDelegationKey, policy) :
+ blob.generateUserDelegationSharedAccessSignature(userDelegationKey, policy, optionalHeaders, null, null);
}
- private ClientSecretCredential getClientSecretCredential() {
- return new ClientSecretCredentialBuilder()
- .clientId(clientId)
- .clientSecret(clientSecret)
- .tenantId(tenantId)
- .build();
+ /* set empty headers as blank string due to a bug in Azure SDK
+ * Azure SDK considers null headers as 'null' string which corrupts the string to sign and generates an invalid
+ * sas token
+ * */
+ private void fillEmptyHeaders(SharedAccessBlobHeaders sharedAccessBlobHeaders) {
+ final String EMPTY_STRING = "";
+ Optional.ofNullable(sharedAccessBlobHeaders)
+ .ifPresent(headers -> {
+ if (StringUtils.isBlank(headers.getCacheControl())) {
+ headers.setCacheControl(EMPTY_STRING);
+ }
+ if (StringUtils.isBlank(headers.getContentDisposition())) {
+ headers.setContentDisposition(EMPTY_STRING);
+ }
+ if (StringUtils.isBlank(headers.getContentEncoding())) {
+ headers.setContentEncoding(EMPTY_STRING);
+ }
+ if (StringUtils.isBlank(headers.getContentLanguage())) {
+ headers.setContentLanguage(EMPTY_STRING);
+ }
+ if (StringUtils.isBlank(headers.getContentType())) {
+ headers.setContentType(EMPTY_STRING);
+ }
+ });
}
@NotNull
- private BlobContainerClient getBlobContainerFromServicePrincipals(String accountName, RequestRetryOptions retryOptions) {
- ClientSecretCredential clientSecretCredential = getClientSecretCredential();
- AzureHttpRequestLoggingPolicy loggingPolicy = new AzureHttpRequestLoggingPolicy();
-
- String endpoint = getEndpointUrl(accountName, blobEndpoint);
- return new BlobContainerClientBuilder()
- .endpoint(endpoint)
- .containerName(containerName)
- .credential(clientSecretCredential)
- .retryOptions(retryOptions)
- .addPolicy(loggingPolicy)
- .buildClient();
+ private String generateSas(CloudBlockBlob blob,
+ SharedAccessBlobPolicy policy,
+ SharedAccessBlobHeaders optionalHeaders) throws InvalidKeyException, StorageException {
+ return optionalHeaders == null ? blob.generateSharedAccessSignature(policy, null) :
+ blob.generateSharedAccessSignature(policy,
+ optionalHeaders, null, null, null, true);
}
- @NotNull
- private String generateSas(BlockBlobClient blob,
- BlobServiceSasSignatureValues blobServiceSasSignatureValues) {
- return blob.generateSas(blobServiceSasSignatureValues, null);
+ private boolean authenticateViaServicePrincipal() {
+ return StringUtils.isBlank(azureConnectionString) &&
+ StringUtils.isNoneBlank(accountName, tenantId, clientId, clientSecret);
}
- /**
- * Constructs the Azure Storage endpoint URL.
- * If a custom blobEndpoint is configured, it will be used.
- * Otherwise, constructs the default endpoint using the account name.
- *
- * @param accountName the storage account name
- * @param customBlobEndpoint optional custom blob endpoint (can be null or empty)
- * @return the endpoint URL to use
- */
- @NotNull
- private static String getEndpointUrl(String accountName, String customBlobEndpoint) {
- if (StringUtils.isNotBlank(customBlobEndpoint)) {
- // Use custom endpoint (e.g., for private endpoints)
- // Ensure it starts with https:// if not already present
- if (!customBlobEndpoint.startsWith("http://") && !customBlobEndpoint.startsWith("https://")) {
- return "https://" + customBlobEndpoint;
+ private class TokenRefresher implements Runnable {
+ @Override
+ public void run() {
+ try {
+ log.debug("Checking for azure access token expiry at: {}", LocalDateTime.now());
+ OffsetDateTime tokenExpiryThreshold = OffsetDateTime.now().plusMinutes(5);
+ if (accessToken.getExpiresAt() != null && accessToken.getExpiresAt().isBefore(tokenExpiryThreshold)) {
+ log.info("Access token is about to expire (5 minutes or less) at: {}. New access token will be generated",
+ accessToken.getExpiresAt().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME));
+ AccessToken newToken = clientSecretCredential.getTokenSync(new TokenRequestContext().addScopes(AZURE_DEFAULT_SCOPE));
+ log.info("New azure access token generated at: {}", LocalDateTime.now());
+ if (newToken == null || StringUtils.isBlank(newToken.getToken())) {
+ log.error("New access token is null or empty");
+ return;
+ }
+ // update access token with newly generated token
+ accessToken = newToken;
+ storageCredentialsToken.updateToken(accessToken.getToken());
+ }
+ } catch (Exception e) {
+ log.error("Error while acquiring new access token: ", e);
}
- return customBlobEndpoint;
}
- // Default public endpoint
- return String.format("https://%s.blob.%s", accountName, DEFAULT_ENDPOINT_SUFFIX);
}
-}
\ No newline at end of file
+
+ @Override
+ public void close() {
+ new ExecutorCloser(executorService).close();
+ log.info("Refresh token executor service shutdown completed");
+ }
+}
diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
index ac047fd52d..5beb537614 100644
--- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
+++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
@@ -18,107 +18,110 @@
*/
package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;
-import com.azure.core.http.rest.Response;
-import com.azure.storage.blob.BlobClient;
-import com.azure.storage.blob.BlobContainerClient;
-import com.azure.storage.blob.models.BlobItem;
-import com.azure.storage.blob.models.BlobProperties;
-import com.azure.storage.blob.models.BlobStorageException;
-import com.azure.storage.blob.models.Block;
-import com.azure.storage.blob.models.BlockBlobItem;
-import com.azure.storage.blob.models.BlockListType;
-import com.azure.storage.blob.models.ListBlobsOptions;
-import com.azure.storage.blob.models.ParallelTransferOptions;
-import com.azure.storage.blob.options.BlobUploadFromFileOptions;
-import com.azure.storage.blob.sas.BlobSasPermission;
-import com.azure.storage.blob.specialized.BlockBlobClient;
-import com.azure.storage.common.policy.RequestRetryOptions;
-import com.microsoft.azure.storage.RetryPolicy;
-import org.apache.commons.io.IOUtils;
-import org.apache.jackrabbit.oak.spi.blob.data.DataIdentifier;
-import org.apache.jackrabbit.oak.spi.blob.data.DataRecord;
-import org.apache.jackrabbit.oak.spi.blob.data.DataStoreException;
-import org.apache.jackrabbit.oak.cache.api.Cache;
-import org.apache.jackrabbit.oak.cache.api.CacheBuilder;
-import org.apache.jackrabbit.oak.commons.PropertiesUtil;
-import org.apache.jackrabbit.oak.commons.conditions.Validate;
-import org.apache.jackrabbit.oak.spi.blob.AbstractDataRecord;
-import org.apache.jackrabbit.oak.spi.blob.AbstractSharedBackend;
-import org.apache.jackrabbit.util.Base64;
-import org.jetbrains.annotations.NotNull;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import static java.lang.Thread.currentThread;
import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
-import java.io.UncheckedIOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.StandardCopyOption;
import java.security.InvalidKeyException;
-import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.Iterator;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Properties;
+import java.util.Queue;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
-
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.stream.Collectors;
+import java.util.function.Function;
import org.apache.commons.lang3.StringUtils;
-
-import org.apache.jackrabbit.oak.commons.time.Stopwatch;
+import org.apache.jackrabbit.guava.common.cache.Cache;
+import org.apache.jackrabbit.guava.common.cache.CacheBuilder;
+import org.apache.jackrabbit.oak.commons.collections.AbstractIterator;
+import com.microsoft.azure.storage.AccessCondition;
+import com.microsoft.azure.storage.LocationMode;
+import com.microsoft.azure.storage.ResultContinuation;
+import com.microsoft.azure.storage.ResultSegment;
+import com.microsoft.azure.storage.RetryPolicy;
+import com.microsoft.azure.storage.StorageException;
+import com.microsoft.azure.storage.blob.BlobListingDetails;
+import com.microsoft.azure.storage.blob.BlobRequestOptions;
+import com.microsoft.azure.storage.blob.BlockEntry;
+import com.microsoft.azure.storage.blob.BlockListingFilter;
+import com.microsoft.azure.storage.blob.CloudBlob;
+import com.microsoft.azure.storage.blob.CloudBlobContainer;
+import com.microsoft.azure.storage.blob.CloudBlobDirectory;
+import com.microsoft.azure.storage.blob.CloudBlockBlob;
+import com.microsoft.azure.storage.blob.CopyStatus;
+import com.microsoft.azure.storage.blob.ListBlobItem;
+import com.microsoft.azure.storage.blob.SharedAccessBlobHeaders;
+import com.microsoft.azure.storage.blob.SharedAccessBlobPermissions;
+import org.apache.commons.io.IOUtils;
+import org.apache.jackrabbit.core.data.DataIdentifier;
+import org.apache.jackrabbit.core.data.DataRecord;
+import org.apache.jackrabbit.core.data.DataStoreException;
+import org.apache.jackrabbit.oak.commons.PropertiesUtil;
import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordDownloadOptions;
import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUpload;
import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadException;
import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadOptions;
import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadToken;
+import org.apache.jackrabbit.oak.spi.blob.AbstractDataRecord;
+import org.apache.jackrabbit.oak.spi.blob.AbstractSharedBackend;
+import org.apache.jackrabbit.util.Base64;
+import org.jetbrains.annotations.NotNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import static java.nio.file.StandardOpenOption.DELETE_ON_CLOSE;
-import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT;
-import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_LAST_MODIFIED_KEY;
-import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MAX_ALLOWABLE_UPLOAD_URIS;
-import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MAX_BINARY_UPLOAD_SIZE;
-import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT;
-import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MAX_MULTIPART_UPLOAD_PART_SIZE;
-import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MAX_SINGLE_PUT_UPLOAD_SIZE;
-import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_META_KEY_PREFIX;
-import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE;
-import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_REF_KEY;
-
-
-public class AzureBlobStoreBackend extends AbstractAzureBlobStoreBackend {
+public class AzureBlobStoreBackend extends AbstractSharedBackend {
private static final Logger LOG = LoggerFactory.getLogger(AzureBlobStoreBackend.class);
private static final Logger LOG_STREAMS_DOWNLOAD = LoggerFactory.getLogger("oak.datastore.download.streams");
private static final Logger LOG_STREAMS_UPLOAD = LoggerFactory.getLogger("oak.datastore.upload.streams");
+ private static final String META_DIR_NAME = "META";
+ private static final String META_KEY_PREFIX = META_DIR_NAME + "/";
+
+ private static final String REF_KEY = "reference.key";
+ private static final String LAST_MODIFIED_KEY = "lastModified";
+
+ private static final long BUFFERED_STREAM_THRESHOLD = 1024 * 1024;
+ static final long MIN_MULTIPART_UPLOAD_PART_SIZE = 1024 * 1024 * 10; // 10MB
+ static final long MAX_MULTIPART_UPLOAD_PART_SIZE = 1024 * 1024 * 100; // 100MB
+ static final long MAX_SINGLE_PUT_UPLOAD_SIZE = 1024 * 1024 * 256; // 256MB, Azure limit
+ static final long MAX_BINARY_UPLOAD_SIZE = (long) Math.floor(1024L * 1024L * 1024L * 1024L * 4.75); // 4.75TB, Azure limit
+ private static final int MAX_ALLOWABLE_UPLOAD_URIS = 50000; // Azure limit
+ private static final int MAX_UNIQUE_RECORD_TRIES = 10;
+ private static final int DEFAULT_CONCURRENT_REQUEST_COUNT = 2;
+ private static final int MAX_CONCURRENT_REQUEST_COUNT = 50;
+
private Properties properties;
private AzureBlobContainerProvider azureBlobContainerProvider;
- private int concurrentRequestCount = AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT;
- private RequestRetryOptions retryOptions;
+ private int concurrentRequestCount = DEFAULT_CONCURRENT_REQUEST_COUNT;
+ private RetryPolicy retryPolicy;
private Integer requestTimeout;
private int httpDownloadURIExpirySeconds = 0; // disabled by default
private int httpUploadURIExpirySeconds = 0; // disabled by default
private String uploadDomainOverride = null;
private String downloadDomainOverride = null;
- private boolean presignedDownloadURIVerifyExists = true;
+ private boolean createBlobContainer = true;
+ private boolean presignedDownloadURIVerifyExists = true;
+ private boolean enableSecondaryLocation = AzureConstants.AZURE_BLOB_ENABLE_SECONDARY_LOCATION_DEFAULT;
private Cache<String, URI> httpDownloadURICache;
@@ -127,60 +130,88 @@ public class AzureBlobStoreBackend extends AbstractAzureBlobStoreBackend {
public void setProperties(final Properties properties) {
this.properties = properties;
}
- private final AtomicReference<BlobContainerClient> azureContainerReference = new AtomicReference<>();
- protected BlobContainerClient getAzureContainer() throws DataStoreException {
- azureContainerReference.compareAndSet(null, azureBlobContainerProvider.getBlobContainer(retryOptions, properties));
- return azureContainerReference.get();
+ private volatile CloudBlobContainer azureContainer = null;
+
+ protected CloudBlobContainer getAzureContainer() throws DataStoreException {
+ if (azureContainer == null) {
+ synchronized (this) {
+ if (azureContainer == null) {
+ azureContainer = azureBlobContainerProvider.getBlobContainer(getBlobRequestOptions());
+ }
+ }
+ }
+ return azureContainer;
+ }
+
+ @NotNull
+ protected BlobRequestOptions getBlobRequestOptions() {
+ BlobRequestOptions requestOptions = new BlobRequestOptions();
+ if (null != retryPolicy) {
+ requestOptions.setRetryPolicyFactory(retryPolicy);
+ }
+ if (null != requestTimeout) {
+ requestOptions.setTimeoutIntervalInMs(requestTimeout);
+ }
+ requestOptions.setConcurrentRequestCount(concurrentRequestCount);
+ if (enableSecondaryLocation) {
+ requestOptions.setLocationMode(LocationMode.PRIMARY_THEN_SECONDARY);
+ }
+ return requestOptions;
}
@Override
public void init() throws DataStoreException {
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
- Stopwatch stopwatch = Stopwatch.createStarted();
+ long start = System.currentTimeMillis();
try {
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
LOG.debug("Started backend initialization");
- if (properties == null) {
+ if (null == properties) {
try {
properties = Utils.readConfig(Utils.DEFAULT_CONFIG_FILE);
- } catch (IOException e) {
+ }
+ catch (IOException e) {
throw new DataStoreException("Unable to initialize Azure Data Store from " + Utils.DEFAULT_CONFIG_FILE, e);
}
}
try {
- boolean createBlobContainer = PropertiesUtil.toBoolean(
- org.apache.jackrabbit.oak.commons.StringUtils.emptyToNull(properties.getProperty(AzureConstants.AZURE_CREATE_CONTAINER)), true);
+ Utils.setProxyIfNeeded(properties);
+ createBlobContainer = PropertiesUtil.toBoolean(
+ org.apache.jackrabbit.oak.commons.StringUtils.emptyToNull(properties.getProperty(AzureConstants.AZURE_CREATE_CONTAINER)), true);
initAzureDSConfig();
concurrentRequestCount = PropertiesUtil.toInteger(
properties.getProperty(AzureConstants.AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION),
- AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT);
- if (concurrentRequestCount < AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT) {
+ DEFAULT_CONCURRENT_REQUEST_COUNT);
+ if (concurrentRequestCount < DEFAULT_CONCURRENT_REQUEST_COUNT) {
LOG.warn("Invalid setting [{}] for concurrentRequestsPerOperation (too low); resetting to {}",
concurrentRequestCount,
- AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT);
- concurrentRequestCount = AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT;
- } else if (concurrentRequestCount > AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT) {
+ DEFAULT_CONCURRENT_REQUEST_COUNT);
+ concurrentRequestCount = DEFAULT_CONCURRENT_REQUEST_COUNT;
+ } else if (concurrentRequestCount > MAX_CONCURRENT_REQUEST_COUNT) {
LOG.warn("Invalid setting [{}] for concurrentRequestsPerOperation (too high); resetting to {}",
concurrentRequestCount,
- AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT);
- concurrentRequestCount = AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT;
+ MAX_CONCURRENT_REQUEST_COUNT);
+ concurrentRequestCount = MAX_CONCURRENT_REQUEST_COUNT;
}
LOG.info("Using concurrentRequestsPerOperation={}", concurrentRequestCount);
+ retryPolicy = Utils.getRetryPolicy(properties.getProperty(AzureConstants.AZURE_BLOB_MAX_REQUEST_RETRY));
if (properties.getProperty(AzureConstants.AZURE_BLOB_REQUEST_TIMEOUT) != null) {
requestTimeout = PropertiesUtil.toInteger(properties.getProperty(AzureConstants.AZURE_BLOB_REQUEST_TIMEOUT), RetryPolicy.DEFAULT_CLIENT_RETRY_COUNT);
}
-
- retryOptions = Utils.getRetryOptions(properties.getProperty(AzureConstants.AZURE_BLOB_MAX_REQUEST_RETRY), requestTimeout, computeSecondaryLocationEndpoint());
-
presignedDownloadURIVerifyExists = PropertiesUtil.toBoolean(
org.apache.jackrabbit.oak.commons.StringUtils.emptyToNull(properties.getProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_VERIFY_EXISTS)), true);
- BlobContainerClient azureContainer = getAzureContainer();
+ enableSecondaryLocation = PropertiesUtil.toBoolean(
+ properties.getProperty(AzureConstants.AZURE_BLOB_ENABLE_SECONDARY_LOCATION_NAME),
+ AzureConstants.AZURE_BLOB_ENABLE_SECONDARY_LOCATION_DEFAULT
+ );
+
+ CloudBlobContainer azureContainer = getAzureContainer();
if (createBlobContainer && !azureContainer.exists()) {
azureContainer.create();
@@ -188,20 +219,21 @@ public class AzureBlobStoreBackend extends AbstractAzureBlobStoreBackend {
} else {
LOG.info("Reusing existing container. containerName={}", getContainerName());
}
- LOG.debug("Backend initialized. duration={}", stopwatch.elapsed(TimeUnit.MILLISECONDS));
+ LOG.debug("Backend initialized. duration={}", (System.currentTimeMillis() - start));
// settings pertaining to DataRecordAccessProvider functionality
String putExpiry = properties.getProperty(AzureConstants.PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS);
- if (putExpiry != null) {
+ if (null != putExpiry) {
this.setHttpUploadURIExpirySeconds(Integer.parseInt(putExpiry));
}
String getExpiry = properties.getProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS);
- if (getExpiry != null) {
+ if (null != getExpiry) {
this.setHttpDownloadURIExpirySeconds(Integer.parseInt(getExpiry));
String cacheMaxSize = properties.getProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_CACHE_MAX_SIZE);
- if (cacheMaxSize != null) {
+ if (null != cacheMaxSize) {
this.setHttpDownloadURICacheSize(Integer.parseInt(cacheMaxSize));
- } else {
+ }
+ else {
this.setHttpDownloadURICacheSize(0); // default
}
}
@@ -211,14 +243,16 @@ public class AzureBlobStoreBackend extends AbstractAzureBlobStoreBackend {
// Initialize reference key secret
boolean createRefSecretOnInit = PropertiesUtil.toBoolean(
org.apache.jackrabbit.oak.commons.StringUtils.emptyToNull(properties.getProperty(AzureConstants.AZURE_REF_ON_INIT)), true);
+
if (createRefSecretOnInit) {
getOrCreateReferenceKey();
}
- } catch (BlobStorageException e) {
- LOG.error("Error setting up Azure Blob store backend: {}", e.getMessage());
+ }
+ catch (StorageException e) {
throw new DataStoreException(e);
}
- } finally {
+ }
+ finally {
Thread.currentThread().setContextClassLoader(contextClassLoader);
}
}
@@ -238,30 +272,33 @@ public class AzureBlobStoreBackend extends AbstractAzureBlobStoreBackend {
@Override
public InputStream read(DataIdentifier identifier) throws DataStoreException {
- Objects.requireNonNull(identifier, "identifier must not be null");
+ if (null == identifier) throw new NullPointerException("identifier");
String key = getKeyName(identifier);
- Stopwatch stopwatch = Stopwatch.createStarted();
+ long start = System.currentTimeMillis();
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
- InputStream is = null;
try {
Thread.currentThread().setContextClassLoader(
getClass().getClassLoader());
- BlockBlobClient blob = getAzureContainer().getBlobClient(key).getBlockBlobClient();
+ CloudBlockBlob blob = getAzureContainer().getBlockBlobReference(key);
if (!blob.exists()) {
throw new DataStoreException(String.format("Trying to read missing blob. identifier=%s", key));
}
- is = blob.openInputStream();
- LOG.debug("Got input stream for blob. identifier={} duration={}", key, stopwatch.elapsed(TimeUnit.MILLISECONDS));
+ InputStream is = blob.openInputStream();
+ LOG.debug("Got input stream for blob. identifier={} duration={}", key, (System.currentTimeMillis() - start));
if (LOG_STREAMS_DOWNLOAD.isDebugEnabled()) {
- // Log message, with exception, so we can get a trace to see where the call came from
+ // Log message, with exception so we can get a trace to see where the call came from
LOG_STREAMS_DOWNLOAD.debug("Binary downloaded from Azure Blob Storage - identifier={}", key, new Exception());
}
return is;
- } catch (BlobStorageException e) {
- LOG.error("Error reading blob. identifier={}", key);
- tryClose(is);
+ }
+ catch (StorageException e) {
+ LOG.info("Error reading blob. identifier={}", key);
+ throw new DataStoreException(String.format("Cannot read blob. identifier=%s", key), e);
+ }
+ catch (URISyntaxException e) {
+ LOG.debug("Error reading blob. identifier={}", key);
throw new DataStoreException(String.format("Cannot read blob. identifier=%s", key), e);
} finally {
if (contextClassLoader != null) {
@@ -270,45 +307,16 @@ public class AzureBlobStoreBackend extends AbstractAzureBlobStoreBackend {
}
}
- private void tryClose(InputStream is) {
- if (is != null) {
- try {
- is.close();
- } catch (IOException ioe) {
- LOG.warn("Failed to close the InputStream {}", is, ioe);
- }
- }
- }
-
- private void uploadBlob(BlockBlobClient client, File file, long len, Stopwatch stopwatch, String key) throws IOException {
- ParallelTransferOptions parallelTransferOptions = new ParallelTransferOptions()
- .setBlockSizeLong(len)
- .setMaxConcurrency(concurrentRequestCount)
- .setMaxSingleUploadSizeLong(AZURE_BLOB_MAX_SINGLE_PUT_UPLOAD_SIZE);
- BlobUploadFromFileOptions options = new BlobUploadFromFileOptions(file.getPath());
- options.setParallelTransferOptions(parallelTransferOptions);
- try {
- BlobClient blobClient = client.getContainerClient().getBlobClient(key);
- Response<BlockBlobItem> blockBlob = blobClient.uploadFromFileWithResponse(options, null, null);
- LOG.debug("Upload status is {} for blob {}", blockBlob.getStatusCode(), key);
- } catch (UncheckedIOException ex) {
- LOG.debug("Failed to upload from file:{}}", ex.getMessage());
- throw new IOException("Failed to upload blob: " + key, ex);
- }
- LOG.debug("Blob created. identifier={} length={} duration={}", key, len, stopwatch.elapsed(TimeUnit.MILLISECONDS));
- if (LOG_STREAMS_UPLOAD.isDebugEnabled()) {
- // Log message, with exception, so we can get a trace to see where the call came from
- LOG_STREAMS_UPLOAD.debug("Binary uploaded to Azure Blob Storage - identifier={}", key, new Exception());
- }
- }
-
@Override
public void write(DataIdentifier identifier, File file) throws DataStoreException {
- Objects.requireNonNull(identifier, "identifier must not be null");
- Objects.requireNonNull(file, "file must not be null");
-
+ if (null == identifier) {
+ throw new NullPointerException("identifier");
+ }
+ if (null == file) {
+ throw new NullPointerException("file");
+ }
String key = getKeyName(identifier);
- Stopwatch stopwatch = Stopwatch.createStarted();
+ long start = System.currentTimeMillis();
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
try {
@@ -316,64 +324,142 @@ public class AzureBlobStoreBackend extends AbstractAzureBlobStoreBackend {
long len = file.length();
LOG.debug("Blob write started. identifier={} length={}", key, len);
- BlockBlobClient blob = getAzureContainer().getBlobClient(key).getBlockBlobClient();
+ CloudBlockBlob blob = getAzureContainer().getBlockBlobReference(key);
if (!blob.exists()) {
- uploadBlob(blob, file, len, stopwatch, key);
- updateLastModifiedMetadata(blob);
+ addLastModified(blob);
+
+ BlobRequestOptions options = new BlobRequestOptions();
+ options.setConcurrentRequestCount(concurrentRequestCount);
+ boolean useBufferedStream = len < BUFFERED_STREAM_THRESHOLD;
+ final InputStream in = useBufferedStream ? new BufferedInputStream(new FileInputStream(file)) : new FileInputStream(file);
+ try {
+ blob.upload(in, len, null, options, null);
+ LOG.debug("Blob created. identifier={} length={} duration={} buffered={}", key, len, (System.currentTimeMillis() - start), useBufferedStream);
+ if (LOG_STREAMS_UPLOAD.isDebugEnabled()) {
+ // Log message, with exception so we can get a trace to see where the call came from
+ LOG_STREAMS_UPLOAD.debug("Binary uploaded to Azure Blob Storage - identifier={}", key, new Exception());
+ }
+ } finally {
+ in.close();
+ }
return;
}
- if (blob.getProperties().getBlobSize() != len) {
+ blob.downloadAttributes();
+ if (blob.getProperties().getLength() != len) {
throw new DataStoreException("Length Collision. identifier=" + key +
- " new length=" + len +
- " old length=" + blob.getProperties().getBlobSize());
+ " new length=" + len +
+ " old length=" + blob.getProperties().getLength());
}
LOG.trace("Blob already exists. identifier={} lastModified={}", key, getLastModified(blob));
- updateLastModifiedMetadata(blob);
+ addLastModified(blob);
+ blob.uploadMetadata();
LOG.debug("Blob updated. identifier={} lastModified={} duration={}", key,
- getLastModified(blob), stopwatch.elapsed(TimeUnit.MILLISECONDS));
- } catch (BlobStorageException e) {
+ getLastModified(blob), (System.currentTimeMillis() - start));
+ }
+ catch (StorageException e) {
LOG.info("Error writing blob. identifier={}", key, e);
throw new DataStoreException(String.format("Cannot write blob. identifier=%s", key), e);
- } catch (IOException e) {
+ }
+ catch (URISyntaxException | IOException e) {
LOG.debug("Error writing blob. identifier={}", key, e);
throw new DataStoreException(String.format("Cannot write blob. identifier=%s", key), e);
} finally {
- if (contextClassLoader != null) {
+ if (null != contextClassLoader) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
}
}
}
+ private static boolean waitForCopy(CloudBlob blob) throws StorageException, InterruptedException {
+ boolean continueLoop = true;
+ CopyStatus status = CopyStatus.PENDING;
+ while (continueLoop) {
+ blob.downloadAttributes();
+ status = blob.getCopyState().getStatus();
+ continueLoop = status == CopyStatus.PENDING;
+ // Sleep if retry is needed
+ if (continueLoop) {
+ Thread.sleep(500);
+ }
+ }
+ return status == CopyStatus.SUCCESS;
+ }
+
@Override
- public DataRecord getRecord(DataIdentifier identifier) throws DataStoreException {
- Objects.requireNonNull(identifier, "identifier must not be null");
+ public byte[] getOrCreateReferenceKey() throws DataStoreException {
+ try {
+ if (secret != null && secret.length != 0) {
+ return secret;
+ } else {
+ byte[] key;
+ // Try reading from the metadata folder if it exists
+ key = readMetadataBytes(REF_KEY);
+ if (key == null) {
+ key = super.getOrCreateReferenceKey();
+ addMetadataRecord(new ByteArrayInputStream(key), REF_KEY);
+ key = readMetadataBytes(REF_KEY);
+ }
+ secret = key;
+ return secret;
+ }
+ } catch (IOException e) {
+ throw new DataStoreException("Unable to get or create key " + e);
+ }
+ }
+ private byte[] readMetadataBytes(String name) throws IOException, DataStoreException {
+ DataRecord rec = getMetadataRecord(name);
+ byte[] key = null;
+ if (rec != null) {
+ InputStream stream = null;
+ try {
+ stream = rec.getStream();
+ return IOUtils.toByteArray(stream);
+ } finally {
+ IOUtils.closeQuietly(stream);
+ }
+ }
+ return key;
+ }
+
+ @Override
+ public DataRecord getRecord(DataIdentifier identifier) throws DataStoreException {
+ if (null == identifier) {
+ throw new NullPointerException("identifier");
+ }
String key = getKeyName(identifier);
- Stopwatch stopwatch = Stopwatch.createStarted();
+ long start = System.currentTimeMillis();
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
try {
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
- BlockBlobClient blob = getAzureContainer().getBlobClient(key).getBlockBlobClient();
+ CloudBlockBlob blob = getAzureContainer().getBlockBlobReference(key);
+ blob.downloadAttributes();
AzureBlobStoreDataRecord record = new AzureBlobStoreDataRecord(
this,
azureBlobContainerProvider,
- new DataIdentifier(getIdentifierName(blob.getBlobName())),
+ new DataIdentifier(getIdentifierName(blob.getName())),
getLastModified(blob),
- blob.getProperties().getBlobSize());
+ blob.getProperties().getLength());
LOG.debug("Data record read for blob. identifier={} duration={} record={}",
- key, stopwatch.elapsed(TimeUnit.MILLISECONDS), record);
+ key, (System.currentTimeMillis() - start), record);
return record;
- } catch (BlobStorageException e) {
- if (e.getStatusCode() == 404) {
+ }
+ catch (StorageException e) {
+ if (404 == e.getHttpStatusCode()) {
LOG.debug("Unable to get record for blob; blob does not exist. identifier={}", key);
- } else {
+ }
+ else {
LOG.info("Error getting data record for blob. identifier={}", key, e);
}
throw new DataStoreException(String.format("Cannot retrieve blob. identifier=%s", key), e);
+ }
+ catch (URISyntaxException e) {
+ LOG.debug("Error getting data record for blob. identifier={}", key, e);
+ throw new DataStoreException(String.format("Cannot retrieve blob. identifier=%s", key), e);
} finally {
if (contextClassLoader != null) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
@@ -382,68 +468,75 @@ public class AzureBlobStoreBackend extends AbstractAzureBlobStoreBackend {
}
@Override
- public Iterator<DataIdentifier> getAllIdentifiers() throws DataStoreException {
- return getAzureContainer().listBlobs().stream()
- .map(blobItem -> new DataIdentifier(getIdentifierName(blobItem.getName())))
- .iterator();
+ public Iterator<DataIdentifier> getAllIdentifiers() {
+ return new RecordsIterator<>(
+ input -> new DataIdentifier(getIdentifierName(input.getName())));
}
+
+
@Override
- public Iterator<DataRecord> getAllRecords() throws DataStoreException {
+ public Iterator<DataRecord> getAllRecords() {
final AbstractSharedBackend backend = this;
- final BlobContainerClient containerClient = getAzureContainer();
- return containerClient.listBlobs().stream()
- .map(blobItem -> (DataRecord) new AzureBlobStoreDataRecord(
+ return new RecordsIterator<>(
+ input -> new AzureBlobStoreDataRecord(
backend,
azureBlobContainerProvider,
- new DataIdentifier(getIdentifierName(blobItem.getName())),
- getLastModified(containerClient.getBlobClient(blobItem.getName()).getBlockBlobClient()),
- blobItem.getProperties().getContentLength()))
- .iterator();
+ new DataIdentifier(getIdentifierName(input.getName())),
+ input.getLastModified(),
+ input.getLength())
+ );
}
@Override
public boolean exists(DataIdentifier identifier) throws DataStoreException {
- Stopwatch stopwatch = Stopwatch.createStarted();
+ long start = System.currentTimeMillis();
String key = getKeyName(identifier);
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
try {
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
- boolean exists = getAzureContainer().getBlobClient(key).getBlockBlobClient().exists();
- LOG.debug("Blob exists={} identifier={} duration={}", exists, key, stopwatch.elapsed(TimeUnit.MILLISECONDS));
+ boolean exists =getAzureContainer().getBlockBlobReference(key).exists();
+ LOG.debug("Blob exists={} identifier={} duration={}", exists, key, (System.currentTimeMillis() - start));
return exists;
- } catch (Exception e) {
+ }
+ catch (Exception e) {
throw new DataStoreException(e);
- } finally {
- if (contextClassLoader != null) {
+ }
+ finally {
+ if (null != contextClassLoader) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
}
}
}
@Override
- public void close() {
- //Nothing to close
+ public void close() throws DataStoreException {
+ azureBlobContainerProvider.close();
+ LOG.info("AzureBlobBackend closed.");
}
@Override
public void deleteRecord(DataIdentifier identifier) throws DataStoreException {
- Objects.requireNonNull(identifier, "identifier must not be null");
+ if (null == identifier) throw new NullPointerException("identifier");
String key = getKeyName(identifier);
- Stopwatch stopwatch = Stopwatch.createStarted();
+ long start = System.currentTimeMillis();
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
try {
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
- boolean result = getAzureContainer().getBlobClient(key).getBlockBlobClient().deleteIfExists();
+ boolean result = getAzureContainer().getBlockBlobReference(key).deleteIfExists();
LOG.debug("Blob {}. identifier={} duration={}",
result ? "deleted" : "delete requested, but it does not exist (perhaps already deleted)",
- key, stopwatch.elapsed(TimeUnit.MILLISECONDS));
- } catch (BlobStorageException e) {
+ key, (System.currentTimeMillis() - start));
+ }
+ catch (StorageException e) {
LOG.info("Error deleting blob. identifier={}", key, e);
throw new DataStoreException(e);
+ }
+ catch (URISyntaxException e) {
+ throw new DataStoreException(e);
} finally {
if (contextClassLoader != null) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
@@ -453,152 +546,102 @@ public class AzureBlobStoreBackend extends AbstractAzureBlobStoreBackend {
@Override
public void addMetadataRecord(InputStream input, String name) throws DataStoreException {
- Objects.requireNonNull(input, "input must not be null");
- Validate.checkArgument(StringUtils.isNotEmpty(name), "name should not be empty");
- Stopwatch stopwatch = Stopwatch.createStarted();
+ if (null == input) {
+ throw new NullPointerException("input");
+ }
+ if (StringUtils.isEmpty(name)) {
+ throw new IllegalArgumentException("name");
+ }
+ long start = System.currentTimeMillis();
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
try {
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
- addMetadataRecordImpl(input, name, -1);
- LOG.debug("Metadata record added. metadataName={} duration={}", name, stopwatch.elapsed(TimeUnit.MILLISECONDS));
+ addMetadataRecordImpl(input, name, -1L);
+ LOG.debug("Metadata record added. metadataName={} duration={}", name, (System.currentTimeMillis() - start));
}
finally {
- if (contextClassLoader != null) {
+ if (null != contextClassLoader) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
}
}
}
@Override
- public void addMetadataRecord(File inputFile, String name) throws DataStoreException {
- Objects.requireNonNull(inputFile, "input must not be null");
- Validate.checkArgument(StringUtils.isNoneEmpty(name), "name should not be empty");
-
- Stopwatch stopwatch = Stopwatch.createStarted();
+ public void addMetadataRecord(File input, String name) throws DataStoreException {
+ if (null == input) {
+ throw new NullPointerException("input");
+ }
+ if (StringUtils.isEmpty(name)) {
+ throw new IllegalArgumentException("name");
+ }
+ long start = System.currentTimeMillis();
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
try {
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
- try (InputStream input = new FileInputStream(inputFile)) {
- addMetadataRecordImpl(input, name, inputFile.length());
- }
- LOG.debug("Metadata record added. metadataName={} duration={}", name, stopwatch.elapsed(TimeUnit.MILLISECONDS));
- } catch (IOException e) {
+
+ addMetadataRecordImpl(new FileInputStream(input), name, input.length());
+ LOG.debug("Metadata record added. metadataName={} duration={}", name, (System.currentTimeMillis() - start));
+ }
+ catch (FileNotFoundException e) {
throw new DataStoreException(e);
- } finally {
- if (contextClassLoader != null) {
+ }
+ finally {
+ if (null != contextClassLoader) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
}
}
}
- private BlockBlobClient getMetaBlobClient(String name) throws DataStoreException {
- return getAzureContainer().getBlobClient(AzureConstants.AZURE_BlOB_META_DIR_NAME + "/" + name).getBlockBlobClient();
- }
-
private void addMetadataRecordImpl(final InputStream input, String name, long recordLength) throws DataStoreException {
try {
- BlockBlobClient blockBlobClient = getMetaBlobClient(name);
-
- // If length is unknown (-1), use a file buffer for the stream first
- // This is necessary because Azure SDK requires a known length for upload
- // and loading the entire stream into memory is too risky
- if (recordLength < 0) {
- LOG.debug("Metadata record length unknown. metadataName={}. Saving to temporary file before upload", name);
- Path tempFile = createTempFileFromStream(input, name);
- long fileSize = Files.size(tempFile);
- LOG.debug("Metadata record temporary file created. metadataName={} path={}", name, tempFile);
- try (InputStream fis = new BufferedInputStream(Files.newInputStream(tempFile, DELETE_ON_CLOSE))) {
- blockBlobClient.upload(fis, fileSize, true);
- } finally {
- //will be true only if the file exists and was deleted
- boolean wasDeleted = Files.deleteIfExists(tempFile);
- if(wasDeleted) {
- LOG.debug("Cleanup: Metadata record temporary file deleted. metadataName={}, filePath={}, size={}", name, tempFile, fileSize);
- }
- else {
- LOG.debug("Cleanup: No metadata record temporary file not deleted. metadataName={}, filePath={}, size={}", name, tempFile, fileSize);
- }
- }
- } else {
- LOG.debug("Metadata record length known: {} bytes. metadataName={}. Uploading directly", recordLength, name);
- InputStream markableInput = input.markSupported() ? input : new BufferedInputStream(input);
- blockBlobClient.upload(markableInput, recordLength, true);
- }
- updateLastModifiedMetadata(blockBlobClient);
- } catch (BlobStorageException | IOException e) {
+ CloudBlobDirectory metaDir = getAzureContainer().getDirectoryReference(META_DIR_NAME);
+ CloudBlockBlob blob = metaDir.getBlockBlobReference(name);
+ addLastModified(blob);
+ blob.upload(input, recordLength);
+ }
+ catch (StorageException e) {
LOG.info("Error adding metadata record. metadataName={} length={}", name, recordLength, e);
throw new DataStoreException(e);
}
- }
-
- /**
- * Saves an InputStream to a temporary file with automatic cleanup support.
- *
- * <p>This method creates a temporary file and copies the entire contents of the input stream
- * to it. The temporary file is marked for deletion on JVM exit as a safety measure, but
- * callers should explicitly delete it when done.</p>
- *
- * @param input The InputStream to save to a temporary file
- * @param name The name string for the temporary file name (min 3 characters)
- * @return A File object representing the temporary file
- * @throws IOException if an I/O error occurs
- */
- private Path createTempFileFromStream(InputStream input, String name) throws IOException {
- Objects.requireNonNull(input, "input must not be null");
-
- Path tempPath = null;
- try {
- // Create temporary file
- tempPath = Files.createTempFile(name, null);
-
- // Copy stream contents to temporary file
- long bytesWritten = Files.copy(input, tempPath, StandardCopyOption.REPLACE_EXISTING);
- LOG.debug("Stream saved to temporary file. path={} size={} bytes", tempPath.toAbsolutePath(), bytesWritten);
-
- return tempPath;
- } catch (IOException e) {
- // Clean up the temporary file if an error occurs
- if (tempPath != null) {
- try {
- Files.deleteIfExists(tempPath);
- } catch (IOException cleanupException) {
- // Log but don't throw - we want to propagate the original exception
- LOG.info("Failed to delete temporary file after error: {}", tempPath, cleanupException);
- }
- }
- throw new IOException("Failed to save stream to temporary file", e);
+ catch (URISyntaxException | IOException e) {
+ throw new DataStoreException(e);
}
}
@Override
public DataRecord getMetadataRecord(String name) {
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
- Stopwatch stopwatch = Stopwatch.createStarted();
+ long start = System.currentTimeMillis();
try {
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
- BlockBlobClient blockBlobClient = getMetaBlobClient(name);
- if (!blockBlobClient.exists()) {
+ CloudBlobDirectory metaDir = getAzureContainer().getDirectoryReference(META_DIR_NAME);
+ CloudBlockBlob blob = metaDir.getBlockBlobReference(name);
+ if (!blob.exists()) {
LOG.warn("Trying to read missing metadata. metadataName={}", name);
return null;
}
-
- long lastModified = getLastModified(blockBlobClient);
- long length = blockBlobClient.getProperties().getBlobSize();
+ blob.downloadAttributes();
+ long lastModified = getLastModified(blob);
+ long length = blob.getProperties().getLength();
AzureBlobStoreDataRecord record = new AzureBlobStoreDataRecord(this,
azureBlobContainerProvider,
new DataIdentifier(name),
lastModified,
length,
true);
- LOG.debug("Metadata record read. metadataName={} duration={} record={}", name, stopwatch.elapsed(TimeUnit.MILLISECONDS), record);
+ LOG.debug("Metadata record read. metadataName={} duration={} record={}", name, (System.currentTimeMillis() - start), record);
return record;
- } catch (BlobStorageException | DataStoreException e) {
+
+ } catch (StorageException e) {
LOG.info("Error reading metadata record. metadataName={}", name, e);
throw new RuntimeException(e);
+ } catch (Exception e) {
+ LOG.debug("Error reading metadata record. metadataName={}", name, e);
+ throw new RuntimeException(e);
} finally {
- if (contextClassLoader != null) {
+ if (null != contextClassLoader) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
}
}
@@ -606,33 +649,39 @@ public class AzureBlobStoreBackend extends AbstractAzureBlobStoreBackend {
@Override
public List<DataRecord> getAllMetadataRecords(String prefix) {
- Objects.requireNonNull(prefix, "prefix must not be null");
-
- Stopwatch stopwatch = Stopwatch.createStarted();
+ if (null == prefix) {
+ throw new NullPointerException("prefix");
+ }
+ long start = System.currentTimeMillis();
final List<DataRecord> records = new ArrayList<>();
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
try {
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
- ListBlobsOptions listBlobsOptions = new ListBlobsOptions();
- listBlobsOptions.setPrefix(AzureConstants.AZURE_BlOB_META_DIR_NAME + "/" + prefix);
-
- for (BlobItem blobItem : getAzureContainer().listBlobs(listBlobsOptions, null)) {
- BlobClient blobClient = getAzureContainer().getBlobClient(blobItem.getName());
- BlobProperties properties = blobClient.getProperties();
-
- records.add(new AzureBlobStoreDataRecord(this,
- azureBlobContainerProvider,
- new DataIdentifier(stripMetaKeyPrefix(blobClient.getBlobName())),
- getLastModified(blobClient.getBlockBlobClient()),
- properties.getBlobSize(),
- true));
+ CloudBlobDirectory metaDir = getAzureContainer().getDirectoryReference(META_DIR_NAME);
+ for (ListBlobItem item : metaDir.listBlobs(prefix)) {
+ if (item instanceof CloudBlob) {
+ CloudBlob blob = (CloudBlob) item;
+ blob.downloadAttributes();
+ records.add(new AzureBlobStoreDataRecord(
+ this,
+ azureBlobContainerProvider,
+ new DataIdentifier(stripMetaKeyPrefix(blob.getName())),
+ getLastModified(blob),
+ blob.getProperties().getLength(),
+ true));
+ }
}
- LOG.debug("Metadata records read. recordsRead={} metadataFolder={} duration={}", records.size(), prefix, stopwatch.elapsed(TimeUnit.MILLISECONDS));
- } catch (BlobStorageException | DataStoreException e) {
+ LOG.debug("Metadata records read. recordsRead={} metadataFolder={} duration={}", records.size(), prefix, (System.currentTimeMillis() - start));
+ }
+ catch (StorageException e) {
LOG.info("Error reading all metadata records. metadataFolder={}", prefix, e);
- } finally {
- if (contextClassLoader != null) {
+ }
+ catch (DataStoreException | URISyntaxException e) {
+ LOG.debug("Error reading all metadata records. metadataFolder={}", prefix, e);
+ }
+ finally {
+ if (null != contextClassLoader) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
}
}
@@ -641,20 +690,26 @@ public class AzureBlobStoreBackend extends AbstractAzureBlobStoreBackend {
@Override
public boolean deleteMetadataRecord(String name) {
- Stopwatch stopwatch = Stopwatch.createStarted();
+ long start = System.currentTimeMillis();
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
try {
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
- BlobClient blob = getAzureContainer().getBlobClient(addMetaKeyPrefix(name));
+ CloudBlockBlob blob = getAzureContainer().getBlockBlobReference(addMetaKeyPrefix(name));
boolean result = blob.deleteIfExists();
LOG.debug("Metadata record {}. metadataName={} duration={}",
result ? "deleted" : "delete requested, but it does not exist (perhaps already deleted)",
- name, stopwatch.elapsed(TimeUnit.MILLISECONDS));
+ name, (System.currentTimeMillis() - start));
return result;
- } catch (BlobStorageException | DataStoreException e) {
+
+ }
+ catch (StorageException e) {
LOG.info("Error deleting metadata record. metadataName={}", name, e);
- } finally {
+ }
+ catch (DataStoreException | URISyntaxException e) {
+ LOG.debug("Error deleting metadata record. metadataName={}", name, e);
+ }
+ finally {
if (contextClassLoader != null) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
}
@@ -664,31 +719,35 @@ public class AzureBlobStoreBackend extends AbstractAzureBlobStoreBackend {
@Override
public void deleteAllMetadataRecords(String prefix) {
- Objects.requireNonNull(prefix, "prefix must not be null");
-
- Stopwatch stopwatch = Stopwatch.createStarted();
+ if (null == prefix) {
+ throw new NullPointerException("prefix");
+ }
+ long start = System.currentTimeMillis();
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
try {
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
+ CloudBlobDirectory metaDir = getAzureContainer().getDirectoryReference(META_DIR_NAME);
int total = 0;
-
- ListBlobsOptions listBlobsOptions = new ListBlobsOptions();
- listBlobsOptions.setPrefix(AzureConstants.AZURE_BlOB_META_DIR_NAME + "/" + prefix);
-
- for (BlobItem blobItem : getAzureContainer().listBlobs(listBlobsOptions, null)) {
- BlobClient blobClient = getAzureContainer().getBlobClient(blobItem.getName());
- if (blobClient.deleteIfExists()) {
- total++;
+ for (ListBlobItem item : metaDir.listBlobs(prefix)) {
+ if (item instanceof CloudBlob) {
+ if (((CloudBlob)item).deleteIfExists()) {
+ total++;
+ }
}
}
LOG.debug("Metadata records deleted. recordsDeleted={} metadataFolder={} duration={}",
- total, prefix, stopwatch.elapsed(TimeUnit.MILLISECONDS));
+ total, prefix, (System.currentTimeMillis() - start));
- } catch (BlobStorageException | DataStoreException e) {
+ }
+ catch (StorageException e) {
LOG.info("Error deleting all metadata records. metadataFolder={}", prefix, e);
- } finally {
- if (contextClassLoader != null) {
+ }
+ catch (DataStoreException | URISyntaxException e) {
+ LOG.debug("Error deleting all metadata records. metadataFolder={}", prefix, e);
+ }
+ finally {
+ if (null != contextClassLoader) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
}
}
@@ -696,17 +755,19 @@ public class AzureBlobStoreBackend extends AbstractAzureBlobStoreBackend {
@Override
public boolean metadataRecordExists(String name) {
- Stopwatch stopwatch = Stopwatch.createStarted();
+ long start = System.currentTimeMillis();
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
try {
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
- BlobClient blob = getAzureContainer().getBlobClient(addMetaKeyPrefix(name));
+ CloudBlockBlob blob = getAzureContainer().getBlockBlobReference(addMetaKeyPrefix(name));
boolean exists = blob.exists();
- LOG.debug("Metadata record {} exists {}. duration={}", name, exists, stopwatch.elapsed(TimeUnit.MILLISECONDS));
+ LOG.debug("Metadata record {} exists {}. duration={}", name, exists, (System.currentTimeMillis() - start));
return exists;
- } catch (DataStoreException | BlobStorageException e) {
- LOG.info("Error checking existence of metadata record = {}", name, e);
- } finally {
+ }
+ catch (DataStoreException | StorageException | URISyntaxException e) {
+ LOG.debug("Error checking existence of metadata record = {}", name, e);
+ }
+ finally {
if (contextClassLoader != null) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
}
@@ -714,6 +775,7 @@ public class AzureBlobStoreBackend extends AbstractAzureBlobStoreBackend {
return false;
}
+
/**
* Get key from data identifier. Object is stored with key in ADS.
*/
@@ -728,48 +790,45 @@ public class AzureBlobStoreBackend extends AbstractAzureBlobStoreBackend {
private static String getIdentifierName(String key) {
if (!key.contains(Utils.DASH)) {
return null;
- } else if (key.contains(AZURE_BLOB_META_KEY_PREFIX)) {
+ } else if (key.contains(META_KEY_PREFIX)) {
return key;
}
return key.substring(0, 4) + key.substring(5);
}
private static String addMetaKeyPrefix(final String key) {
- return AZURE_BLOB_META_KEY_PREFIX + key;
+ return META_KEY_PREFIX + key;
}
private static String stripMetaKeyPrefix(String name) {
- if (name.startsWith(AZURE_BLOB_META_KEY_PREFIX)) {
- return name.substring(AZURE_BLOB_META_KEY_PREFIX.length());
+ if (name.startsWith(META_KEY_PREFIX)) {
+ return name.substring(META_KEY_PREFIX.length());
}
return name;
}
- private static void updateLastModifiedMetadata(BlockBlobClient blockBlobClient) {
- Map<String, String> metadata = Objects.requireNonNullElse(blockBlobClient.getProperties().getMetadata(), new HashMap<>());
- metadata.put(AZURE_BLOB_LAST_MODIFIED_KEY, String.valueOf(System.currentTimeMillis()));
- blockBlobClient.setMetadata(metadata);
+ private static void addLastModified(CloudBlockBlob blob) {
+ blob.getMetadata().put(LAST_MODIFIED_KEY, String.valueOf(System.currentTimeMillis()));
}
- private static long getLastModified(BlockBlobClient blockBlobClient) {
- Map<String, String> metadata = blockBlobClient.getProperties().getMetadata();
- if (metadata == null || !metadata.containsKey(AZURE_BLOB_LAST_MODIFIED_KEY)) {
- return blockBlobClient.getProperties().getLastModified().toInstant().toEpochMilli();
+ private static long getLastModified(CloudBlob blob) {
+ if (blob.getMetadata().containsKey(LAST_MODIFIED_KEY)) {
+ return Long.parseLong(blob.getMetadata().get(LAST_MODIFIED_KEY));
}
- return Long.parseLong(metadata.get(AZURE_BLOB_LAST_MODIFIED_KEY));
+ return blob.getProperties().getLastModified().getTime();
}
- protected void setHttpDownloadURIExpirySeconds(int seconds) {
+ void setHttpDownloadURIExpirySeconds(int seconds) {
httpDownloadURIExpirySeconds = seconds;
}
- protected void setHttpDownloadURICacheSize(int maxSize) {
+ void setHttpDownloadURICacheSize(int maxSize) {
// max size 0 or smaller is used to turn off the cache
if (maxSize > 0) {
LOG.info("presigned GET URI cache enabled, maxSize = {} items, expiry = {} seconds", maxSize, httpDownloadURIExpirySeconds / 2);
- httpDownloadURICache = CacheBuilder.<String, URI>newBuilder()
+ httpDownloadURICache = CacheBuilder.newBuilder()
.maximumSize(maxSize)
- .expireAfterWrite(Duration.ofSeconds(httpDownloadURIExpirySeconds / 2))
+ .expireAfterWrite(httpDownloadURIExpirySeconds / 2, TimeUnit.SECONDS)
.build();
} else {
LOG.info("presigned GET URI cache disabled");
@@ -777,27 +836,29 @@ public class AzureBlobStoreBackend extends AbstractAzureBlobStoreBackend {
}
}
- @Override
- protected URI createHttpDownloadURI(@NotNull DataIdentifier identifier,
- @NotNull DataRecordDownloadOptions downloadOptions) {
+ URI createHttpDownloadURI(@NotNull DataIdentifier identifier,
+ @NotNull DataRecordDownloadOptions downloadOptions) {
URI uri = null;
- Objects.requireNonNull(identifier, "identifier must not be null");
- Objects.requireNonNull(downloadOptions, "downloadOptions must not be null");
+ // When running unit test from Maven, it doesn't always honor the @NotNull decorators
+ if (null == identifier) throw new NullPointerException("identifier");
+ if (null == downloadOptions) throw new NullPointerException("downloadOptions");
if (httpDownloadURIExpirySeconds > 0) {
String domain = getDirectDownloadBlobStorageDomain(downloadOptions.isDomainOverrideIgnored());
- Objects.requireNonNull(domain, "Could not determine domain for direct download");
+ if (null == domain) {
+ throw new NullPointerException("Could not determine domain for direct download");
+ }
- String cacheKey = identifier
+ String cacheKey = identifier.toString()
+ domain
+ Objects.toString(downloadOptions.getContentTypeHeader(), "")
+ Objects.toString(downloadOptions.getContentDispositionHeader(), "");
- if (httpDownloadURICache != null) {
+ if (null != httpDownloadURICache) {
uri = httpDownloadURICache.getIfPresent(cacheKey);
}
- if (uri == null) {
+ if (null == uri) {
if (presignedDownloadURIVerifyExists) {
// Check if this identifier exists. If not, we want to return null
// even if the identifier is in the download URI cache.
@@ -813,19 +874,25 @@ public class AzureBlobStoreBackend extends AbstractAzureBlobStoreBackend {
}
String key = getKeyName(identifier);
+ SharedAccessBlobHeaders headers = new SharedAccessBlobHeaders();
+ headers.setCacheControl(String.format("private, max-age=%d, immutable", httpDownloadURIExpirySeconds));
- // Prepare headers for the presigned URI
- BlobSasHeaders headers = new BlobSasHeaders()
- .setCacheControl(String.format("private, max-age=%d, immutable", httpDownloadURIExpirySeconds))
- .setContentType(downloadOptions.getContentTypeHeader())
- .setContentDisposition(downloadOptions.getContentDispositionHeader());
+ String contentType = downloadOptions.getContentTypeHeader();
+ if (! StringUtils.isEmpty(contentType)) {
+ headers.setContentType(contentType);
+ }
+
+ String contentDisposition =
+ downloadOptions.getContentDispositionHeader();
+ if (! StringUtils.isEmpty(contentDisposition)) {
+ headers.setContentDisposition(contentDisposition);
+ }
uri = createPresignedURI(key,
- new BlobSasPermission().setReadPermission(true),
+ EnumSet.of(SharedAccessBlobPermissions.READ),
httpDownloadURIExpirySeconds,
- Map.of(),
- domain,
- headers);
+ headers,
+ domain);
if (uri != null && httpDownloadURICache != null) {
httpDownloadURICache.put(cacheKey, uri);
}
@@ -834,28 +901,46 @@ public class AzureBlobStoreBackend extends AbstractAzureBlobStoreBackend {
return uri;
}
- protected void setHttpUploadURIExpirySeconds(int seconds) {
- httpUploadURIExpirySeconds = seconds;
- }
+ void setHttpUploadURIExpirySeconds(int seconds) { httpUploadURIExpirySeconds = seconds; }
private DataIdentifier generateSafeRandomIdentifier() {
return new DataIdentifier(
String.format("%s-%d",
- UUID.randomUUID(),
+ UUID.randomUUID().toString(),
Instant.now().toEpochMilli()
)
);
}
- protected DataRecordUpload initiateHttpUpload(long maxUploadSizeInBytes, int maxNumberOfURIs, @NotNull final DataRecordUploadOptions options) {
+ DataRecordUpload initiateHttpUpload(long maxUploadSizeInBytes, int maxNumberOfURIs, @NotNull final DataRecordUploadOptions options) {
List<URI> uploadPartURIs = new ArrayList<>();
- long minPartSize = AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE;
- long maxPartSize = AZURE_BLOB_MAX_MULTIPART_UPLOAD_PART_SIZE;
+ long minPartSize = MIN_MULTIPART_UPLOAD_PART_SIZE;
+ long maxPartSize = MAX_MULTIPART_UPLOAD_PART_SIZE;
- Validate.checkArgument(maxUploadSizeInBytes > 0L, "maxUploadSizeInBytes must be > 0");
- Validate.checkArgument(maxNumberOfURIs > 0 || maxNumberOfURIs == -1, "maxNumberOfURIs must either be > 0 or -1");
- Validate.checkArgument(!(maxUploadSizeInBytes > AZURE_BLOB_MAX_SINGLE_PUT_UPLOAD_SIZE && maxNumberOfURIs == 1), "Cannot do single-put upload with file size %d - exceeds max single-put upload size of %d", maxUploadSizeInBytes, AZURE_BLOB_MAX_SINGLE_PUT_UPLOAD_SIZE);
- Validate.checkArgument(maxUploadSizeInBytes <= AZURE_BLOB_MAX_BINARY_UPLOAD_SIZE, "Cannot do upload with file size %d - exceeds max upload size of %d", maxUploadSizeInBytes, AZURE_BLOB_MAX_BINARY_UPLOAD_SIZE);
+ if (0L >= maxUploadSizeInBytes) {
+ throw new IllegalArgumentException("maxUploadSizeInBytes must be > 0");
+ }
+ else if (0 == maxNumberOfURIs) {
+ throw new IllegalArgumentException("maxNumberOfURIs must either be > 0 or -1");
+ }
+ else if (-1 > maxNumberOfURIs) {
+ throw new IllegalArgumentException("maxNumberOfURIs must either be > 0 or -1");
+ }
+ else if (maxUploadSizeInBytes > MAX_SINGLE_PUT_UPLOAD_SIZE &&
+ maxNumberOfURIs == 1) {
+ throw new IllegalArgumentException(
+ String.format("Cannot do single-put upload with file size %d - exceeds max single-put upload size of %d",
+ maxUploadSizeInBytes,
+ MAX_SINGLE_PUT_UPLOAD_SIZE)
+ );
+ }
+ else if (maxUploadSizeInBytes > MAX_BINARY_UPLOAD_SIZE) {
+ throw new IllegalArgumentException(
+ String.format("Cannot do upload with file size %d - exceeds max upload size of %d",
+ maxUploadSizeInBytes,
+ MAX_BINARY_UPLOAD_SIZE)
+ );
+ }
DataIdentifier newIdentifier = generateSafeRandomIdentifier();
String blobId = getKeyName(newIdentifier);
@@ -888,7 +973,7 @@ public class AzureBlobStoreBackend extends AbstractAzureBlobStoreBackend {
maxNumberOfURIs,
Math.min(
(long) Math.ceil(((double) maxUploadSizeInBytes) / ((double) minPartSize)),
- AZURE_BLOB_MAX_ALLOWABLE_UPLOAD_URIS
+ MAX_ALLOWABLE_UPLOAD_URIS
)
);
} else {
@@ -896,17 +981,19 @@ public class AzureBlobStoreBackend extends AbstractAzureBlobStoreBackend {
String.format("Cannot do multi-part upload with requested part size %d", requestedPartSize)
);
}
- } else {
- long maximalNumParts = (long) Math.ceil(((double) maxUploadSizeInBytes) / ((double) AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE));
- numParts = Math.min(maximalNumParts, AZURE_BLOB_MAX_ALLOWABLE_UPLOAD_URIS);
+ }
+ else {
+ long maximalNumParts = (long) Math.ceil(((double) maxUploadSizeInBytes) / ((double) MIN_MULTIPART_UPLOAD_PART_SIZE));
+ numParts = Math.min(maximalNumParts, MAX_ALLOWABLE_UPLOAD_URIS);
}
String key = getKeyName(newIdentifier);
String domain = getDirectUploadBlobStorageDomain(options.isDomainOverrideIgnored());
- Objects.requireNonNull(domain, "Could not determine domain for direct upload");
+ if (null == domain) {
+ throw new NullPointerException("Could not determine domain for direct upload");
+ }
- BlobSasPermission perms = new BlobSasPermission()
- .setWritePermission(true);
+ EnumSet<SharedAccessBlobPermissions> perms = EnumSet.of(SharedAccessBlobPermissions.WRITE);
Map<String, String> presignedURIRequestParams = new HashMap<>();
// see https://docs.microsoft.com/en-us/rest/api/storageservices/put-block#uri-parameters
presignedURIRequestParams.put("comp", "block");
@@ -956,21 +1043,12 @@ public class AzureBlobStoreBackend extends AbstractAzureBlobStoreBackend {
return null;
}
- private Long getUncommittedBlocksListSize(BlockBlobClient client) throws DataStoreException {
- List<Block> blocks = client.listBlocks(BlockListType.UNCOMMITTED).getUncommittedBlocks();
- client.commitBlockList(blocks.stream().map(Block::getName).collect(Collectors.toList()));
- updateLastModifiedMetadata(client);
- long size = 0L;
- for (Block block : blocks) {
- size += block.getSizeLong();
- }
- return size;
- }
-
- protected DataRecord completeHttpUpload(@NotNull String uploadTokenStr)
+ DataRecord completeHttpUpload(@NotNull String uploadTokenStr)
throws DataRecordUploadException, DataStoreException {
- Validate.checkArgument(StringUtils.isNotEmpty(uploadTokenStr), "uploadToken required");
+ if (StringUtils.isEmpty(uploadTokenStr)) {
+ throw new IllegalArgumentException("uploadToken required");
+ }
DataRecordUploadToken uploadToken = DataRecordUploadToken.fromEncodedToken(uploadTokenStr, getOrCreateReferenceKey());
String key = uploadToken.getBlobId();
@@ -982,19 +1060,32 @@ public class AzureBlobStoreBackend extends AbstractAzureBlobStoreBackend {
// If this succeeds this means either it was a "single put" upload
// (we don't need to do anything in this case - blob is already uploaded)
// or it was completed before with the same token.
- } catch (DataStoreException e1) {
+ }
+ catch (DataStoreException e1) {
// record doesn't exist - so this means we are safe to do the complete request
try {
if (uploadToken.getUploadId().isPresent()) {
- BlockBlobClient blockBlobClient = getAzureContainer().getBlobClient(key).getBlockBlobClient();
- long size = getUncommittedBlocksListSize(blockBlobClient);
+ CloudBlockBlob blob = getAzureContainer().getBlockBlobReference(key);
+ // An existing upload ID means this is a multi-part upload
+ List<BlockEntry> blocks = blob.downloadBlockList(
+ BlockListingFilter.UNCOMMITTED,
+ AccessCondition.generateEmptyCondition(),
+ null,
+ null);
+ addLastModified(blob);
+ blob.commitBlockList(blocks);
+ long size = 0L;
+ for (BlockEntry block : blocks) {
+ size += block.getSize();
+ }
record = new AzureBlobStoreDataRecord(
this,
azureBlobContainerProvider,
blobId,
- getLastModified(blockBlobClient),
+ getLastModified(blob),
size);
- } else {
+ }
+ else {
// Something is wrong - upload ID missing from upload token
// but record doesn't exist already, so this is invalid
throw new DataRecordUploadException(
@@ -1002,7 +1093,7 @@ public class AzureBlobStoreBackend extends AbstractAzureBlobStoreBackend {
blobId)
);
}
- } catch (BlobStorageException e2) {
+ } catch (URISyntaxException | StorageException e2) {
throw new DataRecordUploadException(
String.format("Unable to finalize direct write of binary %s", blobId),
e2
@@ -1043,35 +1134,36 @@ public class AzureBlobStoreBackend extends AbstractAzureBlobStoreBackend {
}
private URI createPresignedURI(String key,
- BlobSasPermission blobSasPermissions,
+ EnumSet<SharedAccessBlobPermissions> permissions,
int expirySeconds,
+ SharedAccessBlobHeaders optionalHeaders,
String domain) {
- return createPresignedURI(key, blobSasPermissions, expirySeconds, new HashMap<>(), domain, null);
+ return createPresignedURI(key, permissions, expirySeconds, new HashMap<>(), optionalHeaders, domain);
}
private URI createPresignedURI(String key,
- BlobSasPermission blobSasPermissions,
+ EnumSet<SharedAccessBlobPermissions> permissions,
int expirySeconds,
Map<String, String> additionalQueryParams,
String domain) {
- return createPresignedURI(key, blobSasPermissions, expirySeconds, additionalQueryParams, domain, null);
+ return createPresignedURI(key, permissions, expirySeconds, additionalQueryParams, null, domain);
}
private URI createPresignedURI(String key,
- BlobSasPermission blobSasPermissions,
+ EnumSet<SharedAccessBlobPermissions> permissions,
int expirySeconds,
Map<String, String> additionalQueryParams,
- String domain,
- BlobSasHeaders optionalHeaders) {
- if (Objects.toString(domain, "").isEmpty()) {
+ SharedAccessBlobHeaders optionalHeaders,
+ String domain) {
+ if (StringUtils.isEmpty(domain)) {
LOG.warn("Can't generate presigned URI - no Azure domain provided (is Azure account name configured?)");
return null;
}
URI presignedURI = null;
try {
- String sharedAccessSignature = azureBlobContainerProvider.generateSharedAccessSignature(retryOptions, key,
- blobSasPermissions, expirySeconds, properties, optionalHeaders);
+ String sharedAccessSignature = azureBlobContainerProvider.generateSharedAccessSignature(getBlobRequestOptions(), key,
+ permissions, expirySeconds, optionalHeaders);
// Shared access signature is returned encoded already.
String uriString = String.format("https://%s/%s/%s?%s",
@@ -1080,7 +1172,7 @@ public class AzureBlobStoreBackend extends AbstractAzureBlobStoreBackend {
key,
sharedAccessSignature);
- if (!additionalQueryParams.isEmpty()) {
+ if (! additionalQueryParams.isEmpty()) {
StringBuilder builder = new StringBuilder();
for (Map.Entry<String, String> e : additionalQueryParams.entrySet()) {
builder.append("&");
@@ -1092,24 +1184,117 @@ public class AzureBlobStoreBackend extends AbstractAzureBlobStoreBackend {
}
presignedURI = new URI(uriString);
- } catch (DataStoreException e) {
+ }
+ catch (DataStoreException e) {
LOG.error("No connection to Azure Blob Storage", e);
- } catch (URISyntaxException | InvalidKeyException e) {
+ }
+ catch (URISyntaxException | InvalidKeyException e) {
LOG.error("Can't generate a presigned URI for key {}", key, e);
- } catch (BlobStorageException e) {
+ }
+ catch (StorageException e) {
LOG.error("Azure request to create presigned Azure Blob Storage {} URI failed. " +
"Key: {}, Error: {}, HTTP Code: {}, Azure Error Code: {}",
- blobSasPermissions.hasReadPermission() ? "GET" :
- ((blobSasPermissions.hasWritePermission()) ? "PUT" : ""),
+ permissions.contains(SharedAccessBlobPermissions.READ) ? "GET" :
+ (permissions.contains(SharedAccessBlobPermissions.WRITE) ? "PUT" : ""),
key,
e.getMessage(),
- e.getStatusCode(),
+ e.getHttpStatusCode(),
e.getErrorCode());
}
return presignedURI;
}
+ private static class AzureBlobInfo {
+ private final String name;
+ private final long lastModified;
+ private final long length;
+
+ public AzureBlobInfo(String name, long lastModified, long length) {
+ this.name = name;
+ this.lastModified = lastModified;
+ this.length = length;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public long getLastModified() {
+ return lastModified;
+ }
+
+ public long getLength() {
+ return length;
+ }
+
+ public static AzureBlobInfo fromCloudBlob(CloudBlob cloudBlob) throws StorageException {
+ cloudBlob.downloadAttributes();
+ return new AzureBlobInfo(cloudBlob.getName(),
+ AzureBlobStoreBackend.getLastModified(cloudBlob),
+ cloudBlob.getProperties().getLength());
+ }
+ }
+
+ private class RecordsIterator<T> extends AbstractIterator<T> {
+ // Seems to be thread-safe (in 5.0.0)
+ ResultContinuation resultContinuation;
+ boolean firstCall = true;
+ final Function<AzureBlobInfo, T> transformer;
+ final Queue<AzureBlobInfo> items = new LinkedList<>();
+
+ public RecordsIterator (Function<AzureBlobInfo, T> transformer) {
+ this.transformer = transformer;
+ }
+
+ @Override
+ protected T computeNext() {
+ if (items.isEmpty()) {
+ loadItems();
+ }
+ if (!items.isEmpty()) {
+ return transformer.apply(items.remove());
+ }
+ return endOfData();
+ }
+
+ private boolean loadItems() {
+ long start = System.currentTimeMillis();
+ ClassLoader contextClassLoader = currentThread().getContextClassLoader();
+ try {
+ currentThread().setContextClassLoader(getClass().getClassLoader());
+
+ CloudBlobContainer container = azureBlobContainerProvider.getBlobContainer();
+ if (!firstCall && (resultContinuation == null || !resultContinuation.hasContinuation())) {
+ LOG.trace("No more records in container. containerName={}", container);
+ return false;
+ }
+ firstCall = false;
+ ResultSegment<ListBlobItem> results = container.listBlobsSegmented(null, false, EnumSet.noneOf(BlobListingDetails.class), null, resultContinuation, null, null);
+ resultContinuation = results.getContinuationToken();
+ for (ListBlobItem item : results.getResults()) {
+ if (item instanceof CloudBlob) {
+ items.add(AzureBlobInfo.fromCloudBlob((CloudBlob)item));
+ }
+ }
+ LOG.debug("Container records batch read. batchSize={} containerName={} duration={}",
+ results.getLength(), getContainerName(), (System.currentTimeMillis() - start));
+ return results.getLength() > 0;
+ }
+ catch (StorageException e) {
+ LOG.info("Error listing blobs. containerName={}", getContainerName(), e);
+ }
+ catch (DataStoreException e) {
+ LOG.debug("Cannot list blobs. containerName={}", getContainerName(), e);
+ } finally {
+ if (contextClassLoader != null) {
+ currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+ return false;
+ }
+ }
+
static class AzureBlobStoreDataRecord extends AbstractDataRecord {
final AzureBlobContainerProvider azureBlobContainerProvider;
final long lastModified;
@@ -1138,20 +1323,20 @@ public class AzureBlobStoreBackend extends AbstractAzureBlobStoreBackend {
@Override
public InputStream getStream() throws DataStoreException {
String id = getKeyName(getIdentifier());
- BlobContainerClient container = azureBlobContainerProvider.getBlobContainer();
+ CloudBlobContainer container = azureBlobContainerProvider.getBlobContainer();
if (isMeta) {
id = addMetaKeyPrefix(getIdentifier().toString());
}
else {
// Don't worry about stream logging for metadata records
if (LOG_STREAMS_DOWNLOAD.isDebugEnabled()) {
- // Log message, with exception, so we can get a trace to see where the call came from
+ // Log message, with exception so we can get a trace to see where the call came from
LOG_STREAMS_DOWNLOAD.debug("Binary downloaded from Azure Blob Storage - identifier={} ", id, new Exception());
}
}
try {
- return container.getBlobClient(id).openInputStream();
- } catch (Exception e) {
+ return container.getBlockBlobReference(id).openInputStream();
+ } catch (StorageException | URISyntaxException e) {
throw new DataStoreException(e);
}
}
@@ -1177,54 +1362,4 @@ public class AzureBlobStoreBackend extends AbstractAzureBlobStoreBackend {
.map(AzureBlobContainerProvider::getContainerName)
.orElse(null);
}
-
- @Override
- public byte[] getOrCreateReferenceKey() throws DataStoreException {
- try {
- if (secret != null && secret.length != 0) {
- return secret;
- } else {
- byte[] key;
- // Try reading from the metadata folder if it exists
- key = readMetadataBytes(AZURE_BLOB_REF_KEY);
- if (key == null) {
- key = super.getOrCreateReferenceKey();
- addMetadataRecord(new ByteArrayInputStream(key), AZURE_BLOB_REF_KEY);
- key = readMetadataBytes(AZURE_BLOB_REF_KEY);
- }
- secret = key;
- return secret;
- }
- } catch (IOException e) {
- throw new DataStoreException("Unable to get or create key " + e);
- }
- }
-
- protected byte[] readMetadataBytes(String name) throws IOException, DataStoreException {
- DataRecord rec = getMetadataRecord(name);
- byte[] key = null;
- if (rec != null) {
- InputStream stream = null;
- try {
- stream = rec.getStream();
- return IOUtils.toByteArray(stream);
- } finally {
- IOUtils.closeQuietly(stream);
- }
- }
- return key;
- }
-
- private String computeSecondaryLocationEndpoint() {
- String accountName = properties.getProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, "");
-
- boolean enableSecondaryLocation = PropertiesUtil.toBoolean(properties.getProperty(AzureConstants.AZURE_BLOB_ENABLE_SECONDARY_LOCATION_NAME),
- AzureConstants.AZURE_BLOB_ENABLE_SECONDARY_LOCATION_DEFAULT);
-
- if(enableSecondaryLocation) {
- return String.format("https://%s-secondary.blob.core.windows.net", accountName);
- }
-
- return null;
- }
}
diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureConstants.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureConstants.java
index c13a5fc085..a7fa4249d2 100644
--- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureConstants.java
+++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureConstants.java
@@ -148,86 +148,6 @@ public final class AzureConstants {
* Property to enable/disable creation of reference secret on init.
*/
public static final String AZURE_REF_ON_INIT = "refOnInit";
-
- /**
- * Directory name for storing metadata files in the blob storage
- */
- public static final String AZURE_BlOB_META_DIR_NAME = "META";
-
- /**
- * Key prefix for metadata entries, includes trailing slash for directory structure
- */
- public static final String AZURE_BLOB_META_KEY_PREFIX = AZURE_BlOB_META_DIR_NAME + "/";
-
- /**
- * Key name for storing blob reference information
- */
- public static final String AZURE_BLOB_REF_KEY = "reference.key";
-
- /**
- * Key name for storing last modified timestamp metadata
- */
- public static final String AZURE_BLOB_LAST_MODIFIED_KEY = "lastModified";
-
- /**
- * Threshold size (8 MiB) above which streams are buffered to disk during upload operations
- */
- public static final long AZURE_BLOB_BUFFERED_STREAM_THRESHOLD = 8L * 1024L * 1024L;
-
- /**
- * Minimum part size (256 KiB) required for Azure Blob Storage multipart uploads
- */
- public static final long AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE = 256L * 1024L;
-
- /**
- * Maximum part size (4000 MiB / 4 GiB) allowed by Azure Blob Storage for multipart uploads
- */
- public static final long AZURE_BLOB_MAX_MULTIPART_UPLOAD_PART_SIZE = 4000L * 1024L * 1024L;
-
- /**
- * Maximum size (256 MiB) for single PUT operations in Azure Blob Storage
- */
- public static final long AZURE_BLOB_MAX_SINGLE_PUT_UPLOAD_SIZE = 256L * 1024L * 1024L;
-
- /**
- * Maximum total binary size (~190.7 TiB) that can be uploaded to Azure Blob Storage
- */
- public static final long AZURE_BLOB_MAX_BINARY_UPLOAD_SIZE = 190L * 1024L * 1024L * 1024L * 1024L;
-
- /**
- * Maximum number of blocks (50,000) allowed per blob in Azure Blob Storage
- */
- public static final int AZURE_BLOB_MAX_ALLOWABLE_UPLOAD_URIS = 50000;
-
- /**
- * Maximum number of attempts to generate a unique record identifier
- */
- public static final int AZURE_BLOB_MAX_UNIQUE_RECORD_TRIES = 10;
-
- /**
- * Default number of concurrent requests (5) for optimal performance with Azure SDK 12
- */
- public static final int AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT = 5;
-
- /**
- * Maximum recommended concurrent requests (10) for optimal performance with Azure SDK 12
- */
- public static final int AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT = 10;
-
- /**
- * Maximum block size (100 MiB) supported by Azure Blob Storage SDK 12
- */
- public static final long AZURE_BLOB_MAX_BLOCK_SIZE = 100L * 1024L * 1024L;
-
- /**
- * Default number of retry attempts (4) for failed requests in Azure SDK 12
- */
- public static final int AZURE_BLOB_MAX_RETRY_REQUESTS = 4;
-
- /**
- * Default timeout duration (60 seconds) for Azure Blob Storage operations
- */
- public static final int AZURE_BLOB_DEFAULT_TIMEOUT_SECONDS = 60;
-
+
private AzureConstants() { }
}
diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStore.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStore.java
index 6020e17f26..11575ad966 100644
--- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStore.java
+++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStore.java
@@ -22,11 +22,9 @@ package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;
import java.net.URI;
import java.util.Properties;
-import org.apache.jackrabbit.oak.spi.blob.data.DataIdentifier;
-import org.apache.jackrabbit.oak.spi.blob.data.DataRecord;
-import org.apache.jackrabbit.oak.spi.blob.data.DataStoreException;
-import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v8.AzureBlobStoreBackendV8;
-import org.apache.jackrabbit.oak.commons.properties.SystemPropertySupplier;
+import org.apache.jackrabbit.core.data.DataIdentifier;
+import org.apache.jackrabbit.core.data.DataRecord;
+import org.apache.jackrabbit.core.data.DataStoreException;
import org.apache.jackrabbit.oak.plugins.blob.AbstractSharedCachingDataStore;
import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.ConfigurableDataRecordAccessProvider;
import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadException;
@@ -37,34 +35,18 @@ import org.apache.jackrabbit.oak.spi.blob.AbstractSharedBackend;
import org.apache.jackrabbit.oak.spi.blob.SharedBackend;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
public class AzureDataStore extends AbstractSharedCachingDataStore implements ConfigurableDataRecordAccessProvider {
-
- private static final Logger log = LoggerFactory.getLogger(AzureDataStore.class);
-
private int minRecordLength = 16*1024;
protected Properties properties;
- private AbstractAzureBlobStoreBackend azureBlobStoreBackend;
-
- private static final String AZURE_SDK_12_ENABLED = "blob.azure.v12.enabled";
+ private AzureBlobStoreBackend azureBlobStoreBackend;
@Override
protected AbstractSharedBackend createBackend() {
- boolean useAzureSdkV12 = SystemPropertySupplier.create(AZURE_SDK_12_ENABLED, false).get();
-
- if (useAzureSdkV12) {
- log.info("Starting blob store using Azure SDK 12");
- azureBlobStoreBackend = new AzureBlobStoreBackend();
- } else {
- log.info("Starting blob store using Azure SDK 8");
- azureBlobStoreBackend = new AzureBlobStoreBackendV8();
- }
-
- if (properties != null) {
+ azureBlobStoreBackend = new AzureBlobStoreBackend();
+ if (null != properties) {
azureBlobStoreBackend.setProperties(properties);
}
return azureBlobStoreBackend;
@@ -92,7 +74,7 @@ public class AzureDataStore extends AbstractSharedCachingDataStore implements Co
//
@Override
public void setDirectUploadURIExpirySeconds(int seconds) {
- if (azureBlobStoreBackend != null) {
+ if (null != azureBlobStoreBackend) {
azureBlobStoreBackend.setHttpUploadURIExpirySeconds(seconds);
}
}
@@ -105,15 +87,15 @@ public class AzureDataStore extends AbstractSharedCachingDataStore implements Co
@Nullable
@Override
public DataRecordUpload initiateDataRecordUpload(long maxUploadSizeInBytes, int maxNumberOfURIs)
- throws DataRecordUploadException {
+ throws IllegalArgumentException, DataRecordUploadException {
return initiateDataRecordUpload(maxUploadSizeInBytes, maxNumberOfURIs, DataRecordUploadOptions.DEFAULT);
}
@Nullable
@Override
public DataRecordUpload initiateDataRecordUpload(long maxUploadSizeInBytes, int maxNumberOfURIs, @NotNull final DataRecordUploadOptions options)
- throws DataRecordUploadException {
- if (azureBlobStoreBackend == null) {
+ throws IllegalArgumentException, DataRecordUploadException {
+ if (null == azureBlobStoreBackend) {
throw new DataRecordUploadException("Backend not initialized");
}
return azureBlobStoreBackend.initiateHttpUpload(maxUploadSizeInBytes, maxNumberOfURIs, options);
@@ -121,9 +103,9 @@ public class AzureDataStore extends AbstractSharedCachingDataStore implements Co
@NotNull
@Override
- public DataRecord completeDataRecordUpload(@NotNull String uploadToken)
- throws DataRecordUploadException, DataStoreException {
- if (azureBlobStoreBackend == null) {
+ public DataRecord completeDataRecordUpload(String uploadToken)
+ throws IllegalArgumentException, DataRecordUploadException, DataStoreException {
+ if (null == azureBlobStoreBackend) {
throw new DataRecordUploadException("Backend not initialized");
}
return azureBlobStoreBackend.completeHttpUpload(uploadToken);
@@ -131,7 +113,7 @@ public class AzureDataStore extends AbstractSharedCachingDataStore implements Co
@Override
public void setDirectDownloadURIExpirySeconds(int seconds) {
- if (azureBlobStoreBackend != null) {
+ if (null != azureBlobStoreBackend) {
azureBlobStoreBackend.setHttpDownloadURIExpirySeconds(seconds);
}
}
@@ -145,7 +127,7 @@ public class AzureDataStore extends AbstractSharedCachingDataStore implements Co
@Override
public URI getDownloadURI(@NotNull DataIdentifier identifier,
@NotNull DataRecordDownloadOptions downloadOptions) {
- if (azureBlobStoreBackend != null) {
+ if (null != azureBlobStoreBackend) {
return azureBlobStoreBackend.createHttpDownloadURI(identifier, downloadOptions);
}
return null;
diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreService.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreService.java
index 6159c946b4..3ad2e5e46e 100644
--- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreService.java
+++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreService.java
@@ -20,7 +20,6 @@
package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;
import org.apache.jackrabbit.oak.stats.StatisticsProvider;
-import org.jetbrains.annotations.NotNull;
import org.osgi.service.component.annotations.Component;
import org.osgi.service.component.annotations.ConfigurationPolicy;
import org.osgi.service.component.annotations.Reference;
@@ -33,7 +32,7 @@ public class AzureDataStoreService extends AbstractAzureDataStoreService {
public static final String NAME = "org.apache.jackrabbit.oak.plugins.blob.datastore.AzureDataStore";
- protected @NotNull StatisticsProvider getStatisticsProvider() {
+ protected StatisticsProvider getStatisticsProvider(){
return statisticsProvider;
}
diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureHttpRequestLoggingPolicy.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureHttpRequestLoggingPolicy.java
deleted file mode 100644
index 70f18568ed..0000000000
--- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureHttpRequestLoggingPolicy.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;
-
-import com.azure.core.http.HttpPipelineCallContext;
-import com.azure.core.http.HttpPipelineNextPolicy;
-import com.azure.core.http.HttpResponse;
-import com.azure.core.http.policy.HttpPipelinePolicy;
-
-import org.apache.jackrabbit.oak.commons.properties.SystemPropertySupplier;
-import org.apache.jackrabbit.oak.commons.time.Stopwatch;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import reactor.core.publisher.Mono;
-
-import java.util.concurrent.TimeUnit;
-
-/**
- * HTTP pipeline policy for logging Azure Blob Storage requests in the oak-blob-cloud-azure module.
- *
- * This policy logs HTTP request details including method, URL, status code, and duration.
- * Verbose logging can be enabled by setting the system property:
- * -Dblob.azure.http.verbose.enabled=true
- *
- * This is similar to the AzureHttpRequestLoggingPolicy in oak-segment-azure but specifically
- * designed for the blob storage operations in oak-blob-cloud-azure.
- */
-public class AzureHttpRequestLoggingPolicy implements HttpPipelinePolicy {
-
- private static final Logger log = LoggerFactory.getLogger(AzureHttpRequestLoggingPolicy.class);
-
- private static final String AZURE_SDK_VERBOSE_LOGGING_ENABLED = "blob.azure.v12.http.verbose.enabled";
-
- private final boolean verboseEnabled = SystemPropertySupplier.create(AZURE_SDK_VERBOSE_LOGGING_ENABLED, false).get();
-
- @Override
- public Mono<HttpResponse> process(HttpPipelineCallContext context, HttpPipelineNextPolicy next) {
- Stopwatch stopwatch = Stopwatch.createStarted();
-
- return next.process().flatMap(httpResponse -> {
- if (verboseEnabled) {
- log.info("HTTP Blob Request: {} {} {} {} ms",
- context.getHttpRequest().getHttpMethod(),
- context.getHttpRequest().getUrl(),
- httpResponse.getStatusCode(),
- stopwatch.elapsed(TimeUnit.MILLISECONDS));
- }
-
- return Mono.just(httpResponse);
- });
- }
-}
diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/BlobSasHeaders.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/BlobSasHeaders.java
deleted file mode 100644
index 1d066eabf6..0000000000
--- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/BlobSasHeaders.java
+++ /dev/null
@@ -1,211 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;
-
-import com.azure.storage.blob.sas.BlobServiceSasSignatureValues;
-import org.jetbrains.annotations.Nullable;
-
-/**
- * Represents the optional headers that can be returned using SAS (Shared Access Signature).
- * This class is the Azure SDK 12 equivalent of the legacy {@code com.microsoft.azure.storage.blob.SharedAccessBlobHeaders}.
- *
- * <p>These headers are set on the {@link BlobServiceSasSignatureValues} object and will be
- * returned to the client when the SAS token is used to access the blob.</p>
- *
- * @see <a href="https://learn.microsoft.com/en-us/java/api/com.azure.storage.blob.sas.blobservicesassignaturevalues">BlobServiceSasSignatureValues</a>
- */
-public class BlobSasHeaders {
-
- private String cacheControl;
- private String contentDisposition;
- private String contentEncoding;
- private String contentLanguage;
- private String contentType;
-
- /**
- * Creates an empty BlobSasHeaders object.
- */
- public BlobSasHeaders() {
- }
-
- /**
- * Creates a BlobSasHeaders object with the specified values.
- *
- * @param cacheControl the cache-control header value
- * @param contentDisposition the content-disposition header value
- * @param contentEncoding the content-encoding header value
- * @param contentLanguage the content-language header value
- * @param contentType the content-type header value
- */
- public BlobSasHeaders(@Nullable String cacheControl,
- @Nullable String contentDisposition,
- @Nullable String contentEncoding,
- @Nullable String contentLanguage,
- @Nullable String contentType) {
- this.cacheControl = cacheControl;
- this.contentDisposition = contentDisposition;
- this.contentEncoding = contentEncoding;
- this.contentLanguage = contentLanguage;
- this.contentType = contentType;
- }
-
- /**
- * Gets the cache-control header value.
- *
- * @return the cache-control header value
- */
- @Nullable
- public String getCacheControl() {
- return cacheControl;
- }
-
- /**
- * Sets the cache-control header value.
- *
- * @param cacheControl the cache-control header value
- * @return this BlobSasHeaders object for method chaining
- */
- public BlobSasHeaders setCacheControl(@Nullable String cacheControl) {
- this.cacheControl = cacheControl;
- return this;
- }
-
- /**
- * Gets the content-disposition header value.
- *
- * @return the content-disposition header value
- */
- @Nullable
- public String getContentDisposition() {
- return contentDisposition;
- }
-
- /**
- * Sets the content-disposition header value.
- *
- * @param contentDisposition the content-disposition header value
- * @return this BlobSasHeaders object for method chaining
- */
- public BlobSasHeaders setContentDisposition(@Nullable String contentDisposition) {
- this.contentDisposition = contentDisposition;
- return this;
- }
-
- /**
- * Gets the content-encoding header value.
- *
- * @return the content-encoding header value
- */
- @Nullable
- public String getContentEncoding() {
- return contentEncoding;
- }
-
- /**
- * Sets the content-encoding header value.
- *
- * @param contentEncoding the content-encoding header value
- * @return this BlobSasHeaders object for method chaining
- */
- public BlobSasHeaders setContentEncoding(@Nullable String contentEncoding) {
- this.contentEncoding = contentEncoding;
- return this;
- }
-
- /**
- * Gets the content-language header value.
- *
- * @return the content-language header value
- */
- @Nullable
- public String getContentLanguage() {
- return contentLanguage;
- }
-
- /**
- * Sets the content-language header value.
- *
- * @param contentLanguage the content-language header value
- * @return this BlobSasHeaders object for method chaining
- */
- public BlobSasHeaders setContentLanguage(@Nullable String contentLanguage) {
- this.contentLanguage = contentLanguage;
- return this;
- }
-
- /**
- * Gets the content-type header value.
- *
- * @return the content-type header value
- */
- @Nullable
- public String getContentType() {
- return contentType;
- }
-
- /**
- * Sets the content-type header value.
- *
- * @param contentType the content-type header value
- * @return this BlobSasHeaders object for method chaining
- */
- public BlobSasHeaders setContentType(@Nullable String contentType) {
- this.contentType = contentType;
- return this;
- }
-
- /**
- * Applies these headers to the given {@link BlobServiceSasSignatureValues} object.
- * Only non-null headers are set.
- *
- * @param sasSignatureValues the BlobServiceSasSignatureValues object to apply headers to
- */
- public void applyTo(BlobServiceSasSignatureValues sasSignatureValues) {
- if (sasSignatureValues == null) {
- return;
- }
-
- if (cacheControl != null) {
- sasSignatureValues.setCacheControl(cacheControl);
- }
- if (contentDisposition != null) {
- sasSignatureValues.setContentDisposition(contentDisposition);
- }
- if (contentEncoding != null) {
- sasSignatureValues.setContentEncoding(contentEncoding);
- }
- if (contentLanguage != null) {
- sasSignatureValues.setContentLanguage(contentLanguage);
- }
- if (contentType != null) {
- sasSignatureValues.setContentType(contentType);
- }
- }
-
- /**
- * Checks if any headers are set (non-null).
- *
- * @return true if at least one header is set, false otherwise
- */
- public boolean hasHeaders() {
- return cacheControl != null || contentDisposition != null || contentEncoding != null
- || contentLanguage != null || contentType != null;
- }
-}
-
diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/Utils.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/Utils.java
index 1aa1edcacd..185816f7c0 100644
--- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/Utils.java
+++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/Utils.java
@@ -16,6 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
+
package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;
import java.io.File;
@@ -23,84 +24,106 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.InetSocketAddress;
-import java.util.Objects;
+import java.net.Proxy;
+import java.net.SocketAddress;
+import java.net.URISyntaxException;
+import java.security.InvalidKeyException;
import java.util.Properties;
-import com.azure.core.http.HttpClient;
-import com.azure.core.http.ProxyOptions;
-import com.azure.core.http.netty.NettyAsyncHttpClientBuilder;
-import com.azure.storage.blob.BlobContainerClient;
-import com.azure.storage.blob.BlobContainerClientBuilder;
-import com.azure.storage.blob.BlobServiceClient;
-import com.azure.storage.blob.BlobServiceClientBuilder;
-import com.azure.storage.common.policy.RequestRetryOptions;
-import com.azure.storage.common.policy.RetryPolicyType;
+import com.microsoft.azure.storage.CloudStorageAccount;
+import com.microsoft.azure.storage.OperationContext;
+import com.microsoft.azure.storage.RetryExponentialRetry;
+import com.microsoft.azure.storage.RetryNoRetry;
+import com.microsoft.azure.storage.RetryPolicy;
+import com.microsoft.azure.storage.StorageException;
+import com.microsoft.azure.storage.blob.BlobRequestOptions;
+import com.microsoft.azure.storage.blob.CloudBlobClient;
+import com.microsoft.azure.storage.blob.CloudBlobContainer;
import org.apache.commons.lang3.StringUtils;
-import org.apache.jackrabbit.oak.spi.blob.data.DataStoreException;
+import org.apache.jackrabbit.core.data.DataStoreException;
import org.apache.jackrabbit.oak.commons.PropertiesUtil;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
public final class Utils {
- public static final String DASH = "-";
- public static final String DEFAULT_CONFIG_FILE = "azure.properties";
- private Utils() {}
+ public static final String DEFAULT_CONFIG_FILE = "azure.properties";
- public static BlobContainerClient getBlobContainer(@NotNull final String connectionString,
- @NotNull final String containerName,
- @Nullable final RequestRetryOptions retryOptions,
- final Properties properties) throws DataStoreException {
- try {
- AzureHttpRequestLoggingPolicy loggingPolicy = new AzureHttpRequestLoggingPolicy();
+ public static final String DASH = "-";
- BlobServiceClientBuilder builder = new BlobServiceClientBuilder()
- .connectionString(connectionString)
- .retryOptions(retryOptions)
- .addPolicy(loggingPolicy);
+ /**
+ * private constructor so that class cannot initialized from outside.
+ */
+ private Utils() {
+ }
- HttpClient httpClient = new NettyAsyncHttpClientBuilder()
- .proxy(computeProxyOptions(properties))
- .build();
+ /**
+ * Create CloudBlobClient from properties.
+ *
+ * @param connectionString connectionString to configure @link {@link CloudBlobClient}
+ * @return {@link CloudBlobClient}
+ */
+ public static CloudBlobClient getBlobClient(@NotNull final String connectionString) throws URISyntaxException, InvalidKeyException {
+ return getBlobClient(connectionString, null);
+ }
- builder.httpClient(httpClient);
+ public static CloudBlobClient getBlobClient(@NotNull final String connectionString,
+ @Nullable final BlobRequestOptions requestOptions) throws URISyntaxException, InvalidKeyException {
+ CloudStorageAccount account = CloudStorageAccount.parse(connectionString);
+ CloudBlobClient client = account.createCloudBlobClient();
+ if (null != requestOptions) {
+ client.setDefaultRequestOptions(requestOptions);
+ }
+ return client;
+ }
- BlobServiceClient blobServiceClient = builder.buildClient();
- return blobServiceClient.getBlobContainerClient(containerName);
+ public static CloudBlobContainer getBlobContainer(@NotNull final String connectionString,
+ @NotNull final String containerName) throws DataStoreException {
+ return getBlobContainer(connectionString, containerName, null);
+ }
- } catch (Exception e) {
+ public static CloudBlobContainer getBlobContainer(@NotNull final String connectionString,
+ @NotNull final String containerName,
+ @Nullable final BlobRequestOptions requestOptions) throws DataStoreException {
+ try {
+ CloudBlobClient client = (
+ (null == requestOptions)
+ ? Utils.getBlobClient(connectionString)
+ : Utils.getBlobClient(connectionString, requestOptions)
+ );
+ return client.getContainerReference(containerName);
+ } catch (InvalidKeyException | URISyntaxException | StorageException e) {
throw new DataStoreException(e);
}
}
- public static ProxyOptions computeProxyOptions(final Properties properties) {
+ public static void setProxyIfNeeded(final Properties properties) {
String proxyHost = properties.getProperty(AzureConstants.PROXY_HOST);
String proxyPort = properties.getProperty(AzureConstants.PROXY_PORT);
- if(!(Objects.toString(proxyHost, "").isEmpty() || Objects.toString(proxyPort, "").isEmpty())) {
- return new ProxyOptions(ProxyOptions.Type.HTTP,
- new InetSocketAddress(proxyHost, Integer.parseInt(proxyPort)));
+ if (!StringUtils.isEmpty(proxyHost) &&
+ StringUtils.isEmpty(proxyPort)) {
+ int port = Integer.parseInt(proxyPort);
+ SocketAddress proxyAddr = new InetSocketAddress(proxyHost, port);
+ Proxy proxy = new Proxy(Proxy.Type.HTTP, proxyAddr);
+ OperationContext.setDefaultProxy(proxy);
}
- return null;
}
- public static RequestRetryOptions getRetryOptions(final String maxRequestRetryCount, Integer requestTimeout, String secondaryLocation) {
- int retries = PropertiesUtil.toInteger(maxRequestRetryCount, -1);
- if(retries < 0) {
+ public static RetryPolicy getRetryPolicy(final String maxRequestRetry) {
+ int retries = PropertiesUtil.toInteger(maxRequestRetry, -1);
+ if (retries < 0) {
return null;
}
-
if (retries == 0) {
- return new RequestRetryOptions(RetryPolicyType.FIXED, 1,
- requestTimeout, null, null,
- secondaryLocation);
+ return new RetryNoRetry();
}
- return new RequestRetryOptions(RetryPolicyType.EXPONENTIAL, retries,
- requestTimeout, null, null,
- secondaryLocation);
+ return new RetryExponentialRetry(RetryPolicy.DEFAULT_CLIENT_BACKOFF, retries);
}
+
public static String getConnectionStringFromProperties(Properties properties) {
+
String sasUri = properties.getProperty(AzureConstants.AZURE_SAS, "");
String blobEndpoint = properties.getProperty(AzureConstants.AZURE_BLOB_ENDPOINT, "");
String connectionString = properties.getProperty(AzureConstants.AZURE_CONNECTION_STRING, "");
@@ -117,7 +140,7 @@ public final class Utils {
return getConnectionString(
accountName,
- accountKey,
+ accountKey,
blobEndpoint);
}
@@ -129,28 +152,24 @@ public final class Utils {
}
}
+ public static String getConnectionString(final String accountName, final String accountKey) {
+ return getConnectionString(accountName, accountKey, null);
+ }
+
public static String getConnectionString(final String accountName, final String accountKey, String blobEndpoint) {
StringBuilder connString = new StringBuilder("DefaultEndpointsProtocol=https");
connString.append(";AccountName=").append(accountName);
connString.append(";AccountKey=").append(accountKey);
- if (!Objects.toString(blobEndpoint, "").isEmpty()) {
+
+ if (!StringUtils.isEmpty(blobEndpoint)) {
connString.append(";BlobEndpoint=").append(blobEndpoint);
}
return connString.toString();
}
- public static BlobContainerClient getBlobContainerFromConnectionString(final String azureConnectionString, final String containerName) {
- AzureHttpRequestLoggingPolicy loggingPolicy = new AzureHttpRequestLoggingPolicy();
-
- return new BlobContainerClientBuilder()
- .connectionString(azureConnectionString)
- .containerName(containerName)
- .addPolicy(loggingPolicy)
- .buildClient();
- }
-
/**
- * Read a configuration properties file.
+ * Read a configuration properties file. If the file name ends with ";burn",
+ * the file is deleted after reading.
*
* @param fileName the properties file name
* @return the properties
diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobContainerProviderV8.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobContainerProviderV8.java
deleted file mode 100644
index b2fbc67ff6..0000000000
--- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobContainerProviderV8.java
+++ /dev/null
@@ -1,346 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v8;
-
-import com.azure.core.credential.AccessToken;
-import com.azure.core.credential.TokenRequestContext;
-import com.azure.identity.ClientSecretCredential;
-import com.azure.identity.ClientSecretCredentialBuilder;
-import com.microsoft.azure.storage.CloudStorageAccount;
-import com.microsoft.azure.storage.StorageCredentialsToken;
-import com.microsoft.azure.storage.StorageException;
-import com.microsoft.azure.storage.UserDelegationKey;
-import com.microsoft.azure.storage.blob.BlobRequestOptions;
-import com.microsoft.azure.storage.blob.CloudBlobClient;
-import com.microsoft.azure.storage.blob.CloudBlobContainer;
-import com.microsoft.azure.storage.blob.CloudBlockBlob;
-import com.microsoft.azure.storage.blob.SharedAccessBlobHeaders;
-import com.microsoft.azure.storage.blob.SharedAccessBlobPermissions;
-import com.microsoft.azure.storage.blob.SharedAccessBlobPolicy;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.jackrabbit.oak.spi.blob.data.DataStoreException;
-import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants;
-import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser;
-import org.jetbrains.annotations.NotNull;
-import org.jetbrains.annotations.Nullable;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Closeable;
-import java.net.URISyntaxException;
-import java.security.InvalidKeyException;
-import java.time.Instant;
-import java.time.LocalDateTime;
-import java.time.OffsetDateTime;
-import java.time.format.DateTimeFormatter;
-import java.util.Date;
-import java.util.EnumSet;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.Properties;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-public class AzureBlobContainerProviderV8 implements Closeable {
- private static final Logger log = LoggerFactory.getLogger(AzureBlobContainerProviderV8.class);
- private static final String DEFAULT_ENDPOINT_SUFFIX = "core.windows.net";
- private static final String AZURE_DEFAULT_SCOPE = "https://storage.azure.com/.default";
- private final String azureConnectionString;
- private final String accountName;
- private final String containerName;
- private final String blobEndpoint;
- private final String sasToken;
- private final String accountKey;
- private final String tenantId;
- private final String clientId;
- private final String clientSecret;
- private ClientSecretCredential clientSecretCredential;
- private AccessToken accessToken;
- private StorageCredentialsToken storageCredentialsToken;
- private static final long TOKEN_REFRESHER_INITIAL_DELAY = 45L;
- private static final long TOKEN_REFRESHER_DELAY = 1L;
- private final ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
-
- private AzureBlobContainerProviderV8(Builder builder) {
- this.azureConnectionString = builder.azureConnectionString;
- this.accountName = builder.accountName;
- this.containerName = builder.containerName;
- this.blobEndpoint = builder.blobEndpoint;
- this.sasToken = builder.sasToken;
- this.accountKey = builder.accountKey;
- this.tenantId = builder.tenantId;
- this.clientId = builder.clientId;
- this.clientSecret = builder.clientSecret;
- }
-
- public static class Builder {
- private final String containerName;
-
- private Builder(String containerName) {
- this.containerName = containerName;
- }
-
- public static Builder builder(String containerName) {
- return new Builder(containerName);
- }
-
- private String azureConnectionString;
- private String accountName;
- private String blobEndpoint;
- private String sasToken;
- private String accountKey;
- private String tenantId;
- private String clientId;
- private String clientSecret;
-
- public Builder withAzureConnectionString(String azureConnectionString) {
- this.azureConnectionString = azureConnectionString;
- return this;
- }
-
- public Builder withAccountName(String accountName) {
- this.accountName = accountName;
- return this;
- }
-
- public Builder withBlobEndpoint(String blobEndpoint) {
- this.blobEndpoint = blobEndpoint;
- return this;
- }
-
- public Builder withSasToken(String sasToken) {
- this.sasToken = sasToken;
- return this;
- }
-
- public Builder withAccountKey(String accountKey) {
- this.accountKey = accountKey;
- return this;
- }
-
- public Builder withTenantId(String tenantId) {
- this.tenantId = tenantId;
- return this;
- }
-
- public Builder withClientId(String clientId) {
- this.clientId = clientId;
- return this;
- }
-
- public Builder withClientSecret(String clientSecret) {
- this.clientSecret = clientSecret;
- return this;
- }
-
- public Builder initializeWithProperties(Properties properties) {
- withAzureConnectionString(properties.getProperty(AzureConstants.AZURE_CONNECTION_STRING, ""));
- withAccountName(properties.getProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, ""));
- withBlobEndpoint(properties.getProperty(AzureConstants.AZURE_BLOB_ENDPOINT, ""));
- withSasToken(properties.getProperty(AzureConstants.AZURE_SAS, ""));
- withAccountKey(properties.getProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_KEY, ""));
- withTenantId(properties.getProperty(AzureConstants.AZURE_TENANT_ID, ""));
- withClientId(properties.getProperty(AzureConstants.AZURE_CLIENT_ID, ""));
- withClientSecret(properties.getProperty(AzureConstants.AZURE_CLIENT_SECRET, ""));
- return this;
- }
-
- public AzureBlobContainerProviderV8 build() {
- return new AzureBlobContainerProviderV8(this);
- }
- }
-
- public String getContainerName() {
- return containerName;
- }
-
- @NotNull
- public CloudBlobContainer getBlobContainer() throws DataStoreException {
- return this.getBlobContainer(null);
- }
-
- @NotNull
- public CloudBlobContainer getBlobContainer(@Nullable BlobRequestOptions blobRequestOptions) throws DataStoreException {
- // connection string will be given preference over service principals / sas / account key
- if (StringUtils.isNotBlank(azureConnectionString)) {
- log.debug("connecting to azure blob storage via azureConnectionString");
- return UtilsV8.getBlobContainer(azureConnectionString, containerName, blobRequestOptions);
- } else if (authenticateViaServicePrincipal()) {
- log.debug("connecting to azure blob storage via service principal credentials");
- return getBlobContainerFromServicePrincipals(blobRequestOptions);
- } else if (StringUtils.isNotBlank(sasToken)) {
- log.debug("connecting to azure blob storage via sas token");
- final String connectionStringWithSasToken = UtilsV8.getConnectionStringForSas(sasToken, blobEndpoint, accountName);
- return UtilsV8.getBlobContainer(connectionStringWithSasToken, containerName, blobRequestOptions);
- }
- log.debug("connecting to azure blob storage via access key");
- final String connectionStringWithAccountKey = UtilsV8.getConnectionString(accountName, accountKey, blobEndpoint);
- return UtilsV8.getBlobContainer(connectionStringWithAccountKey, containerName, blobRequestOptions);
- }
-
- @NotNull
- private CloudBlobContainer getBlobContainerFromServicePrincipals(@Nullable BlobRequestOptions blobRequestOptions) throws DataStoreException {
- StorageCredentialsToken storageCredentialsToken = getStorageCredentials();
- try {
- CloudStorageAccount cloud = new CloudStorageAccount(storageCredentialsToken, true, DEFAULT_ENDPOINT_SUFFIX, accountName);
- CloudBlobClient cloudBlobClient = cloud.createCloudBlobClient();
- if (blobRequestOptions != null) {
- cloudBlobClient.setDefaultRequestOptions(blobRequestOptions);
- }
- return cloudBlobClient.getContainerReference(containerName);
- } catch (URISyntaxException | StorageException e) {
- throw new DataStoreException(e);
- }
- }
-
- @NotNull
- private StorageCredentialsToken getStorageCredentials() {
- boolean isAccessTokenGenerated = false;
- /* generate access token, the same token will be used for subsequent access
- * generated token is valid for 1 hour only and will be refreshed in background
- * */
- if (accessToken == null) {
- clientSecretCredential = new ClientSecretCredentialBuilder()
- .clientId(clientId)
- .clientSecret(clientSecret)
- .tenantId(tenantId)
- .build();
- accessToken = clientSecretCredential.getTokenSync(new TokenRequestContext().addScopes(AZURE_DEFAULT_SCOPE));
- if (accessToken == null || StringUtils.isBlank(accessToken.getToken())) {
- log.error("Access token is null or empty");
- throw new IllegalArgumentException("Could not connect to azure storage, access token is null or empty");
- }
- storageCredentialsToken = new StorageCredentialsToken(accountName, accessToken.getToken());
- isAccessTokenGenerated = true;
- }
-
- Objects.requireNonNull(storageCredentialsToken, "storage credentials token cannot be null");
-
- // start refresh token executor only when the access token is first generated
- if (isAccessTokenGenerated) {
- log.info("starting refresh token task at: {}", OffsetDateTime.now());
- TokenRefresher tokenRefresher = new TokenRefresher();
- executorService.scheduleWithFixedDelay(tokenRefresher, TOKEN_REFRESHER_INITIAL_DELAY, TOKEN_REFRESHER_DELAY, TimeUnit.MINUTES);
- }
- return storageCredentialsToken;
- }
-
- @NotNull
- public String generateSharedAccessSignature(BlobRequestOptions requestOptions,
- String key,
- EnumSet<SharedAccessBlobPermissions> permissions,
- int expirySeconds,
- SharedAccessBlobHeaders optionalHeaders) throws DataStoreException, URISyntaxException, StorageException, InvalidKeyException {
- SharedAccessBlobPolicy policy = new SharedAccessBlobPolicy();
- Date expiry = Date.from(Instant.now().plusSeconds(expirySeconds));
- policy.setSharedAccessExpiryTime(expiry);
- policy.setPermissions(permissions);
-
- CloudBlockBlob blob = getBlobContainer(requestOptions).getBlockBlobReference(key);
-
- if (authenticateViaServicePrincipal()) {
- return generateUserDelegationKeySignedSas(blob, policy, optionalHeaders, expiry);
- }
- return generateSas(blob, policy, optionalHeaders);
- }
-
- @NotNull
- private String generateUserDelegationKeySignedSas(CloudBlockBlob blob,
- SharedAccessBlobPolicy policy,
- SharedAccessBlobHeaders optionalHeaders,
- Date expiry) throws StorageException {
- fillEmptyHeaders(optionalHeaders);
- UserDelegationKey userDelegationKey = blob.getServiceClient().getUserDelegationKey(Date.from(Instant.now().minusSeconds(900)),
- expiry);
- return optionalHeaders == null ? blob.generateUserDelegationSharedAccessSignature(userDelegationKey, policy) :
- blob.generateUserDelegationSharedAccessSignature(userDelegationKey, policy, optionalHeaders, null, null);
- }
-
- /* set empty headers as blank string due to a bug in Azure SDK
- * Azure SDK considers null headers as 'null' string which corrupts the string to sign and generates an invalid
- * sas token
- * */
- private void fillEmptyHeaders(SharedAccessBlobHeaders sharedAccessBlobHeaders) {
- final String EMPTY_STRING = "";
- Optional.ofNullable(sharedAccessBlobHeaders)
- .ifPresent(headers -> {
- if (StringUtils.isBlank(headers.getCacheControl())) {
- headers.setCacheControl(EMPTY_STRING);
- }
- if (StringUtils.isBlank(headers.getContentDisposition())) {
- headers.setContentDisposition(EMPTY_STRING);
- }
- if (StringUtils.isBlank(headers.getContentEncoding())) {
- headers.setContentEncoding(EMPTY_STRING);
- }
- if (StringUtils.isBlank(headers.getContentLanguage())) {
- headers.setContentLanguage(EMPTY_STRING);
- }
- if (StringUtils.isBlank(headers.getContentType())) {
- headers.setContentType(EMPTY_STRING);
- }
- });
- }
-
- @NotNull
- private String generateSas(CloudBlockBlob blob,
- SharedAccessBlobPolicy policy,
- SharedAccessBlobHeaders optionalHeaders) throws InvalidKeyException, StorageException {
- return optionalHeaders == null ? blob.generateSharedAccessSignature(policy, null) :
- blob.generateSharedAccessSignature(policy,
- optionalHeaders, null, null, null, true);
- }
-
- private boolean authenticateViaServicePrincipal() {
- return StringUtils.isBlank(azureConnectionString) &&
- StringUtils.isNoneBlank(accountName, tenantId, clientId, clientSecret);
- }
-
- class TokenRefresher implements Runnable {
- @Override
- public void run() {
- try {
- log.debug("Checking for azure access token expiry at: {}", LocalDateTime.now());
- OffsetDateTime tokenExpiryThreshold = OffsetDateTime.now().plusMinutes(5);
- if (accessToken.getExpiresAt() != null && accessToken.getExpiresAt().isBefore(tokenExpiryThreshold)) {
- log.info("Access token is about to expire (5 minutes or less) at: {}. New access token will be generated",
- accessToken.getExpiresAt().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME));
- AccessToken newToken = clientSecretCredential.getTokenSync(new TokenRequestContext().addScopes(AZURE_DEFAULT_SCOPE));
- log.info("New azure access token generated at: {}", LocalDateTime.now());
- if (newToken == null || StringUtils.isBlank(newToken.getToken())) {
- log.error("New access token is null or empty");
- return;
- }
- // update access token with newly generated token
- accessToken = newToken;
- storageCredentialsToken.updateToken(accessToken.getToken());
- }
- } catch (Exception e) {
- log.error("Error while acquiring new access token: ", e);
- }
- }
- }
-
- @Override
- public void close() {
- new ExecutorCloser(executorService).close();
- log.info("Refresh token executor service shutdown completed");
- }
-}
diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobStoreBackendV8.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobStoreBackendV8.java
deleted file mode 100644
index ce9a07cf5f..0000000000
--- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobStoreBackendV8.java
+++ /dev/null
@@ -1,1280 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v8;
-
-import static java.lang.Thread.currentThread;
-import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_BUFFERED_STREAM_THRESHOLD;
-import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT;
-import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_LAST_MODIFIED_KEY;
-import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MAX_ALLOWABLE_UPLOAD_URIS;
-import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MAX_BINARY_UPLOAD_SIZE;
-import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT;
-import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MAX_MULTIPART_UPLOAD_PART_SIZE;
-import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MAX_SINGLE_PUT_UPLOAD_SIZE;
-import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BlOB_META_DIR_NAME;
-import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_META_KEY_PREFIX;
-import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE;
-import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_REF_KEY;
-
-import java.io.BufferedInputStream;
-import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URLEncoder;
-import java.nio.charset.StandardCharsets;
-import java.security.InvalidKeyException;
-import java.time.Duration;
-import java.time.Instant;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.Properties;
-import java.util.Queue;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.Function;
-
-import org.apache.jackrabbit.oak.cache.api.Cache;
-import org.apache.jackrabbit.oak.cache.api.CacheBuilder;
-import org.apache.jackrabbit.oak.commons.collections.AbstractIterator;
-import org.apache.jackrabbit.oak.commons.time.Stopwatch;
-
-import com.microsoft.azure.storage.AccessCondition;
-import com.microsoft.azure.storage.LocationMode;
-import com.microsoft.azure.storage.ResultContinuation;
-import com.microsoft.azure.storage.ResultSegment;
-import com.microsoft.azure.storage.RetryPolicy;
-import com.microsoft.azure.storage.StorageException;
-import com.microsoft.azure.storage.blob.BlobListingDetails;
-import com.microsoft.azure.storage.blob.BlobRequestOptions;
-import com.microsoft.azure.storage.blob.BlockEntry;
-import com.microsoft.azure.storage.blob.BlockListingFilter;
-import com.microsoft.azure.storage.blob.CloudBlob;
-import com.microsoft.azure.storage.blob.CloudBlobContainer;
-import com.microsoft.azure.storage.blob.CloudBlobDirectory;
-import com.microsoft.azure.storage.blob.CloudBlockBlob;
-import com.microsoft.azure.storage.blob.ListBlobItem;
-import com.microsoft.azure.storage.blob.SharedAccessBlobHeaders;
-import com.microsoft.azure.storage.blob.SharedAccessBlobPermissions;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.jackrabbit.oak.spi.blob.data.DataIdentifier;
-import org.apache.jackrabbit.oak.spi.blob.data.DataRecord;
-import org.apache.jackrabbit.oak.spi.blob.data.DataStoreException;
-import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AbstractAzureBlobStoreBackend;
-import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants;
-import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.Utils;
-import org.apache.jackrabbit.oak.commons.PropertiesUtil;
-import org.apache.jackrabbit.oak.commons.conditions.Validate;
-import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordDownloadOptions;
-import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUpload;
-import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadException;
-import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadOptions;
-import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadToken;
-import org.apache.jackrabbit.oak.spi.blob.AbstractDataRecord;
-import org.apache.jackrabbit.oak.spi.blob.AbstractSharedBackend;
-import org.apache.jackrabbit.util.Base64;
-import org.jetbrains.annotations.NotNull;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class AzureBlobStoreBackendV8 extends AbstractAzureBlobStoreBackend {
-
- private static final Logger LOG = LoggerFactory.getLogger(AzureBlobStoreBackendV8.class);
- private static final Logger LOG_STREAMS_DOWNLOAD = LoggerFactory.getLogger("oak.datastore.download.streams");
- private static final Logger LOG_STREAMS_UPLOAD = LoggerFactory.getLogger("oak.datastore.upload.streams");
-
- private Properties properties;
- private AzureBlobContainerProviderV8 azureBlobContainerProvider;
- private int concurrentRequestCount = AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT;
- private RetryPolicy retryPolicy;
- private Integer requestTimeout;
- private int httpDownloadURIExpirySeconds = 0; // disabled by default
- private int httpUploadURIExpirySeconds = 0; // disabled by default
- private String uploadDomainOverride = null;
- private String downloadDomainOverride = null;
- private boolean createBlobContainer = true;
- private boolean presignedDownloadURIVerifyExists = true;
- private boolean enableSecondaryLocation = AzureConstants.AZURE_BLOB_ENABLE_SECONDARY_LOCATION_DEFAULT;
-
- private Cache<String, URI> httpDownloadURICache;
-
- private byte[] secret;
-
- public void setProperties(final Properties properties) {
- this.properties = properties;
- }
-
- private final AtomicReference<CloudBlobContainer> azureContainerReference = new AtomicReference<>();
-
- public CloudBlobContainer getAzureContainer() throws DataStoreException {
- azureContainerReference.compareAndSet(null, azureBlobContainerProvider.getBlobContainer(getBlobRequestOptions()));
- return azureContainerReference.get();
- }
-
- @NotNull
- protected BlobRequestOptions getBlobRequestOptions() {
- BlobRequestOptions requestOptions = new BlobRequestOptions();
- if (retryPolicy != null) {
- requestOptions.setRetryPolicyFactory(retryPolicy);
- }
- if (requestTimeout != null) {
- requestOptions.setTimeoutIntervalInMs(requestTimeout);
- }
- requestOptions.setConcurrentRequestCount(concurrentRequestCount);
- if (enableSecondaryLocation) {
- requestOptions.setLocationMode(LocationMode.PRIMARY_THEN_SECONDARY);
- }
- return requestOptions;
- }
-
- @Override
- public void init() throws DataStoreException {
- ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
- Stopwatch watch = Stopwatch.createStarted();
- try {
- Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
- LOG.debug("Started backend initialization");
-
- if (properties == null) {
- try {
- properties = Utils.readConfig(UtilsV8.DEFAULT_CONFIG_FILE);
- }
- catch (IOException e) {
- throw new DataStoreException("Unable to initialize Azure Data Store from " + UtilsV8.DEFAULT_CONFIG_FILE, e);
- }
- }
-
- try {
- UtilsV8.setProxyIfNeeded(properties);
- createBlobContainer = PropertiesUtil.toBoolean(
- org.apache.jackrabbit.oak.commons.StringUtils.emptyToNull(properties.getProperty(AzureConstants.AZURE_CREATE_CONTAINER)), true);
- initAzureDSConfig();
-
- concurrentRequestCount = PropertiesUtil.toInteger(
- properties.getProperty(AzureConstants.AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION),
- AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT);
- if (concurrentRequestCount < AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT) {
- LOG.warn("Invalid setting [{}] for concurrentRequestsPerOperation (too low); resetting to {}",
- concurrentRequestCount,
- AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT);
- concurrentRequestCount = AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT;
- } else if (concurrentRequestCount > AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT) {
- LOG.warn("Invalid setting [{}] for concurrentRequestsPerOperation (too high); resetting to {}",
- concurrentRequestCount,
- AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT);
- concurrentRequestCount = AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT;
- }
- LOG.info("Using concurrentRequestsPerOperation={}", concurrentRequestCount);
-
- retryPolicy = UtilsV8.getRetryPolicy(properties.getProperty(AzureConstants.AZURE_BLOB_MAX_REQUEST_RETRY));
- if (properties.getProperty(AzureConstants.AZURE_BLOB_REQUEST_TIMEOUT) != null) {
- requestTimeout = PropertiesUtil.toInteger(properties.getProperty(AzureConstants.AZURE_BLOB_REQUEST_TIMEOUT), RetryPolicy.DEFAULT_CLIENT_RETRY_COUNT);
- }
- presignedDownloadURIVerifyExists = PropertiesUtil.toBoolean(
- org.apache.jackrabbit.oak.commons.StringUtils.emptyToNull(properties.getProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_VERIFY_EXISTS)), true);
-
- enableSecondaryLocation = PropertiesUtil.toBoolean(
- properties.getProperty(AzureConstants.AZURE_BLOB_ENABLE_SECONDARY_LOCATION_NAME),
- AzureConstants.AZURE_BLOB_ENABLE_SECONDARY_LOCATION_DEFAULT
- );
-
- CloudBlobContainer azureContainer = getAzureContainer();
-
- if (createBlobContainer && !azureContainer.exists()) {
- azureContainer.create();
- LOG.info("New container created. containerName={}", getContainerName());
- } else {
- LOG.info("Reusing existing container. containerName={}", getContainerName());
- }
- LOG.debug("Backend initialized. duration={}", watch.elapsed(TimeUnit.MILLISECONDS));
-
- // settings pertaining to DataRecordAccessProvider functionality
- String putExpiry = properties.getProperty(AzureConstants.PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS);
- if (putExpiry != null) {
- this.setHttpUploadURIExpirySeconds(Integer.parseInt(putExpiry));
- }
- String getExpiry = properties.getProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS);
- if (getExpiry != null) {
- this.setHttpDownloadURIExpirySeconds(Integer.parseInt(getExpiry));
- String cacheMaxSize = properties.getProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_CACHE_MAX_SIZE);
- if (cacheMaxSize != null) {
- this.setHttpDownloadURICacheSize(Integer.parseInt(cacheMaxSize));
- }
- else {
- this.setHttpDownloadURICacheSize(0); // default
- }
- }
- uploadDomainOverride = properties.getProperty(AzureConstants.PRESIGNED_HTTP_UPLOAD_URI_DOMAIN_OVERRIDE, null);
- downloadDomainOverride = properties.getProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_DOMAIN_OVERRIDE, null);
-
- // Initialize reference key secret
- boolean createRefSecretOnInit = PropertiesUtil.toBoolean(
- org.apache.jackrabbit.oak.commons.StringUtils.emptyToNull(properties.getProperty(AzureConstants.AZURE_REF_ON_INIT)), true);
-
- if (createRefSecretOnInit) {
- getOrCreateReferenceKey();
- }
- }
- catch (StorageException e) {
- LOG.error("Error performing blob storage creation: {}", e.getMessage());
- throw new DataStoreException("Error performing backend initialization", e);
- }
- }
- finally {
- Thread.currentThread().setContextClassLoader(contextClassLoader);
- }
- }
-
- private void initAzureDSConfig() {
- AzureBlobContainerProviderV8.Builder builder = AzureBlobContainerProviderV8.Builder.builder(properties.getProperty(AzureConstants.AZURE_BLOB_CONTAINER_NAME))
- .withAzureConnectionString(properties.getProperty(AzureConstants.AZURE_CONNECTION_STRING, ""))
- .withAccountName(properties.getProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, ""))
- .withBlobEndpoint(properties.getProperty(AzureConstants.AZURE_BLOB_ENDPOINT, ""))
- .withSasToken(properties.getProperty(AzureConstants.AZURE_SAS, ""))
- .withAccountKey(properties.getProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_KEY, ""))
- .withTenantId(properties.getProperty(AzureConstants.AZURE_TENANT_ID, ""))
- .withClientId(properties.getProperty(AzureConstants.AZURE_CLIENT_ID, ""))
- .withClientSecret(properties.getProperty(AzureConstants.AZURE_CLIENT_SECRET, ""));
- azureBlobContainerProvider = builder.build();
- }
-
- @Override
- public InputStream read(DataIdentifier identifier) throws DataStoreException {
- Objects.requireNonNull(identifier, "identifier must not be null");
-
- String key = getKeyName(identifier);
- Stopwatch watch = Stopwatch.createStarted();
- ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
- try {
- Thread.currentThread().setContextClassLoader(
- getClass().getClassLoader());
- CloudBlockBlob blob = getAzureContainer().getBlockBlobReference(key);
- if (!blob.exists()) {
- throw new DataStoreException(String.format("Trying to read missing blob. identifier=%s", key));
- }
-
- InputStream is = blob.openInputStream();
- LOG.debug("Got input stream for blob. identifier={} duration={}", key, watch.elapsed(TimeUnit.MILLISECONDS));
- if (LOG_STREAMS_DOWNLOAD.isDebugEnabled()) {
- // Log message, with exception so we can get a trace to see where the call came from
- LOG_STREAMS_DOWNLOAD.debug("Binary downloaded from Azure Blob Storage - identifier={}", key, new Exception());
- }
- return is;
- }
- catch (StorageException | URISyntaxException e) {
- LOG.error("Error reading blob. identifier={}", key);
- throw new DataStoreException(String.format("Cannot read blob. identifier=%s", key), e);
- }
- finally {
- if (contextClassLoader != null) {
- Thread.currentThread().setContextClassLoader(contextClassLoader);
- }
- }
- }
-
- @Override
- public void write(DataIdentifier identifier, File file) throws DataStoreException {
- Objects.requireNonNull(identifier, "identifier must not be null");
- Objects.requireNonNull(file, "file must not be null");
-
- String key = getKeyName(identifier);
- Stopwatch watch = Stopwatch.createStarted();
- ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
-
- try {
- Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
-
- long len = file.length();
- LOG.debug("Blob write started. identifier={} length={}", key, len);
- CloudBlockBlob blob = getAzureContainer().getBlockBlobReference(key);
- if (!blob.exists()) {
- addLastModified(blob);
-
- BlobRequestOptions options = new BlobRequestOptions();
- options.setConcurrentRequestCount(concurrentRequestCount);
- boolean useBufferedStream = len < AZURE_BLOB_BUFFERED_STREAM_THRESHOLD;
- try (InputStream in = useBufferedStream ? new BufferedInputStream(new FileInputStream(file)) : new FileInputStream(file)) {
- blob.upload(in, len, null, options, null);
- LOG.debug("Blob created. identifier={} length={} duration={} buffered={}", key, len, watch.elapsed(TimeUnit.MILLISECONDS), useBufferedStream);
- if (LOG_STREAMS_UPLOAD.isDebugEnabled()) {
- // Log message, with exception so we can get a trace to see where the call came from
- LOG_STREAMS_UPLOAD.debug("Binary uploaded to Azure Blob Storage - identifier={}", key, new Exception());
- }
- }
- return;
- }
-
- blob.downloadAttributes();
- if (blob.getProperties().getLength() != len) {
- throw new DataStoreException("Length Collision. identifier=" + key +
- " new length=" + len +
- " old length=" + blob.getProperties().getLength());
- }
-
- LOG.trace("Blob already exists. identifier={} lastModified={}", key, getLastModified(blob));
- addLastModified(blob);
- blob.uploadMetadata();
-
- LOG.debug("Blob updated. identifier={} lastModified={} duration={}", key,
- getLastModified(blob), watch.elapsed(TimeUnit.MILLISECONDS));
- }
- catch (StorageException | URISyntaxException | IOException e) {
- LOG.debug("Error writing blob. identifier={}", key, e);
- throw new DataStoreException(String.format("Cannot write blob. identifier=%s", key), e);
- } finally {
- if (contextClassLoader != null) {
- Thread.currentThread().setContextClassLoader(contextClassLoader);
- }
- }
- }
-
- @Override
- public DataRecord getRecord(DataIdentifier identifier) throws DataStoreException {
- Objects.requireNonNull(identifier, "identifier must not be null");
-
- String key = getKeyName(identifier);
- Stopwatch watch = Stopwatch.createStarted();
- ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
- try {
- Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
-
- CloudBlockBlob blob = getAzureContainer().getBlockBlobReference(key);
- blob.downloadAttributes();
- AzureBlobStoreDataRecord record = new AzureBlobStoreDataRecord(
- this,
- azureBlobContainerProvider,
- new DataIdentifier(getIdentifierName(blob.getName())),
- getLastModified(blob),
- blob.getProperties().getLength());
- LOG.debug("Data record read for blob. identifier={} duration={} record={}",
- key, watch.elapsed(TimeUnit.MILLISECONDS), record);
- return record;
- }
- catch (StorageException e) {
- if (404 == e.getHttpStatusCode()) {
- LOG.debug("Unable to get record for blob; blob does not exist. identifier={}", key);
- }
- else {
- LOG.info("Error getting data record for blob. identifier={}", key, e);
- }
- throw new DataStoreException(String.format("Cannot retrieve blob. identifier=%s", key), e);
- }
- catch (URISyntaxException e) {
- LOG.debug("Error getting data record for blob. identifier={}", key, e);
- throw new DataStoreException(String.format("Cannot retrieve blob. identifier=%s", key), e);
- } finally {
- if (contextClassLoader != null) {
- Thread.currentThread().setContextClassLoader(contextClassLoader);
- }
- }
- }
-
- @Override
- public Iterator<DataIdentifier> getAllIdentifiers() {
- return new RecordsIterator<>(
- input -> new DataIdentifier(getIdentifierName(input.getName())));
- }
-
- @Override
- public Iterator<DataRecord> getAllRecords() {
- final AbstractSharedBackend backend = this;
- return new RecordsIterator<>(
- input -> new AzureBlobStoreDataRecord(
- backend,
- azureBlobContainerProvider,
- new DataIdentifier(getIdentifierName(input.getName())),
- input.getLastModified(),
- input.getLength())
- );
- }
-
- @Override
- public boolean exists(DataIdentifier identifier) throws DataStoreException {
- Stopwatch watch = Stopwatch.createStarted();
- String key = getKeyName(identifier);
- ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
- try {
- Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
-
- boolean exists = getAzureContainer().getBlockBlobReference(key).exists();
- LOG.debug("Blob exists={} identifier={} duration={}", exists, key, watch.elapsed(TimeUnit.MILLISECONDS));
- return exists;
- }
- catch (Exception e) {
- throw new DataStoreException(e);
- }
- finally {
- if (contextClassLoader != null) {
- Thread.currentThread().setContextClassLoader(contextClassLoader);
- }
- }
- }
-
- @Override
- public void close() {
- if(azureBlobContainerProvider != null) {
- azureBlobContainerProvider.close();
- }
- LOG.info("AzureBlobBackend closed.");
- }
-
- @Override
- public void deleteRecord(DataIdentifier identifier) throws DataStoreException {
- Objects.requireNonNull(identifier, "identifier must not be null");
-
- String key = getKeyName(identifier);
- Stopwatch watch = Stopwatch.createStarted();
- ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
- try {
- Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
-
- boolean result = getAzureContainer().getBlockBlobReference(key).deleteIfExists();
- LOG.debug("Blob {}. identifier={} duration={}",
- result ? "deleted" : "delete requested, but it does not exist (perhaps already deleted)",
- key, watch.elapsed(TimeUnit.MILLISECONDS));
- }
- catch (StorageException | URISyntaxException e) {
- LOG.error("Error deleting blob. identifier={}", key, e);
- throw new DataStoreException(e);
- } finally {
- if (contextClassLoader != null) {
- Thread.currentThread().setContextClassLoader(contextClassLoader);
- }
- }
- }
-
- @Override
- public void addMetadataRecord(InputStream input, String name) throws DataStoreException {
- Objects.requireNonNull(input, "input must not be null");
- Validate.checkArgument(StringUtils.isNotEmpty(name), "name should not be empty");
-
- Stopwatch watch = Stopwatch.createStarted();
- ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
- try {
- Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
-
- addMetadataRecordImpl(input, name, -1L);
- LOG.debug("Metadata record added. metadataName={} duration={}", name, watch.elapsed(TimeUnit.MILLISECONDS));
- }
- finally {
- if (contextClassLoader != null) {
- Thread.currentThread().setContextClassLoader(contextClassLoader);
- }
- }
- }
-
- @Override
- public void addMetadataRecord(File input, String name) throws DataStoreException {
- Objects.requireNonNull(input, "input must not be null");
- Validate.checkArgument(StringUtils.isNotEmpty(name), "name should not be empty");
-
- Stopwatch watch = Stopwatch.createStarted();
- ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
- try {
- Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
-
- addMetadataRecordImpl(new FileInputStream(input), name, input.length());
- LOG.debug("Metadata record added. metadataName={} duration={}", name, watch.elapsed(TimeUnit.MILLISECONDS));
- }
- catch (FileNotFoundException e) {
- LOG.error("Error adding metadata record. metadataName={}", name, e);
- throw new DataStoreException(e);
- }
- finally {
- if (contextClassLoader != null) {
- Thread.currentThread().setContextClassLoader(contextClassLoader);
- }
- }
- }
-
- private void addMetadataRecordImpl(final InputStream input, String name, long recordLength) throws DataStoreException {
- try {
- CloudBlobDirectory metaDir = getAzureContainer().getDirectoryReference(AZURE_BlOB_META_DIR_NAME);
- CloudBlockBlob blob = metaDir.getBlockBlobReference(name);
- addLastModified(blob);
- blob.upload(input, recordLength);
- }
- catch (StorageException | URISyntaxException | IOException e) {
- LOG.error("Error adding metadata record. metadataName={} length={}", name, recordLength, e);
- throw new DataStoreException(e);
- }
- }
-
- @Override
- public DataRecord getMetadataRecord(String name) {
- ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
- Stopwatch watch = Stopwatch.createStarted();
- try {
- Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
-
- CloudBlobDirectory metaDir = getAzureContainer().getDirectoryReference(AZURE_BlOB_META_DIR_NAME);
- CloudBlockBlob blob = metaDir.getBlockBlobReference(name);
- if (!blob.exists()) {
- LOG.warn("Trying to read missing metadata. metadataName={}", name);
- return null;
- }
- blob.downloadAttributes();
- long lastModified = getLastModified(blob);
- long length = blob.getProperties().getLength();
- AzureBlobStoreDataRecord record = new AzureBlobStoreDataRecord(this,
- azureBlobContainerProvider,
- new DataIdentifier(name),
- lastModified,
- length,
- true);
- LOG.debug("Metadata record read. metadataName={} duration={} record={}", name, watch.elapsed(TimeUnit.MILLISECONDS), record);
- return record;
-
- } catch (StorageException | DataStoreException | URISyntaxException e) {
- LOG.info("Error reading metadata record. metadataName={}", name, e);
- throw new RuntimeException(e);
- } finally {
- if (contextClassLoader != null) {
- Thread.currentThread().setContextClassLoader(contextClassLoader);
- }
- }
- }
-
- @Override
- public List<DataRecord> getAllMetadataRecords(String prefix) {
- Objects.requireNonNull(prefix, "prefix must not be null");
-
- Stopwatch watch = Stopwatch.createStarted();
- final List<DataRecord> records = new ArrayList<>();
- ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
- try {
- Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
-
- CloudBlobDirectory metaDir = getAzureContainer().getDirectoryReference(AZURE_BlOB_META_DIR_NAME);
- for (ListBlobItem item : metaDir.listBlobs(prefix)) {
- if (item instanceof CloudBlob) {
- CloudBlob blob = (CloudBlob) item;
- blob.downloadAttributes();
- records.add(new AzureBlobStoreDataRecord(
- this,
- azureBlobContainerProvider,
- new DataIdentifier(stripMetaKeyPrefix(blob.getName())),
- getLastModified(blob),
- blob.getProperties().getLength(),
- true));
- }
- }
- LOG.debug("Metadata records read. recordsRead={} metadataFolder={} duration={}", records.size(), prefix, watch.elapsed(TimeUnit.MILLISECONDS));
- }
- catch (StorageException | DataStoreException | URISyntaxException e) {
- LOG.error("Error reading all metadata records. metadataFolder={}", prefix, e);
- }
- finally {
- if (contextClassLoader != null) {
- Thread.currentThread().setContextClassLoader(contextClassLoader);
- }
- }
- return records;
- }
-
- @Override
- public boolean deleteMetadataRecord(String name) {
- Stopwatch watch = Stopwatch.createStarted();
- ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
- try {
- Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
-
- CloudBlockBlob blob = getAzureContainer().getBlockBlobReference(addMetaKeyPrefix(name));
- boolean result = blob.deleteIfExists();
- LOG.debug("Metadata record {}. metadataName={} duration={}",
- result ? "deleted" : "delete requested, but it does not exist (perhaps already deleted)",
- name, watch.elapsed(TimeUnit.MILLISECONDS));
- return result;
-
- }
- catch (StorageException | DataStoreException | URISyntaxException e) {
- LOG.error("Error deleting metadata record. metadataName={}", name, e);
- }
- finally {
- if (contextClassLoader != null) {
- Thread.currentThread().setContextClassLoader(contextClassLoader);
- }
- }
- return false;
- }
-
- @Override
- public void deleteAllMetadataRecords(String prefix) {
- Objects.requireNonNull(prefix, "prefix must not be null");
-
- Stopwatch watch = Stopwatch.createStarted();
- ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
- try {
- Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
-
- CloudBlobDirectory metaDir = getAzureContainer().getDirectoryReference(AZURE_BlOB_META_DIR_NAME);
- int total = 0;
- for (ListBlobItem item : metaDir.listBlobs(prefix)) {
- if (item instanceof CloudBlob) {
- if (((CloudBlob)item).deleteIfExists()) {
- total++;
- }
- }
- }
- LOG.debug("Metadata records deleted. recordsDeleted={} metadataFolder={} duration={}",
- total, prefix, watch.elapsed(TimeUnit.MILLISECONDS));
-
- }
- catch (StorageException | DataStoreException | URISyntaxException e) {
- LOG.debug("Error deleting all metadata records. metadataFolder={}", prefix, e);
- }
- finally {
- if (contextClassLoader != null) {
- Thread.currentThread().setContextClassLoader(contextClassLoader);
- }
- }
- }
-
- @Override
- public boolean metadataRecordExists(String name) {
- Stopwatch watch = Stopwatch.createStarted();
- ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
- try {
- Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
- CloudBlockBlob blob = getAzureContainer().getBlockBlobReference(addMetaKeyPrefix(name));
- boolean exists = blob.exists();
- LOG.debug("Metadata record {} exists {}. duration={}", name, exists, watch.elapsed(TimeUnit.MILLISECONDS));
- return exists;
- }
- catch (DataStoreException | StorageException | URISyntaxException e) {
- LOG.debug("Error checking existence of metadata record = {}", name, e);
- }
- finally {
- if (contextClassLoader != null) {
- Thread.currentThread().setContextClassLoader(contextClassLoader);
- }
- }
- return false;
- }
-
- /**
- * Get key from data identifier. Object is stored with key in ADS.
- */
- private static String getKeyName(DataIdentifier identifier) {
- String key = identifier.toString();
- return key.substring(0, 4) + UtilsV8.DASH + key.substring(4);
- }
-
- /**
- * Get data identifier from key.
- */
- private static String getIdentifierName(String key) {
- if (!key.contains(UtilsV8.DASH)) {
- return null;
- } else if (key.contains(AZURE_BLOB_META_KEY_PREFIX)) {
- return key;
- }
- return key.substring(0, 4) + key.substring(5);
- }
-
- private static String addMetaKeyPrefix(final String key) {
- return AZURE_BLOB_META_KEY_PREFIX + key;
- }
-
- private static String stripMetaKeyPrefix(String name) {
- if (name.startsWith(AZURE_BLOB_META_KEY_PREFIX)) {
- return name.substring(AZURE_BLOB_META_KEY_PREFIX.length());
- }
- return name;
- }
-
- private static void addLastModified(CloudBlockBlob blob) {
- blob.getMetadata().put(AZURE_BLOB_LAST_MODIFIED_KEY, String.valueOf(System.currentTimeMillis()));
- }
-
- private static long getLastModified(CloudBlob blob) {
- if (blob.getMetadata().containsKey(AZURE_BLOB_LAST_MODIFIED_KEY)) {
- return Long.parseLong(blob.getMetadata().get(AZURE_BLOB_LAST_MODIFIED_KEY));
- }
- return blob.getProperties().getLastModified().getTime();
- }
-
- protected void setHttpDownloadURIExpirySeconds(int seconds) {
- httpDownloadURIExpirySeconds = seconds;
- }
-
- protected void setHttpDownloadURICacheSize(int maxSize) {
- // max size 0 or smaller is used to turn off the cache
- if (maxSize > 0) {
- LOG.info("presigned GET URI cache enabled, maxSize = {} items, expiry = {} seconds", maxSize, httpDownloadURIExpirySeconds / 2);
- httpDownloadURICache = CacheBuilder.<String, URI>newBuilder()
- .maximumSize(maxSize)
- .expireAfterWrite(Duration.ofSeconds(httpDownloadURIExpirySeconds / 2))
- .build();
- } else {
- LOG.info("presigned GET URI cache disabled");
- httpDownloadURICache = null;
- }
- }
-
- @Override
- protected URI createHttpDownloadURI(@NotNull DataIdentifier identifier,
- @NotNull DataRecordDownloadOptions downloadOptions) {
- URI uri = null;
-
- Objects.requireNonNull(identifier, "identifier must not be null");
- Objects.requireNonNull(downloadOptions, "downloadOptions must not be null");
-
- if (httpDownloadURIExpirySeconds > 0) {
-
- String domain = getDirectDownloadBlobStorageDomain(downloadOptions.isDomainOverrideIgnored());
- Objects.requireNonNull(domain, "Could not determine domain for direct download");
-
- String cacheKey = identifier
- + domain
- + Objects.toString(downloadOptions.getContentTypeHeader(), "")
- + Objects.toString(downloadOptions.getContentDispositionHeader(), "");
- if (httpDownloadURICache != null) {
- uri = httpDownloadURICache.getIfPresent(cacheKey);
- }
- if (uri == null) {
- if (presignedDownloadURIVerifyExists) {
- // Check if this identifier exists. If not, we want to return null
- // even if the identifier is in the download URI cache.
- try {
- if (!exists(identifier)) {
- LOG.warn("Cannot create download URI for nonexistent blob {}; returning null", getKeyName(identifier));
- return null;
- }
- } catch (DataStoreException e) {
- LOG.warn("Cannot create download URI for blob {} (caught DataStoreException); returning null", getKeyName(identifier), e);
- return null;
- }
- }
-
- String key = getKeyName(identifier);
- SharedAccessBlobHeaders headers = new SharedAccessBlobHeaders();
- headers.setCacheControl(String.format("private, max-age=%d, immutable", httpDownloadURIExpirySeconds));
-
- String contentType = downloadOptions.getContentTypeHeader();
- if (!Objects.toString(contentType, "").isEmpty()) {
- headers.setContentType(contentType);
- }
-
- String contentDisposition =
- downloadOptions.getContentDispositionHeader();
- if (!Objects.toString(contentDisposition, "").isEmpty()) {
- headers.setContentDisposition(contentDisposition);
- }
-
- uri = createPresignedURI(key,
- EnumSet.of(SharedAccessBlobPermissions.READ),
- httpDownloadURIExpirySeconds,
- headers,
- domain);
- if (uri != null && httpDownloadURICache != null) {
- httpDownloadURICache.put(cacheKey, uri);
- }
- }
- }
- return uri;
- }
-
- protected void setHttpUploadURIExpirySeconds(int seconds) { httpUploadURIExpirySeconds = seconds; }
-
- private DataIdentifier generateSafeRandomIdentifier() {
- return new DataIdentifier(
- String.format("%s-%d",
- UUID.randomUUID().toString(),
- Instant.now().toEpochMilli()
- )
- );
- }
-
- protected DataRecordUpload initiateHttpUpload(long maxUploadSizeInBytes, int maxNumberOfURIs, @NotNull final DataRecordUploadOptions options) {
- List<URI> uploadPartURIs = new ArrayList<>();
- long minPartSize = AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE;
- long maxPartSize = AZURE_BLOB_MAX_MULTIPART_UPLOAD_PART_SIZE;
-
- Validate.checkArgument(maxUploadSizeInBytes > 0L, "maxUploadSizeInBytes must be > 0");
- Validate.checkArgument(maxNumberOfURIs > 0 || maxNumberOfURIs == -1, "maxNumberOfURIs must either be > 0 or -1");
- Validate.checkArgument(!(maxUploadSizeInBytes > AZURE_BLOB_MAX_SINGLE_PUT_UPLOAD_SIZE && maxNumberOfURIs == 1), "Cannot do single-put upload with file size %d - exceeds max single-put upload size of %d", maxUploadSizeInBytes, AZURE_BLOB_MAX_SINGLE_PUT_UPLOAD_SIZE);
- Validate.checkArgument(maxUploadSizeInBytes <= AZURE_BLOB_MAX_BINARY_UPLOAD_SIZE, "Cannot do upload with file size %d - exceeds max upload size of %d", maxUploadSizeInBytes, AZURE_BLOB_MAX_BINARY_UPLOAD_SIZE);
-
- DataIdentifier newIdentifier = generateSafeRandomIdentifier();
- String blobId = getKeyName(newIdentifier);
- String uploadId = null;
-
- if (httpUploadURIExpirySeconds > 0) {
- // Always do multi-part uploads for Azure, even for small binaries.
- //
- // This is because Azure requires a unique header, "x-ms-blob-type=BlockBlob", to be
- // set but only for single-put uploads, not multi-part.
- // This would require clients to know not only the type of service provider being used
- // but also the type of upload (single-put vs multi-part), which breaks abstraction.
- // Instead we can insist that clients always do multi-part uploads to Azure, even
- // if the multi-part upload consists of only one upload part. This doesn't require
- // additional work on the part of the client since the "complete" request must always
- // be sent regardless, but it helps us avoid the client having to know what type
- // of provider is being used, or us having to instruct the client to use specific
- // types of headers, etc.
-
- // Azure doesn't use upload IDs like AWS does
- // Generate a fake one for compatibility - we use them to determine whether we are
- // doing multi-part or single-put upload
- uploadId = Base64.encode(UUID.randomUUID().toString());
-
- long numParts = 0L;
- if (maxNumberOfURIs > 0) {
- long requestedPartSize = (long) Math.ceil(((double) maxUploadSizeInBytes) / ((double) maxNumberOfURIs));
- if (requestedPartSize <= maxPartSize) {
- numParts = Math.min(
- maxNumberOfURIs,
- Math.min(
- (long) Math.ceil(((double) maxUploadSizeInBytes) / ((double) minPartSize)),
- AZURE_BLOB_MAX_ALLOWABLE_UPLOAD_URIS
- )
- );
- } else {
- throw new IllegalArgumentException(
- String.format("Cannot do multi-part upload with requested part size %d", requestedPartSize)
- );
- }
- }
- else {
- long maximalNumParts = (long) Math.ceil(((double) maxUploadSizeInBytes) / ((double) AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE));
- numParts = Math.min(maximalNumParts, AZURE_BLOB_MAX_ALLOWABLE_UPLOAD_URIS);
- }
-
- String key = getKeyName(newIdentifier);
- String domain = getDirectUploadBlobStorageDomain(options.isDomainOverrideIgnored());
- Objects.requireNonNull(domain, "Could not determine domain for direct upload");
-
- EnumSet<SharedAccessBlobPermissions> perms = EnumSet.of(SharedAccessBlobPermissions.WRITE);
- Map<String, String> presignedURIRequestParams = new HashMap<>();
- // see https://docs.microsoft.com/en-us/rest/api/storageservices/put-block#uri-parameters
- presignedURIRequestParams.put("comp", "block");
- for (long blockId = 1; blockId <= numParts; ++blockId) {
- presignedURIRequestParams.put("blockid",
- Base64.encode(String.format("%06d", blockId)));
- uploadPartURIs.add(
- createPresignedURI(key,
- perms,
- httpUploadURIExpirySeconds,
- presignedURIRequestParams,
- domain)
- );
- }
-
- try {
- byte[] secret = getOrCreateReferenceKey();
- String uploadToken = new DataRecordUploadToken(blobId, uploadId).getEncodedToken(secret);
- return new DataRecordUpload() {
- @Override
- @NotNull
- public String getUploadToken() {
- return uploadToken;
- }
-
- @Override
- public long getMinPartSize() {
- return minPartSize;
- }
-
- @Override
- public long getMaxPartSize() {
- return maxPartSize;
- }
-
- @Override
- @NotNull
- public Collection<URI> getUploadURIs() {
- return uploadPartURIs;
- }
- };
- } catch (DataStoreException e) {
- LOG.warn("Unable to obtain data store key");
- }
- }
-
- return null;
- }
-
- protected DataRecord completeHttpUpload(@NotNull String uploadTokenStr)
- throws DataRecordUploadException, DataStoreException {
-
- Validate.checkArgument(StringUtils.isNotEmpty(uploadTokenStr), "uploadToken required");
-
- DataRecordUploadToken uploadToken = DataRecordUploadToken.fromEncodedToken(uploadTokenStr, getOrCreateReferenceKey());
- String key = uploadToken.getBlobId();
- DataIdentifier blobId = new DataIdentifier(getIdentifierName(key));
-
- DataRecord record = null;
- try {
- record = getRecord(blobId);
- // If this succeeds this means either it was a "single put" upload
- // (we don't need to do anything in this case - blob is already uploaded)
- // or it was completed before with the same token.
- }
- catch (DataStoreException e1) {
- // record doesn't exist - so this means we are safe to do the complete request
- try {
- if (uploadToken.getUploadId().isPresent()) {
- CloudBlockBlob blob = getAzureContainer().getBlockBlobReference(key);
- // An existing upload ID means this is a multi-part upload
- List<BlockEntry> blocks = blob.downloadBlockList(
- BlockListingFilter.UNCOMMITTED,
- AccessCondition.generateEmptyCondition(),
- null,
- null);
- addLastModified(blob);
- blob.commitBlockList(blocks);
- long size = 0L;
- for (BlockEntry block : blocks) {
- size += block.getSize();
- }
- record = new AzureBlobStoreDataRecord(
- this,
- azureBlobContainerProvider,
- blobId,
- getLastModified(blob),
- size);
- }
- else {
- // Something is wrong - upload ID missing from upload token
- // but record doesn't exist already, so this is invalid
- throw new DataRecordUploadException(
- String.format("Unable to finalize direct write of binary %s - upload ID missing from upload token",
- blobId)
- );
- }
- } catch (URISyntaxException | StorageException e2) {
- throw new DataRecordUploadException(
- String.format("Unable to finalize direct write of binary %s", blobId),
- e2
- );
- }
- }
-
- return record;
- }
-
- private String getDefaultBlobStorageDomain() {
- String accountName = properties.getProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, "");
- if (Objects.toString(accountName, "").isEmpty()) {
- LOG.warn("Can't generate presigned URI - Azure account name not found in properties");
- return null;
- }
- return String.format("%s.blob.core.windows.net", accountName);
- }
-
- private String getDirectDownloadBlobStorageDomain(boolean ignoreDomainOverride) {
- String domain = ignoreDomainOverride
- ? getDefaultBlobStorageDomain()
- : downloadDomainOverride;
- if (Objects.toString(domain, "").isEmpty()) {
- domain = getDefaultBlobStorageDomain();
- }
- return domain;
- }
-
- private String getDirectUploadBlobStorageDomain(boolean ignoreDomainOverride) {
- String domain = ignoreDomainOverride
- ? getDefaultBlobStorageDomain()
- : uploadDomainOverride;
- if (Objects.toString(domain, "").isEmpty()) {
- domain = getDefaultBlobStorageDomain();
- }
- return domain;
- }
-
- private URI createPresignedURI(String key,
- EnumSet<SharedAccessBlobPermissions> permissions,
- int expirySeconds,
- SharedAccessBlobHeaders optionalHeaders,
- String domain) {
- return createPresignedURI(key, permissions, expirySeconds, new HashMap<>(), optionalHeaders, domain);
- }
-
- private URI createPresignedURI(String key,
- EnumSet<SharedAccessBlobPermissions> permissions,
- int expirySeconds,
- Map<String, String> additionalQueryParams,
- String domain) {
- return createPresignedURI(key, permissions, expirySeconds, additionalQueryParams, null, domain);
- }
-
- private URI createPresignedURI(String key,
- EnumSet<SharedAccessBlobPermissions> permissions,
- int expirySeconds,
- Map<String, String> additionalQueryParams,
- SharedAccessBlobHeaders optionalHeaders,
- String domain) {
- if (Objects.toString(domain, "").isEmpty()) {
- LOG.warn("Can't generate presigned URI - no Azure domain provided (is Azure account name configured?)");
- return null;
- }
-
- URI presignedURI = null;
- try {
- String sharedAccessSignature = azureBlobContainerProvider.generateSharedAccessSignature(getBlobRequestOptions(), key,
- permissions, expirySeconds, optionalHeaders);
-
- // Shared access signature is returned encoded already.
- String uriString = String.format("https://%s/%s/%s?%s",
- domain,
- getContainerName(),
- key,
- sharedAccessSignature);
-
- if (! additionalQueryParams.isEmpty()) {
- StringBuilder builder = new StringBuilder();
- for (Map.Entry<String, String> e : additionalQueryParams.entrySet()) {
- builder.append("&");
- builder.append(URLEncoder.encode(e.getKey(), StandardCharsets.UTF_8));
- builder.append("=");
- builder.append(URLEncoder.encode(e.getValue(), StandardCharsets.UTF_8));
- }
- uriString += builder.toString();
- }
-
- presignedURI = new URI(uriString);
- }
- catch (DataStoreException e) {
- LOG.error("Error calling Azure Blob Storage", e);
- }
- catch (URISyntaxException | InvalidKeyException e) {
- LOG.error("Can't generate a presigned URI for key {}", key, e);
- }
- catch (StorageException e) {
- LOG.error("Azure request to create presigned Azure Blob Storage {} URI failed. " +
- "Key: {}, Error: {}, HTTP Code: {}, Azure Error Code: {}",
- permissions.contains(SharedAccessBlobPermissions.READ) ? "GET" :
- (permissions.contains(SharedAccessBlobPermissions.WRITE) ? "PUT" : ""),
- key,
- e.getMessage(),
- e.getHttpStatusCode(),
- e.getErrorCode());
- }
-
- return presignedURI;
- }
-
- private static class AzureBlobInfo {
- private final String name;
- private final long lastModified;
- private final long length;
-
- public AzureBlobInfo(String name, long lastModified, long length) {
- this.name = name;
- this.lastModified = lastModified;
- this.length = length;
- }
-
- public String getName() {
- return name;
- }
-
- public long getLastModified() {
- return lastModified;
- }
-
- public long getLength() {
- return length;
- }
-
- public static AzureBlobInfo fromCloudBlob(CloudBlob cloudBlob) throws StorageException {
- cloudBlob.downloadAttributes();
- return new AzureBlobInfo(cloudBlob.getName(),
- AzureBlobStoreBackendV8.getLastModified(cloudBlob),
- cloudBlob.getProperties().getLength());
- }
- }
-
- private class RecordsIterator<T> extends AbstractIterator<T> {
- // Seems to be thread-safe (in 5.0.0)
- ResultContinuation resultContinuation;
- boolean firstCall = true;
- final Function<AzureBlobInfo, T> transformer;
- final Queue<AzureBlobInfo> items = new LinkedList<>();
-
- public RecordsIterator (Function<AzureBlobInfo, T> transformer) {
- this.transformer = transformer;
- }
-
- @Override
- protected T computeNext() {
- if (items.isEmpty()) {
- loadItems();
- }
- if (!items.isEmpty()) {
- return transformer.apply(items.remove());
- }
- return endOfData();
- }
-
- private boolean loadItems() {
- Stopwatch watch = Stopwatch.createStarted();
- ClassLoader contextClassLoader = currentThread().getContextClassLoader();
- try {
- currentThread().setContextClassLoader(getClass().getClassLoader());
-
- CloudBlobContainer container = azureBlobContainerProvider.getBlobContainer();
- if (!firstCall && (resultContinuation == null || !resultContinuation.hasContinuation())) {
- LOG.trace("No more records in container. containerName={}", container);
- return false;
- }
- firstCall = false;
- ResultSegment<ListBlobItem> results = container.listBlobsSegmented(null, false, EnumSet.noneOf(BlobListingDetails.class), null, resultContinuation, null, null);
- resultContinuation = results.getContinuationToken();
- for (ListBlobItem item : results.getResults()) {
- if (item instanceof CloudBlob) {
- items.add(AzureBlobInfo.fromCloudBlob((CloudBlob)item));
- }
- }
- LOG.debug("Container records batch read. batchSize={} containerName={} duration={}",
- results.getLength(), getContainerName(), watch.elapsed(TimeUnit.MILLISECONDS));
- return results.getLength() > 0;
- }
- catch (StorageException | DataStoreException e) {
- LOG.error("Error listing blobs. containerName={}", getContainerName(), e);
- } finally {
- if (contextClassLoader != null) {
- currentThread().setContextClassLoader(contextClassLoader);
- }
- }
- return false;
- }
- }
-
- static class AzureBlobStoreDataRecord extends AbstractDataRecord {
- final AzureBlobContainerProviderV8 azureBlobContainerProvider;
- final long lastModified;
- final long length;
- final boolean isMeta;
-
- public AzureBlobStoreDataRecord(AbstractSharedBackend backend, AzureBlobContainerProviderV8 azureBlobContainerProvider,
- DataIdentifier key, long lastModified, long length) {
- this(backend, azureBlobContainerProvider, key, lastModified, length, false);
- }
-
- public AzureBlobStoreDataRecord(AbstractSharedBackend backend, AzureBlobContainerProviderV8 azureBlobContainerProvider,
- DataIdentifier key, long lastModified, long length, boolean isMeta) {
- super(backend, key);
- this.azureBlobContainerProvider = azureBlobContainerProvider;
- this.lastModified = lastModified;
- this.length = length;
- this.isMeta = isMeta;
- }
-
- @Override
- public long getLength() {
- return length;
- }
-
- @Override
- public InputStream getStream() throws DataStoreException {
- String id = getKeyName(getIdentifier());
- CloudBlobContainer container = azureBlobContainerProvider.getBlobContainer();
- if (isMeta) {
- id = addMetaKeyPrefix(getIdentifier().toString());
- }
- else {
- // Don't worry about stream logging for metadata records
- if (LOG_STREAMS_DOWNLOAD.isDebugEnabled()) {
- // Log message, with exception so we can get a trace to see where the call came from
- LOG_STREAMS_DOWNLOAD.debug("Binary downloaded from Azure Blob Storage - identifier={} ", id, new Exception());
- }
- }
- try {
- return container.getBlockBlobReference(id).openInputStream();
- } catch (StorageException | URISyntaxException e) {
- throw new DataStoreException(e);
- }
- }
-
- @Override
- public long getLastModified() {
- return lastModified;
- }
-
- @Override
- public String toString() {
- return "AzureBlobStoreDataRecord{" +
- "identifier=" + getIdentifier() +
- ", length=" + length +
- ", lastModified=" + lastModified +
- ", containerName='" + Optional.ofNullable(azureBlobContainerProvider).map(AzureBlobContainerProviderV8::getContainerName).orElse(null) + '\'' +
- '}';
- }
- }
-
- private String getContainerName() {
- return Optional.ofNullable(this.azureBlobContainerProvider)
- .map(AzureBlobContainerProviderV8::getContainerName)
- .orElse(null);
- }
-
- @Override
- public byte[] getOrCreateReferenceKey() throws DataStoreException {
- try {
- if (secret != null && secret.length != 0) {
- return secret;
- } else {
- byte[] key;
- // Try reading from the metadata folder if it exists
- key = readMetadataBytes(AZURE_BLOB_REF_KEY);
- if (key == null) {
- key = super.getOrCreateReferenceKey();
- addMetadataRecord(new ByteArrayInputStream(key), AZURE_BLOB_REF_KEY);
- key = readMetadataBytes(AZURE_BLOB_REF_KEY);
- }
- secret = key;
- return secret;
- }
- } catch (IOException e) {
- throw new DataStoreException("Unable to get or create key " + e);
- }
- }
-
- protected byte[] readMetadataBytes(String name) throws IOException, DataStoreException {
- DataRecord rec = getMetadataRecord(name);
- byte[] key = null;
- if (rec != null) {
- InputStream stream = null;
- try {
- stream = rec.getStream();
- return IOUtils.toByteArray(stream);
- } finally {
- IOUtils.closeQuietly(stream);
- }
- }
- return key;
- }
-
-}
diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/UtilsV8.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/UtilsV8.java
deleted file mode 100644
index c0b5e92cc0..0000000000
--- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/UtilsV8.java
+++ /dev/null
@@ -1,167 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v8;
-
-import com.microsoft.azure.storage.CloudStorageAccount;
-import com.microsoft.azure.storage.OperationContext;
-import com.microsoft.azure.storage.RetryExponentialRetry;
-import com.microsoft.azure.storage.RetryNoRetry;
-import com.microsoft.azure.storage.RetryPolicy;
-import com.microsoft.azure.storage.StorageException;
-import com.microsoft.azure.storage.blob.BlobRequestOptions;
-import com.microsoft.azure.storage.blob.CloudBlobClient;
-import com.microsoft.azure.storage.blob.CloudBlobContainer;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.jackrabbit.oak.spi.blob.data.DataStoreException;
-import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants;
-import org.apache.jackrabbit.oak.commons.PropertiesUtil;
-import org.jetbrains.annotations.NotNull;
-import org.jetbrains.annotations.Nullable;
-
-import java.net.InetSocketAddress;
-import java.net.Proxy;
-import java.net.SocketAddress;
-import java.net.URISyntaxException;
-import java.security.InvalidKeyException;
-import java.util.Objects;
-import java.util.Properties;
-
-public final class UtilsV8 {
-
- public static final String DEFAULT_CONFIG_FILE = "azure.properties";
-
- public static final String DASH = "-";
-
- /**
- * private constructor so that class cannot initialized from outside.
- */
- private UtilsV8() {
- }
-
- /**
- * Create CloudBlobClient from properties.
- *
- * @param connectionString connectionString to configure @link {@link CloudBlobClient}
- * @return {@link CloudBlobClient}
- */
- public static CloudBlobClient getBlobClient(@NotNull final String connectionString) throws URISyntaxException, InvalidKeyException {
- return getBlobClient(connectionString, null);
- }
-
- public static CloudBlobClient getBlobClient(@NotNull final String connectionString,
- @Nullable final BlobRequestOptions requestOptions) throws URISyntaxException, InvalidKeyException {
- CloudStorageAccount account = CloudStorageAccount.parse(connectionString);
- CloudBlobClient client = account.createCloudBlobClient();
- if (requestOptions != null) {
- client.setDefaultRequestOptions(requestOptions);
- }
- return client;
- }
-
- public static CloudBlobContainer getBlobContainer(@NotNull final String connectionString,
- @NotNull final String containerName) throws DataStoreException {
- return getBlobContainer(connectionString, containerName, null);
- }
-
-
- public static CloudBlobContainer getBlobContainer(@NotNull final String connectionString,
- @NotNull final String containerName,
- @Nullable final BlobRequestOptions requestOptions) throws DataStoreException {
- try {
- CloudBlobClient client = (
- (requestOptions == null)
- ? UtilsV8.getBlobClient(connectionString)
- : UtilsV8.getBlobClient(connectionString, requestOptions)
- );
- return client.getContainerReference(containerName);
- } catch (InvalidKeyException | URISyntaxException | StorageException e) {
- throw new DataStoreException(e);
- }
- }
-
- public static void setProxyIfNeeded(final Properties properties) {
- String proxyHost = properties.getProperty(AzureConstants.PROXY_HOST);
- String proxyPort = properties.getProperty(AzureConstants.PROXY_PORT);
-
- if (!(Objects.toString(proxyHost, "").isEmpty() ||
- Objects.toString(proxyPort, "").isEmpty())) {
- int port = Integer.parseInt(proxyPort);
- SocketAddress proxyAddr = new InetSocketAddress(proxyHost, port);
- Proxy proxy = new Proxy(Proxy.Type.HTTP, proxyAddr);
- OperationContext.setDefaultProxy(proxy);
- }
- }
-
- public static String getConnectionStringFromProperties(Properties properties) {
-
- String sasUri = properties.getProperty(AzureConstants.AZURE_SAS, "");
- String blobEndpoint = properties.getProperty(AzureConstants.AZURE_BLOB_ENDPOINT, "");
- String connectionString = properties.getProperty(AzureConstants.AZURE_CONNECTION_STRING, "");
- String accountName = properties.getProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, "");
- String accountKey = properties.getProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_KEY, "");
-
- if (!connectionString.isEmpty()) {
- return connectionString;
- }
-
- if (!sasUri.isEmpty()) {
- return getConnectionStringForSas(sasUri, blobEndpoint, accountName);
- }
-
- return getConnectionString(
- accountName,
- accountKey,
- blobEndpoint);
- }
-
- public static String getConnectionStringForSas(String sasUri, String blobEndpoint, String accountName) {
- if (StringUtils.isEmpty(blobEndpoint)) {
- return String.format("AccountName=%s;SharedAccessSignature=%s", accountName, sasUri);
- } else {
- return String.format("BlobEndpoint=%s;SharedAccessSignature=%s", blobEndpoint, sasUri);
- }
- }
-
- public static String getConnectionString(final String accountName, final String accountKey) {
- return getConnectionString(accountName, accountKey, null);
- }
-
- public static String getConnectionString(final String accountName, final String accountKey, String blobEndpoint) {
- StringBuilder connString = new StringBuilder("DefaultEndpointsProtocol=https");
- connString.append(";AccountName=").append(accountName);
- connString.append(";AccountKey=").append(accountKey);
-
- if (!Objects.toString(blobEndpoint, "").isEmpty()) {
- connString.append(";BlobEndpoint=").append(blobEndpoint);
- }
- return connString.toString();
- }
-
- public static RetryPolicy getRetryPolicy(final String maxRequestRetry) {
- int retries = PropertiesUtil.toInteger(maxRequestRetry, -1);
- if (retries < 0) {
- return null;
- }
- else if (retries == 0) {
- return new RetryNoRetry();
- }
- return new RetryExponentialRetry(RetryPolicy.DEFAULT_CLIENT_BACKOFF, retries);
- }
-}
diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobContainerProviderTest.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobContainerProviderTest.java
deleted file mode 100644
index e5d40a164a..0000000000
--- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobContainerProviderTest.java
+++ /dev/null
@@ -1,889 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;
-
-import com.azure.identity.ClientSecretCredential;
-import com.azure.storage.blob.BlobContainerClient;
-import com.azure.storage.blob.sas.BlobSasPermission;
-import com.azure.storage.blob.specialized.BlockBlobClient;
-import com.azure.storage.common.policy.RequestRetryOptions;
-import com.microsoft.aad.msal4j.MsalServiceException;
-
-import org.apache.commons.lang3.reflect.MethodUtils;
-import org.apache.jackrabbit.oak.spi.blob.data.DataStoreException;
-import org.junit.ClassRule;
-import org.junit.Test;
-
-import java.lang.reflect.Field;
-import java.net.URISyntaxException;
-import java.security.InvalidKeyException;
-import java.time.OffsetDateTime;
-import java.util.Properties;
-
-import static org.junit.Assert.*;
-import static org.mockito.Mockito.*;
-
-public class AzureBlobContainerProviderTest {
-
- @ClassRule
- public static AzuriteDockerRule azurite = new AzuriteDockerRule();
-
- private static final String CONTAINER_NAME = "test-container";
- private AzureBlobContainerProvider provider;
-
- @Test
- public void testBuilderWithConnectionString() {
- String connectionString = getConnectionString();
-
- provider = AzureBlobContainerProvider.Builder
- .builder(CONTAINER_NAME)
- .withAzureConnectionString(connectionString)
- .build();
-
- assertNotNull("Provider should not be null", provider);
- assertEquals("Container name should match", CONTAINER_NAME, provider.getContainerName());
- assertEquals("Connection string should match", connectionString, provider.getAzureConnectionString());
- }
-
- @Test
- public void testBuilderWithAccountNameAndKey() {
- provider = AzureBlobContainerProvider.Builder
- .builder(CONTAINER_NAME)
- .withAccountName("testaccount")
- .withAccountKey("testkey")
- .build();
-
- assertNotNull("Provider should not be null", provider);
- assertEquals("Container name should match", CONTAINER_NAME, provider.getContainerName());
- }
-
- @Test
- public void testBuilderWithServicePrincipal() {
- provider = AzureBlobContainerProvider.Builder
- .builder(CONTAINER_NAME)
- .withAccountName("testaccount")
- .withTenantId("tenant-id")
- .withClientId("client-id")
- .withClientSecret("client-secret")
- .build();
-
- assertNotNull("Provider should not be null", provider);
- assertEquals("Container name should match", CONTAINER_NAME, provider.getContainerName());
- }
-
- @Test
- public void testBuilderWithSasToken() {
- provider = AzureBlobContainerProvider.Builder
- .builder(CONTAINER_NAME)
- .withAccountName("testaccount")
- .withSasToken("sas-token")
- .withBlobEndpoint("https://testaccount.blob.core.windows.net")
- .build();
-
- assertNotNull("Provider should not be null", provider);
- assertEquals("Container name should match", CONTAINER_NAME, provider.getContainerName());
- }
-
- @Test
- public void testBuilderInitializeWithProperties() {
- Properties properties = new Properties();
- properties.setProperty(AzureConstants.AZURE_CONNECTION_STRING, getConnectionString());
- properties.setProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, "testaccount");
- properties.setProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_KEY, "testkey");
- properties.setProperty(AzureConstants.AZURE_TENANT_ID, "tenant-id");
- properties.setProperty(AzureConstants.AZURE_CLIENT_ID, "client-id");
- properties.setProperty(AzureConstants.AZURE_CLIENT_SECRET, "client-secret");
- properties.setProperty(AzureConstants.AZURE_SAS, "sas-token");
- properties.setProperty(AzureConstants.AZURE_BLOB_ENDPOINT, "https://testaccount.blob.core.windows.net");
-
- provider = AzureBlobContainerProvider.Builder
- .builder(CONTAINER_NAME)
- .initializeWithProperties(properties)
- .build();
-
- assertNotNull("Provider should not be null", provider);
- assertEquals("Container name should match", CONTAINER_NAME, provider.getContainerName());
- assertEquals("Connection string should match", getConnectionString(), provider.getAzureConnectionString());
- }
-
- @Test
- public void testGetBlobContainerWithConnectionString() throws DataStoreException {
- String connectionString = getConnectionString();
-
- provider = AzureBlobContainerProvider.Builder
- .builder(CONTAINER_NAME)
- .withAzureConnectionString(connectionString)
- .build();
-
- BlobContainerClient containerClient = provider.getBlobContainer();
- assertNotNull("Container client should not be null", containerClient);
- assertEquals("Container name should match", CONTAINER_NAME, containerClient.getBlobContainerName());
- }
-
- @Test
- public void testGetBlobContainerWithRetryOptions() throws DataStoreException {
- String connectionString = getConnectionString();
- RequestRetryOptions retryOptions = new RequestRetryOptions();
- Properties properties = new Properties();
-
- provider = AzureBlobContainerProvider.Builder
- .builder(CONTAINER_NAME)
- .withAzureConnectionString(connectionString)
- .build();
-
- BlobContainerClient containerClient = provider.getBlobContainer(retryOptions, properties);
- assertNotNull("Container client should not be null", containerClient);
- assertEquals("Container name should match", CONTAINER_NAME, containerClient.getBlobContainerName());
- }
-
- @Test
- public void testBuilderWithNullContainerName() {
- // Builder accepts null container name - no validation
- AzureBlobContainerProvider.Builder builder = AzureBlobContainerProvider.Builder.builder(null);
- assertNotNull("Builder should not be null", builder);
- }
-
- @Test
- public void testBuilderWithEmptyContainerName() {
- // Builder accepts empty container name - no validation
- AzureBlobContainerProvider.Builder builder = AzureBlobContainerProvider.Builder.builder("");
- assertNotNull("Builder should not be null", builder);
- }
-
- @Test
- public void testGenerateSharedAccessSignatureWithConnectionString() {
- String connectionString = getConnectionString();
-
- provider = AzureBlobContainerProvider.Builder
- .builder(CONTAINER_NAME)
- .withAzureConnectionString(connectionString)
- .build();
-
- try {
- String sas = provider.generateSharedAccessSignature(
- null,
- "test-blob",
- new BlobSasPermission().setReadPermission(true),
- 3600,
- new Properties()
- );
- assertNotNull("SAS token should not be null", sas);
- assertFalse("SAS token should not be empty", sas.isEmpty());
- } catch (Exception e) {
- // Expected for Azurite as it may not support all SAS features
- assertTrue("Should be DataStoreException, URISyntaxException, or InvalidKeyException",
- e instanceof DataStoreException ||
- e instanceof URISyntaxException ||
- e instanceof InvalidKeyException);
- }
- }
-
- @Test
- public void testGetBlobContainerWithInvalidConnectionString() {
- provider = AzureBlobContainerProvider.Builder
- .builder(CONTAINER_NAME)
- .withAzureConnectionString("invalid-connection-string")
- .build();
-
- try {
- provider.getBlobContainer();
- fail("Expected exception with invalid connection string");
- } catch (Exception e) {
- // Expected - can be DataStoreException or IllegalArgumentException
- assertNotNull("Exception should not be null", e);
- assertTrue("Should be DataStoreException or IllegalArgumentException",
- e instanceof DataStoreException || e instanceof IllegalArgumentException);
- }
- }
-
- @Test
- public void testGetBlobContainerWithServicePrincipalMissingCredentials() throws DataStoreException{
- provider = AzureBlobContainerProvider.Builder
- .builder(CONTAINER_NAME)
- .withAccountName("testaccount")
- .withTenantId("tenant-id")
- .withClientId("client-id")
- // Missing client secret
- .build();
-
- BlobContainerClient containerClient = provider.getBlobContainer();
- // May succeed with incomplete credentials - Azure SDK might handle it differently
- assertNotNull("Container client should not be null", containerClient);
- }
-
- @Test(expected = DataStoreException.class)
- public void testGetBlobContainerWithSasTokenMissingEndpoint() throws DataStoreException{
- provider = AzureBlobContainerProvider.Builder
- .builder(CONTAINER_NAME)
- .withAccountName("testaccount")
- .withSasToken("sas-token")
- // Missing blob endpoint
- .build();
-
- BlobContainerClient containerClient = provider.getBlobContainer();
- }
-
- @Test
- public void testBuilderChaining() {
- provider = AzureBlobContainerProvider.Builder
- .builder(CONTAINER_NAME)
- .withAzureConnectionString("connection1")
- .withAccountName("account1")
- .withAccountKey("key1")
- .withBlobEndpoint("endpoint1")
- .withSasToken("sas1")
- .withTenantId("tenant1")
- .withClientId("client1")
- .withClientSecret("secret1")
- .build();
-
- assertNotNull("Provider should not be null", provider);
- assertEquals("Container name should match", CONTAINER_NAME, provider.getContainerName());
- assertEquals("Connection string should match", "connection1", provider.getAzureConnectionString());
- }
-
- @Test
- public void testBuilderWithEmptyStrings() {
- provider = AzureBlobContainerProvider.Builder
- .builder(CONTAINER_NAME)
- .withAzureConnectionString("")
- .withAccountName("")
- .withAccountKey("")
- .withBlobEndpoint("")
- .withSasToken("")
- .withTenantId("")
- .withClientId("")
- .withClientSecret("")
- .build();
-
- assertNotNull("Provider should not be null", provider);
- assertEquals("Container name should match", CONTAINER_NAME, provider.getContainerName());
- assertEquals("Connection string should be empty", "", provider.getAzureConnectionString());
- }
-
- @Test
- public void testInitializeWithPropertiesEmptyValues() {
- Properties properties = new Properties();
- properties.setProperty(AzureConstants.AZURE_CONNECTION_STRING, "");
- properties.setProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, "");
- properties.setProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_KEY, "");
-
- provider = AzureBlobContainerProvider.Builder
- .builder(CONTAINER_NAME)
- .initializeWithProperties(properties)
- .build();
-
- assertNotNull("Provider should not be null", provider);
- assertEquals("Connection string should be empty", "", provider.getAzureConnectionString());
- }
-
- @Test
- public void testInitializeWithPropertiesNullValues() {
- Properties properties = new Properties();
- // Properties with null values (getProperty returns default empty string)
-
- provider = AzureBlobContainerProvider.Builder
- .builder(CONTAINER_NAME)
- .initializeWithProperties(properties)
- .build();
-
- assertNotNull("Provider should not be null", provider);
- assertEquals("Connection string should be empty", "", provider.getAzureConnectionString());
- }
-
- @Test
- public void testGetBlobContainerServicePrincipalPath() {
- provider = AzureBlobContainerProvider.Builder
- .builder(CONTAINER_NAME)
- .withAccountName("testaccount")
- .withTenantId("tenant-id")
- .withClientId("client-id")
- .withClientSecret("client-secret")
- .build();
-
- try {
- BlobContainerClient containerClient = provider.getBlobContainer();
- assertNotNull("Container client should not be null", containerClient);
- } catch (Exception e) {
- // Expected for invalid service principal credentials
- assertTrue("Should be DataStoreException or related",
- e instanceof DataStoreException || e.getCause() instanceof Exception);
- }
- }
-
- @Test
- public void testGetBlobContainerSasTokenPath() {
- provider = AzureBlobContainerProvider.Builder
- .builder(CONTAINER_NAME)
- .withAccountName("testaccount")
- .withSasToken("sv=2020-08-04&ss=b&srt=sco&sp=rwdlacx&se=2023-12-31T23:59:59Z&st=2023-01-01T00:00:00Z&spr=https&sig=test")
- .withBlobEndpoint("https://testaccount.blob.core.windows.net")
- .build();
-
- try {
- BlobContainerClient containerClient = provider.getBlobContainer();
- assertNotNull("Container client should not be null", containerClient);
- } catch (Exception e) {
- // Expected for invalid SAS token
- assertTrue("Should be DataStoreException or related",
- e instanceof DataStoreException || e.getCause() instanceof Exception);
- }
- }
-
- @Test
- public void testGenerateSharedAccessSignatureServicePrincipalPath() throws DataStoreException, URISyntaxException, InvalidKeyException{
- provider = AzureBlobContainerProvider.Builder
- .builder(CONTAINER_NAME)
- .withAccountName("testaccount")
- .withTenantId("tenant-id")
- .withClientId("client-id")
- .withClientSecret("client-secret")
- .build();
-
- try {
- String sas = provider.generateSharedAccessSignature(
- null,
- "test-blob",
- new BlobSasPermission().setReadPermission(true),
- 3600,
- new Properties()
- );
-
- //fail if no exception is thrown
- fail("Expected exception for invalid service principal credentials");
- } catch (MsalServiceException e) {
- // Expected for invalid service principal credentials
- //Message should be: AADSTS900023: Specified tenant identifier 'tenant-id' is neither a valid DNS name, nor a valid external domain
- assertTrue("Message should be different", e.getMessage().contains("AADSTS900023: Specified tenant identifier 'tenant-id' is neither a valid DNS name, nor a valid external domain"));
- // Accept any exception as authentication will fail with invalid credentials
- }
- }
-
- @Test
- public void testGenerateSharedAccessSignatureAccountKeyPath() throws DataStoreException, URISyntaxException, InvalidKeyException {
- provider = AzureBlobContainerProvider.Builder
- .builder(CONTAINER_NAME)
- .withAccountName("testaccount")
- .withAccountKey("testkey")
- .build();
-
- String sas = provider.generateSharedAccessSignature(
- null,
- "test-blob",
- new BlobSasPermission().setReadPermission(true),
- 3600,
- new Properties()
- );
- assertNotNull("SAS token should not be null", sas);
- assertFalse("SAS token should not be empty", sas.isEmpty());
- }
-
- @Test
- public void testBuilderStaticMethod() {
- AzureBlobContainerProvider.Builder builder = AzureBlobContainerProvider.Builder.builder("test-container");
- assertNotNull("Builder should not be null", builder);
-
- AzureBlobContainerProvider provider = builder.build();
- assertNotNull("Provider should not be null", provider);
- assertEquals("Container name should match", "test-container", provider.getContainerName());
- }
-
- @Test
- public void testBuilderConstructorAccess() throws Exception {
- // Test that Builder constructor is private by accessing it via reflection
- java.lang.reflect.Constructor<AzureBlobContainerProvider.Builder> constructor =
- AzureBlobContainerProvider.Builder.class.getDeclaredConstructor(String.class);
- assertFalse("Constructor should not be public", constructor.isAccessible());
-
- // Make it accessible and test
- constructor.setAccessible(true);
- AzureBlobContainerProvider.Builder builder = constructor.newInstance("test-container");
- assertNotNull("Builder should not be null", builder);
- }
-
- @Test
- public void testDefaultEndpointSuffixUsage() throws Exception {
- provider = AzureBlobContainerProvider.Builder
- .builder(CONTAINER_NAME)
- .withAccountName("testaccount")
- .withTenantId("tenant-id")
- .withClientId("client-id")
- .withClientSecret("client-secret")
- .build();
-
- // Use reflection to access the DEFAULT_ENDPOINT_SUFFIX constant
- Field defaultEndpointField = AzureBlobContainerProvider.class.getDeclaredField("DEFAULT_ENDPOINT_SUFFIX");
- defaultEndpointField.setAccessible(true);
- String defaultEndpoint = (String) defaultEndpointField.get(null);
- assertEquals("Default endpoint should be core.windows.net", "core.windows.net", defaultEndpoint);
-
- // Test that the endpoint is used in service principal authentication
- RequestRetryOptions retryOptions = new RequestRetryOptions();
-
- try {
- MethodUtils.invokeMethod(provider, true, "getBlobContainerFromServicePrincipals",
- new Object[]{"testaccount", retryOptions},
- new Class<?>[]{String.class, RequestRetryOptions.class});
- } catch (Exception e) {
- // Expected - we're just testing that the method uses the default endpoint
- assertNotNull("Exception should not be null", e);
- }
- }
-
- @Test
- public void testGenerateUserDelegationKeySignedSasWithMockTime() {
- provider = AzureBlobContainerProvider.Builder
- .builder(CONTAINER_NAME)
- .withAccountName("testaccount")
- .withTenantId("tenant-id")
- .withClientId("client-id")
- .withClientSecret("client-secret")
- .build();
-
- // Test with specific time values
- BlockBlobClient mockBlobClient = mock(BlockBlobClient.class);
- OffsetDateTime specificTime = OffsetDateTime.parse("2023-12-31T23:59:59Z");
-
- try {
- provider.generateUserDelegationKeySignedSas(
- mockBlobClient,
- mock(com.azure.storage.blob.sas.BlobServiceSasSignatureValues.class),
- specificTime
- );
- } catch (Exception e) {
- // Expected for authentication failure, but we're testing the time handling
- assertNotNull("Exception should not be null", e);
- }
- }
-
- @Test
- public void testGetBlobContainerWithNullRetryOptions() throws DataStoreException {
- String connectionString = getConnectionString();
-
- provider = AzureBlobContainerProvider.Builder
- .builder(CONTAINER_NAME)
- .withAzureConnectionString(connectionString)
- .build();
-
- // Test with null retry options and empty properties
- BlobContainerClient containerClient = provider.getBlobContainer(null, new Properties());
- assertNotNull("Container client should not be null", containerClient);
- assertEquals("Container name should match", CONTAINER_NAME, containerClient.getBlobContainerName());
- }
-
- @Test
- public void testGetBlobContainerWithEmptyProperties() throws DataStoreException {
- String connectionString = getConnectionString();
-
- provider = AzureBlobContainerProvider.Builder
- .builder(CONTAINER_NAME)
- .withAzureConnectionString(connectionString)
- .build();
-
- // Test with empty properties
- Properties emptyProperties = new Properties();
- BlobContainerClient containerClient = provider.getBlobContainer(new RequestRetryOptions(), emptyProperties);
- assertNotNull("Container client should not be null", containerClient);
- assertEquals("Container name should match", CONTAINER_NAME, containerClient.getBlobContainerName());
- }
-
- @Test
- public void testBuilderWithNullValues() {
- provider = AzureBlobContainerProvider.Builder
- .builder(CONTAINER_NAME)
- .withAzureConnectionString(null)
- .withAccountName(null)
- .withAccountKey(null)
- .withBlobEndpoint(null)
- .withSasToken(null)
- .withTenantId(null)
- .withClientId(null)
- .withClientSecret(null)
- .build();
-
- assertNotNull("Provider should not be null", provider);
- assertEquals("Container name should match", CONTAINER_NAME, provider.getContainerName());
- assertNull("Connection string should be null", provider.getAzureConnectionString());
- }
-
- @Test
- public void testInitializeWithPropertiesNullProperties() {
- // Test with null properties object
- try {
- AzureBlobContainerProvider.Builder.builder(CONTAINER_NAME)
- .initializeWithProperties(null);
- fail("Should throw NullPointerException with null properties");
- } catch (NullPointerException e) {
- // Expected
- assertNotNull("Exception should not be null", e);
- }
- }
-
- @Test
- public void testAuthenticationPriorityOrder() throws Exception {
- // Test that connection string takes priority over other authentication methods
- provider = AzureBlobContainerProvider.Builder
- .builder(CONTAINER_NAME)
- .withAzureConnectionString(getConnectionString())
- .withAccountName("testaccount")
- .withAccountKey("testkey")
- .withSasToken("sas-token")
- .withTenantId("tenant-id")
- .withClientId("client-id")
- .withClientSecret("client-secret")
- .build();
-
- // Should use connection string path
- BlobContainerClient containerClient = provider.getBlobContainer();
- assertNotNull("Container client should not be null", containerClient);
- assertEquals("Container name should match", CONTAINER_NAME, containerClient.getBlobContainerName());
- }
-
- @Test
- public void testGetBlobContainerWithAccountKeyFallback() {
- provider = AzureBlobContainerProvider.Builder
- .builder(CONTAINER_NAME)
- .withAccountName("testaccount")
- .withAccountKey("testkey")
- .withBlobEndpoint("https://testaccount.blob.core.windows.net")
- .build();
-
- try {
- BlobContainerClient containerClient = provider.getBlobContainer();
- assertNotNull("Container client should not be null", containerClient);
- } catch (Exception e) {
- // Expected for invalid credentials
- assertTrue("Should be DataStoreException", e instanceof DataStoreException);
- }
- }
-
- @Test
- public void testAuthenticateViaServicePrincipalTrue() throws Exception {
- provider = AzureBlobContainerProvider.Builder
- .builder(CONTAINER_NAME)
- .withAccountName("testaccount")
- .withTenantId("tenant-id")
- .withClientId("client-id")
- .withClientSecret("client-secret")
- .build();
-
- // Use reflection to test private method
- boolean result = (boolean) MethodUtils.invokeMethod(provider, true, "authenticateViaServicePrincipal");
- assertTrue("Should authenticate via service principal", result);
- }
-
- @Test
- public void testAuthenticateViaServicePrincipalFalseWithConnectionString() throws Exception {
- provider = AzureBlobContainerProvider.Builder
- .builder(CONTAINER_NAME)
- .withAzureConnectionString("connection-string")
- .withAccountName("testaccount")
- .withTenantId("tenant-id")
- .withClientId("client-id")
- .withClientSecret("client-secret")
- .build();
-
- // Use reflection to test private method
- boolean result = (boolean) MethodUtils.invokeMethod(provider, true, "authenticateViaServicePrincipal");
- assertFalse("Should not authenticate via service principal when connection string is present", result);
- }
-
- @Test
- public void testAuthenticateViaServicePrincipalFalseWithMissingCredentials() throws Exception {
- provider = AzureBlobContainerProvider.Builder
- .builder(CONTAINER_NAME)
- .withAccountName("testaccount")
- .withTenantId("tenant-id")
- .withClientId("client-id")
- // Missing client secret
- .build();
-
- // Use reflection to test private method
- boolean result = (boolean) MethodUtils.invokeMethod(provider, true, "authenticateViaServicePrincipal");
- assertFalse("Should not authenticate via service principal with missing credentials", result);
- }
-
- @Test
- public void testGetClientSecretCredential() throws Exception {
- provider = AzureBlobContainerProvider.Builder
- .builder(CONTAINER_NAME)
- .withTenantId("tenant-id")
- .withClientId("client-id")
- .withClientSecret("client-secret")
- .build();
-
- // Use reflection to test private method
- ClientSecretCredential credential = (ClientSecretCredential) MethodUtils.invokeMethod(provider, true, "getClientSecretCredential");
- assertNotNull("Credential should not be null", credential);
- }
-
- @Test
- public void testGetBlobContainerFromServicePrincipals() throws Exception {
- provider = AzureBlobContainerProvider.Builder
- .builder(CONTAINER_NAME)
- .withAccountName("testaccount")
- .withTenantId("tenant-id")
- .withClientId("client-id")
- .withClientSecret("client-secret")
- .build();
-
- RequestRetryOptions retryOptions = new RequestRetryOptions();
-
- // Use reflection to test private method
- try {
- BlobContainerClient containerClient = (BlobContainerClient) MethodUtils.invokeMethod(provider, true,
- "getBlobContainerFromServicePrincipals",
- new Object[]{"testaccount", retryOptions},
- new Class<?>[]{String.class, RequestRetryOptions.class});
- assertNotNull("Container client should not be null", containerClient);
- } catch (Exception e) {
- // Expected for invalid credentials
- assertNotNull("Exception should not be null", e);
- }
- }
-
- @Test
- public void testGenerateUserDelegationKeySignedSas() {
- provider = AzureBlobContainerProvider.Builder
- .builder(CONTAINER_NAME)
- .withAccountName("testaccount")
- .withTenantId("tenant-id")
- .withClientId("client-id")
- .withClientSecret("client-secret")
- .build();
-
- // Mock dependencies
- BlockBlobClient mockBlobClient = mock(BlockBlobClient.class);
- OffsetDateTime expiryTime = OffsetDateTime.now().plusHours(1);
-
- try {
- String sas = provider.generateUserDelegationKeySignedSas(
- mockBlobClient,
- mock(com.azure.storage.blob.sas.BlobServiceSasSignatureValues.class),
- expiryTime
- );
- // This will likely fail due to authentication, but we're testing the method structure
- assertNotNull("SAS should not be null", sas);
- } catch (Exception e) {
- // Expected for invalid credentials or mock limitations
- assertNotNull("Exception should not be null", e);
- }
- }
-
- @Test
- public void testGenerateSas() throws Exception {
- provider = AzureBlobContainerProvider.Builder
- .builder(CONTAINER_NAME)
- .withAccountName("testaccount")
- .withAccountKey("testkey")
- .build();
-
- // Mock dependencies
- BlockBlobClient mockBlobClient = mock(BlockBlobClient.class);
- when(mockBlobClient.generateSas(any(), any())).thenReturn("mock-sas-token");
-
- // Use reflection to test private method
- String sas = (String) MethodUtils.invokeMethod(provider, true, "generateSas",
- new Object[]{mockBlobClient, mock(com.azure.storage.blob.sas.BlobServiceSasSignatureValues.class)},
- new Class<?>[]{BlockBlobClient.class, com.azure.storage.blob.sas.BlobServiceSasSignatureValues.class});
- assertEquals("SAS should match mock", "mock-sas-token", sas);
- }
-
- @Test
- public void testBuilderFieldAccess() throws Exception {
- AzureBlobContainerProvider.Builder builder = AzureBlobContainerProvider.Builder.builder(CONTAINER_NAME);
-
- // Test all builder methods and verify fields are set correctly
- builder.withAzureConnectionString("conn-string")
- .withAccountName("account")
- .withBlobEndpoint("endpoint")
- .withSasToken("sas")
- .withAccountKey("key")
- .withTenantId("tenant")
- .withClientId("client")
- .withClientSecret("secret");
-
- AzureBlobContainerProvider provider = builder.build();
-
- // Use reflection to verify all fields are set
- assertEquals("Container name should match", CONTAINER_NAME, provider.getContainerName());
- assertEquals("Connection string should match", "conn-string", provider.getAzureConnectionString());
-
- // Test private fields using reflection
- Field accountNameField = AzureBlobContainerProvider.class.getDeclaredField("accountName");
- accountNameField.setAccessible(true);
- assertEquals("Account name should match", "account", accountNameField.get(provider));
-
- Field blobEndpointField = AzureBlobContainerProvider.class.getDeclaredField("blobEndpoint");
- blobEndpointField.setAccessible(true);
- assertEquals("Blob endpoint should match", "endpoint", blobEndpointField.get(provider));
-
- Field sasTokenField = AzureBlobContainerProvider.class.getDeclaredField("sasToken");
- sasTokenField.setAccessible(true);
- assertEquals("SAS token should match", "sas", sasTokenField.get(provider));
-
- Field accountKeyField = AzureBlobContainerProvider.class.getDeclaredField("accountKey");
- accountKeyField.setAccessible(true);
- assertEquals("Account key should match", "key", accountKeyField.get(provider));
-
- Field tenantIdField = AzureBlobContainerProvider.class.getDeclaredField("tenantId");
- tenantIdField.setAccessible(true);
- assertEquals("Tenant ID should match", "tenant", tenantIdField.get(provider));
-
- Field clientIdField = AzureBlobContainerProvider.class.getDeclaredField("clientId");
- clientIdField.setAccessible(true);
- assertEquals("Client ID should match", "client", clientIdField.get(provider));
-
- Field clientSecretField = AzureBlobContainerProvider.class.getDeclaredField("clientSecret");
- clientSecretField.setAccessible(true);
- assertEquals("Client secret should match", "secret", clientSecretField.get(provider));
- }
-
- private String getConnectionString() {
- return String.format("DefaultEndpointsProtocol=http;AccountName=%s;AccountKey=%s;BlobEndpoint=%s",
- AzuriteDockerRule.ACCOUNT_NAME,
- AzuriteDockerRule.ACCOUNT_KEY,
- azurite.getBlobEndpoint());
- }
-
- /**
- * Test that generateSharedAccessSignature works without headers (backward compatibility).
- */
- @Test
- public void testGenerateSharedAccessSignatureWithoutHeaders() throws Exception {
- String connectionString = getConnectionString();
- provider = AzureBlobContainerProvider.Builder
- .builder(CONTAINER_NAME)
- .withAzureConnectionString(connectionString)
- .build();
-
- // Create container and blob
- BlobContainerClient container = azurite.getContainer(CONTAINER_NAME, connectionString);
- String blobName = "test-blob";
- container.getBlobClient(blobName).upload(
- com.azure.core.util.BinaryData.fromString("test content"), true);
-
- // Generate SAS without headers
- BlobSasPermission permissions = new BlobSasPermission().setReadPermission(true);
- String sas = provider.generateSharedAccessSignature(
- null, blobName, permissions, 3600, new Properties());
-
- assertNotNull("SAS should not be null", sas);
- assertTrue("SAS should contain signature", sas.contains("sig="));
- assertTrue("SAS should contain expiry", sas.contains("se="));
- }
-
- /**
- * Test that generateSharedAccessSignature includes headers when provided.
- */
- @Test
- public void testGenerateSharedAccessSignatureWithHeaders() throws Exception {
- String connectionString = getConnectionString();
- provider = AzureBlobContainerProvider.Builder
- .builder(CONTAINER_NAME)
- .withAzureConnectionString(connectionString)
- .build();
-
- // Create container and blob
- BlobContainerClient container = azurite.getContainer(CONTAINER_NAME, connectionString);
- String blobName = "test-blob-with-headers";
- container.getBlobClient(blobName).upload(
- com.azure.core.util.BinaryData.fromString("test content"), true);
-
- // Generate SAS with headers
- BlobSasPermission permissions = new BlobSasPermission().setReadPermission(true);
- BlobSasHeaders headers = new BlobSasHeaders()
- .setCacheControl("private, max-age=3600, immutable")
- .setContentType("image/png")
- .setContentDisposition("attachment; filename=\"test.png\"");
-
- String sas = provider.generateSharedAccessSignature(
- null, blobName, permissions, 3600, new Properties(), headers);
-
- assertNotNull("SAS should not be null", sas);
- assertTrue("SAS should contain signature", sas.contains("sig="));
- assertTrue("SAS should contain expiry", sas.contains("se="));
-
- // Verify headers are encoded in SAS
- // Azure encodes headers as rscc (cache-control), rsct (content-type), rscd (content-disposition)
- assertTrue("SAS should contain cache-control parameter",
- sas.contains("rscc=") || sas.contains("&rscc"));
- assertTrue("SAS should contain content-type parameter",
- sas.contains("rsct=") || sas.contains("&rsct"));
- assertTrue("SAS should contain content-disposition parameter",
- sas.contains("rscd=") || sas.contains("&rscd"));
- }
-
- /**
- * Test that generateSharedAccessSignature handles null headers gracefully.
- */
- @Test
- public void testGenerateSharedAccessSignatureWithNullHeaders() throws Exception {
- String connectionString = getConnectionString();
- provider = AzureBlobContainerProvider.Builder
- .builder(CONTAINER_NAME)
- .withAzureConnectionString(connectionString)
- .build();
-
- // Create container and blob
- BlobContainerClient container = azurite.getContainer(CONTAINER_NAME, connectionString);
- String blobName = "test-blob-null-headers";
- container.getBlobClient(blobName).upload(
- com.azure.core.util.BinaryData.fromString("test content"), true);
-
- // Generate SAS with null headers
- BlobSasPermission permissions = new BlobSasPermission().setReadPermission(true);
- String sas = provider.generateSharedAccessSignature(
- null, blobName, permissions, 3600, new Properties(), null);
-
- assertNotNull("SAS should not be null", sas);
- assertTrue("SAS should contain signature", sas.contains("sig="));
- assertTrue("SAS should contain expiry", sas.contains("se="));
- }
-
- /**
- * Test that generateSharedAccessSignature includes only provided headers.
- */
- @Test
- public void testGenerateSharedAccessSignatureWithPartialHeaders() throws Exception {
- String connectionString = getConnectionString();
- provider = AzureBlobContainerProvider.Builder
- .builder(CONTAINER_NAME)
- .withAzureConnectionString(connectionString)
- .build();
-
- // Create container and blob
- BlobContainerClient container = azurite.getContainer(CONTAINER_NAME, connectionString);
- String blobName = "test-blob-partial-headers";
- container.getBlobClient(blobName).upload(
- com.azure.core.util.BinaryData.fromString("test content"), true);
-
- // Generate SAS with only content-type header
- BlobSasPermission permissions = new BlobSasPermission().setReadPermission(true);
- BlobSasHeaders headers = new BlobSasHeaders().setContentType("application/json");
-
- String sas = provider.generateSharedAccessSignature(
- null, blobName, permissions, 3600, new Properties(), headers);
-
- assertNotNull("SAS should not be null", sas);
- assertTrue("SAS should contain signature", sas.contains("sig="));
- assertTrue("SAS should contain content-type parameter",
- sas.contains("rsct=") || sas.contains("&rsct"));
- }
-}
diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendCompatibilityTest.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendCompatibilityTest.java
deleted file mode 100644
index 053ae0df50..0000000000
--- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendCompatibilityTest.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;
-
-import java.lang.reflect.Field;
-import java.net.URI;
-
-import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordDownloadOptions;
-import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadOptions;
-import org.apache.jackrabbit.oak.spi.blob.data.DataIdentifier;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-
-/**
- * Compatibility tests for direct-download and upload cache configuration in
- * {@link AzureBlobStoreBackend}. The assertions are intentionally behavior-
- * based and do not depend on a specific cache library.
- */
-public class AzureBlobStoreBackendCompatibilityTest {
-
- @Test
- public void setHttpDownloadURIExpirySecondsUpdatesField() throws Exception {
- // Setter coverage for the direct-download expiry value used by presigned URIs.
- AzureBlobStoreBackend backend = new AzureBlobStoreBackend();
-
- backend.setHttpDownloadURIExpirySeconds(3600);
-
- assertEquals(3600, getIntField(backend, "httpDownloadURIExpirySeconds"));
- }
-
- @Test
- public void setHttpUploadURIExpirySecondsUpdatesField() throws Exception {
- // Setter coverage for the direct-upload expiry used during upload initiation.
- AzureBlobStoreBackend backend = new AzureBlobStoreBackend();
-
- backend.setHttpUploadURIExpirySeconds(1800);
-
- assertEquals(1800, getIntField(backend, "httpUploadURIExpirySeconds"));
- }
-
- @Test
- public void setHttpDownloadURICacheSizeCreatesAndDisablesCache() throws Exception {
- // Verify the cache-size toggle actually creates and then removes the backing cache.
- AzureBlobStoreBackend backend = new AzureBlobStoreBackend();
- backend.setHttpDownloadURIExpirySeconds(3600);
-
- backend.setHttpDownloadURICacheSize(100);
- assertNotNull(getField(backend, "httpDownloadURICache"));
-
- backend.setHttpDownloadURICacheSize(0);
- assertNull(getField(backend, "httpDownloadURICache"));
- }
-
- @Test
- public void createHttpDownloadURIReturnsNullWhenDisabled() {
- // With no download expiry configured, direct download access should stay disabled.
- AzureBlobStoreBackend backend = new AzureBlobStoreBackend();
-
- URI downloadURI = backend.createHttpDownloadURI(
- new DataIdentifier("test"),
- DataRecordDownloadOptions.DEFAULT);
-
- assertNull(downloadURI);
- }
-
- @Test
- public void initiateHttpUploadReturnsNullWhenDisabled() {
- // Upload initiation follows the same disabled-by-default contract until configured.
- AzureBlobStoreBackend backend = new AzureBlobStoreBackend();
-
- assertNull(backend.initiateHttpUpload(1024, 1, DataRecordUploadOptions.DEFAULT));
- }
-
- private static int getIntField(AzureBlobStoreBackend backend, String fieldName) throws Exception {
- Field field = AzureBlobStoreBackend.class.getDeclaredField(fieldName);
- field.setAccessible(true);
- return (int) field.get(backend);
- }
-
- private static Object getField(AzureBlobStoreBackend backend, String fieldName) throws Exception {
- Field field = AzureBlobStoreBackend.class.getDeclaredField(fieldName);
- field.setAccessible(true);
- return field.get(backend);
- }
-}
diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendTest.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendTest.java
index 1603c3d693..788ca33e9e 100644
--- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendTest.java
+++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendTest.java
@@ -7,2095 +7,78 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
*/
package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;
-import com.azure.storage.blob.BlobClient;
-import com.azure.storage.blob.BlobContainerClient;
-import com.azure.storage.blob.models.BlobItem;
-import com.azure.storage.blob.models.ListBlobsOptions;
-import com.azure.storage.blob.sas.BlobSasPermission;
-import com.azure.storage.blob.sas.BlobServiceSasSignatureValues;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.reflect.MethodUtils;
-import org.apache.jackrabbit.oak.cache.api.Cache;
-import org.apache.jackrabbit.oak.spi.blob.data.DataIdentifier;
-import org.apache.jackrabbit.oak.spi.blob.data.DataRecord;
-import org.apache.jackrabbit.oak.spi.blob.data.DataStoreException;
-import org.apache.jackrabbit.oak.api.blob.BlobDownloadOptions;
-import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordDownloadOptions;
-import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUpload;
-import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadOptions;
-import org.jetbrains.annotations.NotNull;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.ClassRule;
-import org.junit.Test;
-import org.mockito.MockitoAnnotations;
-import org.slf4j.LoggerFactory;
-import ch.qos.logback.classic.spi.ILoggingEvent;
-import ch.qos.logback.core.read.ListAppender;
-
-import java.io.BufferedInputStream;
-import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.lang.reflect.Field;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.net.URI;
-import java.nio.charset.StandardCharsets;
-import java.time.OffsetDateTime;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Properties;
-import java.util.Set;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-import java.util.stream.StreamSupport;
-
-import static java.util.stream.Collectors.toSet;
-import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BlOB_META_DIR_NAME;
-import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION;
-import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_REF_KEY;
-import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_CONNECTION_STRING;
-import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_CONTAINER_NAME;
-import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_STORAGE_ACCOUNT_NAME;
-import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_ENDPOINT;
-import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_CREATE_CONTAINER;
-import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_REF_ON_INIT;
-import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS;
-import static org.junit.Assert.*;
-import static org.mockito.ArgumentMatchers.any;
-
-/**
- * Comprehensive test class for AzureBlobStoreBackend covering all methods and functionality.
- * Combines unit tests and integration tests.
- */
-public class AzureBlobStoreBackendTest {
-
- @ClassRule
- public static AzuriteDockerRule azurite = new AzuriteDockerRule();
-
- private static final String CONTAINER_NAME = "test-container";
- private static final String TEST_METADATA_CONTENT = "test metadata content";
- private static final Set<String> BLOBS = Set.of("blob1", "blob2");
-
- private BlobContainerClient container;
- private AzureBlobStoreBackend backend;
- private Properties testProperties;
-
- @Before
- public void setUp() {
- MockitoAnnotations.openMocks(this);
-
- // Create real container for integration tests
- container = azurite.getContainer(CONTAINER_NAME, getConnectionString());
-
- // Setup test properties
- testProperties = createTestProperties();
-
- // Create backend instance
- backend = new AzureBlobStoreBackend();
- backend.setProperties(testProperties);
- }
-
- @After
- public void tearDown() {
- if (backend != null) {
- try {
- backend.close();
- } catch (Exception e) {
- // Ignore cleanup errors
- }
- }
- if (container != null) {
- try {
- container.deleteIfExists();
- } catch (Exception e) {
- // Ignore cleanup errors
- }
- }
- }
-
- private Properties createTestProperties() {
- Properties properties = new Properties();
- properties.setProperty(AZURE_BLOB_CONTAINER_NAME, CONTAINER_NAME);
- properties.setProperty(AZURE_STORAGE_ACCOUNT_NAME, AzuriteDockerRule.ACCOUNT_NAME);
- properties.setProperty(AZURE_BLOB_ENDPOINT, azurite.getBlobEndpoint());
- properties.setProperty(AZURE_CONNECTION_STRING, getConnectionString());
- properties.setProperty(AZURE_CREATE_CONTAINER, "true");
- properties.setProperty(AZURE_REF_ON_INIT, "false"); // Disable for most tests
- return properties;
- }
-
- private static String getConnectionString() {
- return Utils.getConnectionString(
- AzuriteDockerRule.ACCOUNT_NAME,
- AzuriteDockerRule.ACCOUNT_KEY,
- azurite.getBlobEndpoint()
- );
- }
-
- // ========== INITIALIZATION AND CONFIGURATION TESTS ==========
-
- @Test
- public void testInitWithValidProperties() throws Exception {
- backend.init();
- assertNotNull("Backend should be initialized", backend);
-
- // Verify container was created
- BlobContainerClient azureContainer = backend.getAzureContainer();
- assertNotNull("Azure container should not be null", azureContainer);
- assertTrue("Container should exist", azureContainer.exists());
- }
-
- @Test
- public void testInitWithNullProperties() {
- AzureBlobStoreBackend nullPropsBackend = new AzureBlobStoreBackend();
- // Should not set properties, will try to read from default config file
-
- try {
- nullPropsBackend.init();
- fail("Expected DataStoreException when no properties and no default config file");
- } catch (DataStoreException e) {
- assertTrue("Should contain config file error",
- e.getMessage().contains("Unable to initialize Azure Data Store"));
- }
- }
-
- @Test
- public void testInitWithNullPropertiesAndValidConfigFile() throws Exception {
- // Create a temporary azure.properties file in the working directory
- File configFile = new File("azure.properties");
- Properties configProps = createTestProperties();
-
- try (FileOutputStream fos = new FileOutputStream(configFile)) {
- configProps.store(fos, "Test configuration for null properties test");
- }
-
- AzureBlobStoreBackend nullPropsBackend = new AzureBlobStoreBackend();
- // Don't set properties - should read from azure.properties file
-
- try {
- nullPropsBackend.init();
- assertNotNull("Backend should be initialized from config file", nullPropsBackend);
-
- // Verify container was created
- BlobContainerClient azureContainer = nullPropsBackend.getAzureContainer();
- assertNotNull("Azure container should not be null", azureContainer);
- assertTrue("Container should exist", azureContainer.exists());
- } finally {
- // Clean up the config file
- if (configFile.exists()) {
- configFile.delete();
- }
- // Clean up the backend
- try {
- nullPropsBackend.close();
- } catch (Exception e) {
- // Ignore cleanup errors
- }
- }
- }
-
- @Test
- public void testSetProperties() {
- Properties newProps = new Properties();
- newProps.setProperty("test.key", "test.value");
-
- backend.setProperties(newProps);
-
- // Verify properties were set (using reflection to access private field)
- try {
- Field propertiesField = AzureBlobStoreBackend.class.getDeclaredField("properties");
- propertiesField.setAccessible(true);
- Properties actualProps = (Properties) propertiesField.get(backend);
- assertEquals("Properties should be set", "test.value", actualProps.getProperty("test.key"));
- } catch (Exception e) {
- fail("Failed to verify properties were set: " + e.getMessage());
- }
- }
-
- @Test
- public void testConcurrentRequestCountValidation() throws Exception {
- // Test with too low concurrent request count
- Properties lowProps = createTestProperties();
- lowProps.setProperty(AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION, "1");
-
- AzureBlobStoreBackend lowBackend = new AzureBlobStoreBackend();
- lowBackend.setProperties(lowProps);
- lowBackend.init();
-
- // Should reset to default minimum (verified through successful initialization)
- assertNotNull("Backend should initialize with low concurrent request count", lowBackend);
- lowBackend.close();
-
- // Test with too high concurrent request count
- Properties highProps = createTestProperties();
- highProps.setProperty(AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION, "100");
-
- AzureBlobStoreBackend highBackend = new AzureBlobStoreBackend();
- highBackend.setProperties(highProps);
- highBackend.init();
-
- // Should reset to default maximum (verified through successful initialization)
- assertNotNull("Backend should initialize with high concurrent request count", highBackend);
- highBackend.close();
- }
-
- @Test
- public void testGetAzureContainerThreadSafety() throws Exception {
- backend.init();
-
- int threadCount = 10;
- ExecutorService executor = Executors.newFixedThreadPool(threadCount);
- CountDownLatch latch = new CountDownLatch(threadCount);
- List<Future<BlobContainerClient>> futures = new ArrayList<>();
-
- // Submit multiple threads to get container simultaneously
- for (int i = 0; i < threadCount; i++) {
- futures.add(executor.submit(() -> {
- try {
- latch.countDown();
- latch.await(); // Wait for all threads to be ready
- return backend.getAzureContainer();
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }));
- }
-
- // Verify all threads get the same container instance
- BlobContainerClient firstContainer = futures.get(0).get(5, TimeUnit.SECONDS);
- for (Future<BlobContainerClient> future : futures) {
- BlobContainerClient container = future.get(5, TimeUnit.SECONDS);
- assertSame("All threads should get the same container instance", firstContainer, container);
- }
-
- executor.shutdown();
- }
-
- @Test
- public void testGetAzureContainerWhenNull() throws Exception {
- // Create a backend with valid properties but don't initialize it
- // This ensures azureContainer field remains null initially
- AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend();
- testBackend.setProperties(testProperties);
-
- // Initialize the backend to set up azureBlobContainerProvider
- testBackend.init();
-
- try {
- // Reset azureContainer to null using reflection to test the null case
- Field azureContainerReferenceField = AzureBlobStoreBackend.class.getDeclaredField("azureContainerReference");
- azureContainerReferenceField.setAccessible(true);
- @SuppressWarnings("unchecked")
- AtomicReference<BlobContainerClient> azureContainerReference = (AtomicReference<BlobContainerClient>) azureContainerReferenceField.get(testBackend);
- azureContainerReference.set(null);
-
- // Verify azureContainer is null
- BlobContainerClient containerBeforeCall = azureContainerReference.get();
- assertNull("azureContainer should be null before getAzureContainer call", containerBeforeCall);
-
- // Call getAzureContainer - this should initialize the container
- BlobContainerClient container = testBackend.getAzureContainer();
-
- // Verify container is not null and properly initialized
- assertNotNull("getAzureContainer should return non-null container when azureContainer was null", container);
- assertTrue("Container should exist", container.exists());
-
- // Verify azureContainer field is now set
- BlobContainerClient containerAfterCall = azureContainerReference.get();
- assertNotNull("azureContainer field should be set after getAzureContainer call", containerAfterCall);
- assertSame("Returned container should be same as stored in field", container, containerAfterCall);
-
- // Call getAzureContainer again - should return same instance
- BlobContainerClient container2 = testBackend.getAzureContainer();
- assertSame("Subsequent calls should return same container instance", container, container2);
-
- } finally {
- testBackend.close();
- }
- }
-
- @Test
- public void testGetAzureContainerWithProviderException() throws Exception {
- // Create a backend with a mock provider that throws exception
- AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend();
- testBackend.setProperties(testProperties);
-
- // Set up mock provider using reflection
- Field providerField = AzureBlobStoreBackend.class.getDeclaredField("azureBlobContainerProvider");
- providerField.setAccessible(true);
-
- // Create mock provider that throws DataStoreException
- AzureBlobContainerProvider mockProvider = org.mockito.Mockito.mock(AzureBlobContainerProvider.class);
- org.mockito.Mockito.when(mockProvider.getBlobContainer(any(), any()))
- .thenThrow(new DataStoreException("Mock connection failure"));
-
- providerField.set(testBackend, mockProvider);
-
- try {
- // Call getAzureContainer - should propagate the DataStoreException
- testBackend.getAzureContainer();
- fail("Expected DataStoreException when azureBlobContainerProvider.getBlobContainer() fails");
- } catch (DataStoreException e) {
- assertEquals("Exception message should match", "Mock connection failure", e.getMessage());
-
- // Verify azureContainer field remains null after exception
- Field azureContainerField = AzureBlobStoreBackend.class.getDeclaredField("azureContainerReference");
- azureContainerField.setAccessible(true);
- @SuppressWarnings("unchecked")
- BlobContainerClient containerAfterException = ((AtomicReference<BlobContainerClient>) azureContainerField.get(testBackend)).get();
- assertNull("azureContainer should remain null after exception", containerAfterException);
- } finally {
- testBackend.close();
- }
- }
-
- // ========== CORE CRUD OPERATIONS TESTS ==========
-
- @Test
- public void testWriteAndRead() throws Exception {
- backend.init();
-
- // Create test file
- File testFile = createTempFile("test-content");
- DataIdentifier identifier = new DataIdentifier("testidentifier123");
-
- try {
- // Write file
- backend.write(identifier, testFile);
-
- // Read file
- try (InputStream inputStream = backend.read(identifier)) {
- String content = IOUtils.toString(inputStream, "UTF-8");
- assertEquals("Content should match", "test-content", content);
- }
- } finally {
- testFile.delete();
- }
- }
-
- @Test
- public void testWriteWithNullIdentifier() throws Exception {
- backend.init();
- File testFile = createTempFile("test");
-
- try {
- backend.write(null, testFile);
- fail("Expected NullPointerException for null identifier");
- } catch (NullPointerException e) {
- assertEquals("identifier must not be null", e.getMessage());
- } finally {
- testFile.delete();
- }
- }
-
- @Test
- public void testWriteWithNullFile() throws Exception {
- backend.init();
- DataIdentifier identifier = new DataIdentifier("test");
-
- try {
- backend.write(identifier, null);
- fail("Expected NullPointerException for null file");
- } catch (NullPointerException e) {
- assertEquals("file must not be null", e.getMessage());
- }
- }
-
- @Test
- public void testWriteExistingBlobWithSameLength() throws Exception {
- backend.init();
-
- File testFile = createTempFile("same-content");
- DataIdentifier identifier = new DataIdentifier("existingblob123");
-
- try {
- // Write file first time
- backend.write(identifier, testFile);
-
- // Write same file again (should update metadata)
- backend.write(identifier, testFile);
-
- // Verify content is still accessible
- try (InputStream inputStream = backend.read(identifier)) {
- String content = IOUtils.toString(inputStream, "UTF-8");
- assertEquals("Content should match", "same-content", content);
- }
- } finally {
- testFile.delete();
- }
- }
-
- @Test
- public void testWriteExistingBlobWithDifferentLength() throws Exception {
- backend.init();
-
- File testFile1 = createTempFile("content1");
- File testFile2 = createTempFile("different-length-content");
- DataIdentifier identifier = new DataIdentifier("lengthcollision");
-
- try {
- // Write first file
- backend.write(identifier, testFile1);
-
- // Try to write file with different length
- try {
- backend.write(identifier, testFile2);
- fail("Expected DataStoreException for length collision");
- } catch (DataStoreException e) {
- assertTrue("Should contain length collision error",
- e.getMessage().contains("Length Collision"));
- }
- } finally {
- testFile1.delete();
- testFile2.delete();
- }
- }
-
- @Test
- public void testReadNonExistentBlob() throws Exception {
- backend.init();
- DataIdentifier identifier = new DataIdentifier("nonexistent123");
-
- try {
- backend.read(identifier);
- fail("Expected DataStoreException for non-existent blob");
- } catch (DataStoreException e) {
- assertTrue("Should contain missing blob error",
- e.getMessage().contains("Trying to read missing blob"));
- }
- }
-
- @Test
- public void testReadWithNullIdentifier() throws Exception {
- backend.init();
-
- try {
- backend.read(null);
- fail("Expected NullPointerException for null identifier");
- } catch (NullPointerException e) {
- assertEquals("identifier must not be null", e.getMessage());
- }
- }
-
- @Test
- public void testGetRecord() throws Exception {
- backend.init();
-
- File testFile = createTempFile("record-content");
- DataIdentifier identifier = new DataIdentifier("testrecord123");
-
- try {
- // Write file first
- backend.write(identifier, testFile);
-
- // Get record
- DataRecord record = backend.getRecord(identifier);
- assertNotNull("Record should not be null", record);
- assertEquals("Record identifier should match", identifier, record.getIdentifier());
- assertEquals("Record length should match", testFile.length(), record.getLength());
- assertTrue("Record should have valid last modified time", record.getLastModified() > 0);
- } finally {
- testFile.delete();
- }
- }
-
- @Test
- public void testGetRecordNonExistent() throws Exception {
- backend.init();
- DataIdentifier identifier = new DataIdentifier("nonexistentrecord");
-
- try {
- backend.getRecord(identifier);
- fail("Expected DataStoreException for non-existent record");
- } catch (DataStoreException e) {
- assertTrue("Should contain retrieve blob error",
- e.getMessage().contains("Cannot retrieve blob"));
- }
- }
-
- @Test
- public void testGetRecordWithNullIdentifier() throws Exception {
- backend.init();
-
- try {
- backend.getRecord(null);
- fail("Expected NullPointerException for null identifier");
- } catch (NullPointerException e) {
- assertEquals("identifier must not be null", e.getMessage());
- }
- }
-
- @Test
- public void testExists() throws Exception {
- backend.init();
-
- File testFile = createTempFile("exists-content");
- DataIdentifier identifier = new DataIdentifier("existstest123");
-
- try {
- // Initially should not exist
- assertFalse("Blob should not exist initially", backend.exists(identifier));
-
- // Write file
- backend.write(identifier, testFile);
-
- // Now should exist
- assertTrue("Blob should exist after write", backend.exists(identifier));
- } finally {
- testFile.delete();
- }
- }
-
- @Test
- public void testDeleteRecord() throws Exception {
- backend.init();
-
- File testFile = createTempFile("delete-content");
- DataIdentifier identifier = new DataIdentifier("deletetest123");
-
- try {
- // Write file
- backend.write(identifier, testFile);
- assertTrue("Blob should exist before delete", backend.exists(identifier));
-
- // Delete record
- backend.deleteRecord(identifier);
- assertFalse("Blob should not exist after delete", backend.exists(identifier));
- } finally {
- testFile.delete();
- }
- }
-
- @Test
- public void testDeleteNonExistentRecord() throws Exception {
- backend.init();
- DataIdentifier identifier = new DataIdentifier("nonexistentdelete");
-
- // Should not throw exception when deleting non-existent record
- backend.deleteRecord(identifier);
- // No exception expected
- assertTrue("Delete should not throw exception for non-existent record", true);
- }
-
- @Test
- public void testDeleteRecordWithNullIdentifier() throws Exception {
- backend.init();
-
- try {
- backend.deleteRecord(null);
- fail("Expected NullPointerException for null identifier");
- } catch (NullPointerException e) {
- assertEquals("identifier must not be null", e.getMessage());
- }
- }
-
- @Test
- public void testGetAllIdentifiers() throws Exception {
- backend.init();
-
- // Create multiple test files
- File testFile1 = createTempFile("content1");
- File testFile2 = createTempFile("content2");
- DataIdentifier id1 = new DataIdentifier("identifier1");
- DataIdentifier id2 = new DataIdentifier("identifier2");
-
- try {
- // Write files
- backend.write(id1, testFile1);
- backend.write(id2, testFile2);
-
- // Get all identifiers
- Iterator<DataIdentifier> identifiers = backend.getAllIdentifiers();
- assertNotNull("Identifiers iterator should not be null", identifiers);
-
- // Collect identifiers
- List<String> identifierStrings = new ArrayList<>();
- while (identifiers.hasNext()) {
- identifierStrings.add(identifiers.next().toString());
- }
-
- // Should contain both identifiers
- assertTrue("Should contain identifier1", identifierStrings.contains("identifier1"));
- assertTrue("Should contain identifier2", identifierStrings.contains("identifier2"));
- } finally {
- testFile1.delete();
- testFile2.delete();
- }
- }
-
- @Test
- public void testGetAllRecords() throws Exception {
- backend.init();
-
- // Create test file
- File testFile = createTempFile("record-content");
- DataIdentifier identifier = new DataIdentifier("recordtest123");
-
- try {
- // Write file
- backend.write(identifier, testFile);
-
- // Get all records
- Iterator<DataRecord> records = backend.getAllRecords();
- assertNotNull("Records iterator should not be null", records);
-
- // Find our record
- boolean foundRecord = false;
- while (records.hasNext()) {
- DataRecord record = records.next();
- if (record.getIdentifier().toString().equals("recordtest123")) {
- foundRecord = true;
- assertEquals("Record length should match", testFile.length(), record.getLength());
- assertTrue("Record should have valid last modified time", record.getLastModified() > 0);
- break;
- }
- }
- assertTrue("Should find our test record", foundRecord);
- } finally {
- testFile.delete();
- }
- }
-
- // ========== METADATA OPERATIONS TESTS ==========
-
- @Test
- public void testAddMetadataRecordWithInputStream() throws Exception {
- backend.init();
-
- String metadataName = "test-metadata-stream";
- String content = TEST_METADATA_CONTENT;
-
- // Add metadata record
- backend.addMetadataRecord(new ByteArrayInputStream(content.getBytes()), metadataName);
-
- // Verify record exists
- assertTrue("Metadata record should exist", backend.metadataRecordExists(metadataName));
-
- // Verify content
- DataRecord record = backend.getMetadataRecord(metadataName);
- assertNotNull("Record should not be null", record);
- assertEquals("Record should have correct length", content.length(), record.getLength());
-
- // Verify content can be read
- try (InputStream stream = record.getStream()) {
- String readContent = IOUtils.toString(stream, "UTF-8");
- assertEquals("Content should match", content, readContent);
- }
-
- // Clean up
- backend.deleteMetadataRecord(metadataName);
- }
-
- @Test
- public void testAddMetadataRecordWithFile() throws Exception {
- backend.init();
-
- String metadataName = "test-metadata-file";
- File metadataFile = createTempFile(TEST_METADATA_CONTENT);
-
- try {
- // Add metadata record from file
- backend.addMetadataRecord(metadataFile, metadataName);
-
- // Verify record exists
- assertTrue("Metadata record should exist", backend.metadataRecordExists(metadataName));
-
- // Verify content
- DataRecord record = backend.getMetadataRecord(metadataName);
- assertNotNull("Record should not be null", record);
- assertEquals("Record should have correct length", metadataFile.length(), record.getLength());
-
- // Clean up
- backend.deleteMetadataRecord(metadataName);
- } finally {
- metadataFile.delete();
- }
- }
-
- @Test
- public void testAddMetadataRecordWithNullInputStream() throws Exception {
- backend.init();
-
- try {
- backend.addMetadataRecord((InputStream) null, "test");
- fail("Expected NullPointerException for null input stream");
- } catch (NullPointerException e) {
- assertEquals("input must not be null", e.getMessage());
- }
- }
-
- @Test
- public void testAddMetadataRecordWithNullFile() throws Exception {
- backend.init();
-
- try {
- backend.addMetadataRecord((File) null, "test");
- fail("Expected NullPointerException for null file");
- } catch (NullPointerException e) {
- assertEquals("input must not be null", e.getMessage());
- }
- }
-
- @Test
- public void testAddMetadataRecordWithNullName() throws Exception {
- backend.init();
-
- try {
- backend.addMetadataRecord(new ByteArrayInputStream("test".getBytes()), null);
- fail("Expected IllegalArgumentException for null name");
- } catch (IllegalArgumentException e) {
- assertEquals("name should not be empty", e.getMessage());
- }
- }
-
- @Test
- public void testAddMetadataRecordWithEmptyName() throws Exception {
- backend.init();
-
- try {
- backend.addMetadataRecord(new ByteArrayInputStream("test".getBytes()), "");
- fail("Expected IllegalArgumentException for empty name");
- } catch (IllegalArgumentException e) {
- assertEquals("name should not be empty", e.getMessage());
- }
- }
-
- @Test
- public void testGetMetadataRecordNonExistent() throws Exception {
- backend.init();
-
- DataRecord record = backend.getMetadataRecord("non-existent-metadata");
- assertNull("Non-existent metadata record should return null", record);
- }
-
- @Test
- public void testMetadataPrefixComposition() throws Exception {
- backend.init();
-
- BlobContainerClient azureContainer = backend.getAzureContainer();
- String actualPrefix = "test-prefix";
- String recordName = actualPrefix + "-record1";
-
- // Add metadata record with the actual prefix
- backend.addMetadataRecord(new ByteArrayInputStream("test content".getBytes()),
- recordName
- );
-
- try {
- // Verify the blob is stored with the META/ prefix in Azure storage
- String expectedBlobName = AzureConstants.AZURE_BLOB_META_KEY_PREFIX + recordName;
- BlobClient blobClient = azureContainer.getBlobClient(expectedBlobName);
- assertTrue("Blob should exist at path with META/ prefix: " + expectedBlobName,
- blobClient.exists());
-
- // Verify the blob is listed under the META directory
- ListBlobsOptions listOptions = new ListBlobsOptions();
- listOptions.setPrefix(AzureConstants.AZURE_BlOB_META_DIR_NAME);
-
- boolean foundBlobWithMetaPrefix = false;
- for (BlobItem blobItem : azureContainer.listBlobs(listOptions, null)) {
- if (blobItem.getName().equals(expectedBlobName)) {
- foundBlobWithMetaPrefix = true;
- break;
- }
- }
- assertTrue("Blob should be found in META directory with full META/ prefix",
- foundBlobWithMetaPrefix);
-
- // Get all metadata records via the API
- List<DataRecord> records = backend.getAllMetadataRecords("");
- assertNotNull("Records list should not be null", records);
- assertEquals("Should find exactly one record", 1, records.size());
-
- // Verify the record identifier does NOT include the META/ prefix
- // The identifier should be the logical name, not the storage path
- DataRecord record = records.get(0);
- String recordId = record.getIdentifier().toString();
- assertEquals("Record identifier should match the provided name (without META/ prefix)",
- recordName, recordId);
-
- // Verify the identifier starts with our prefix
- assertTrue("Record identifier should start with the prefix",
- recordId.startsWith(actualPrefix));
-
- } finally {
- backend.deleteMetadataRecord(recordName);
- }
- }
-
- @Test
- public void testGetAllMetadataRecords() throws Exception {
- backend.init();
-
- String prefix = "test-prefix-";
- String content = "metadata content";
-
- // Add multiple metadata records
- for (int i = 0; i < 3; i++) {
- backend.addMetadataRecord(
- new ByteArrayInputStream((content + i).getBytes()),
- prefix + i
- );
- }
-
- try {
- // Get all metadata records
- List<DataRecord> records = backend.getAllMetadataRecords("");
- assertNotNull("Records list should not be null", records);
-
- // Find our records
- int foundCount = 0;
- for (DataRecord record : records) {
- if (record.getIdentifier().toString().startsWith(prefix)) {
- foundCount++;
- }
- }
- assertEquals("Should find all 3 metadata records", 3, foundCount);
- } finally {
- // Clean up
- for (int i = 0; i < 3; i++) {
- backend.deleteMetadataRecord(prefix + i);
- }
- }
- }
-
- @Test
- public void testGetAllMetadataRecordsWithNullPrefix() throws Exception {
- backend.init();
-
- try {
- backend.getAllMetadataRecords(null);
- fail("Expected NullPointerException for null prefix");
- } catch (NullPointerException e) {
- assertEquals("prefix must not be null", e.getMessage());
- }
- }
-
- @Test
- public void testDeleteMetadataRecord() throws Exception {
- backend.init();
-
- String metadataName = "delete-metadata-test";
- String content = "content to delete";
-
- // Add metadata record
- backend.addMetadataRecord(new ByteArrayInputStream(content.getBytes()), metadataName);
- assertTrue("Metadata record should exist", backend.metadataRecordExists(metadataName));
-
- // Delete metadata record
- boolean deleted = backend.deleteMetadataRecord(metadataName);
- assertTrue("Delete should return true", deleted);
- assertFalse("Metadata record should not exist after delete", backend.metadataRecordExists(metadataName));
- }
-
- @Test
- public void testDeleteNonExistentMetadataRecord() throws Exception {
- backend.init();
-
- boolean deleted = backend.deleteMetadataRecord("non-existent-metadata");
- assertFalse("Delete should return false for non-existent record", deleted);
- }
-
- @Test
- public void testDeleteAllMetadataRecords() throws Exception {
- backend.init();
-
- String prefix = "delete-all-";
- String otherPrefix = "keep-all-";
-
- // Add multiple metadata records with the target prefix
- for (int i = 0; i < 3; i++) {
- backend.addMetadataRecord(
- new ByteArrayInputStream(("content" + i).getBytes()),
- prefix + i
- );
- }
-
- // Add metadata records with a different prefix (should NOT be deleted)
- for (int i = 0; i < 2; i++) {
- backend.addMetadataRecord(
- new ByteArrayInputStream(("other-content" + i).getBytes()),
- otherPrefix + i
- );
- }
-
- // Verify all records exist
- for (int i = 0; i < 3; i++) {
- assertTrue("Record with target prefix should exist", backend.metadataRecordExists(prefix + i));
- }
- for (int i = 0; i < 2; i++) {
- assertTrue("Record with other prefix should exist", backend.metadataRecordExists(otherPrefix + i));
- }
-
- // Delete all records with the target prefix
- backend.deleteAllMetadataRecords(prefix);
-
- // Verify records with target prefix are deleted
- for (int i = 0; i < 3; i++) {
- assertFalse("Record with target prefix should be deleted", backend.metadataRecordExists(prefix + i));
- }
-
- // Verify records with other prefix still exist (not deleted)
- for (int i = 0; i < 2; i++) {
- assertTrue("Record with other prefix should still exist", backend.metadataRecordExists(otherPrefix + i));
- }
-
- // Clean up remaining records
- for (int i = 0; i < 2; i++) {
- backend.deleteMetadataRecord(otherPrefix + i);
- }
- }
-
- @Test
- public void testDeleteAllMetadataRecordsWithNullPrefix() throws Exception {
- backend.init();
-
- try {
- backend.deleteAllMetadataRecords(null);
- fail("Expected NullPointerException for null prefix");
- } catch (NullPointerException e) {
- assertEquals("prefix must not be null", e.getMessage());
- }
- }
-
- @Test
- public void testMetadataRecordExists() throws Exception {
- backend.init();
-
- String metadataName = "exists-test-metadata";
-
- // Initially should not exist
- assertFalse("Metadata record should not exist initially",
- backend.metadataRecordExists(metadataName));
-
- // Add metadata record
- backend.addMetadataRecord(
- new ByteArrayInputStream("test content".getBytes()),
- metadataName
- );
-
- // Now should exist
- assertTrue("Metadata record should exist after add",
- backend.metadataRecordExists(metadataName));
-
- // Clean up
- backend.deleteMetadataRecord(metadataName);
- }
-
- // ========== UTILITY AND HELPER METHOD TESTS ==========
-
- @Test
- public void testGetKeyName() throws Exception {
- // Test the static getKeyName method using reflection
- DataIdentifier identifier = new DataIdentifier("abcd1234567890");
- Method getKeyNameMethod = MethodUtils.getMatchingMethod(AzureBlobStoreBackend.class, "getKeyName", DataIdentifier.class);
- getKeyNameMethod.setAccessible(true);
- String keyName = (String) getKeyNameMethod.invoke(null, identifier);
-
- assertEquals("Key name should be formatted correctly", "abcd-1234567890", keyName);
- }
-
- @Test
- public void testGetIdentifierName() throws Exception {
- // Test the static getIdentifierName method using reflection
- Method getIdentifierNameMethod = MethodUtils.getMatchingMethod(AzureBlobStoreBackend.class, "getIdentifierName", String.class);
- getIdentifierNameMethod.setAccessible(true);
-
- String identifierName = (String) getIdentifierNameMethod.invoke(null, "abcd-1234567890");
- assertEquals("Identifier name should be formatted correctly", "abcd1234567890", identifierName);
-
- // Test with metadata key
- String metaKey = "META/test-key";
- String metaIdentifierName = (String) getIdentifierNameMethod.invoke(null, metaKey);
- assertEquals("Metadata identifier should be returned as-is", metaKey, metaIdentifierName);
-
- // Test with key without dash
- String noDashKey = "nodashkey";
- String noDashResult = (String) getIdentifierNameMethod.invoke(null, noDashKey);
- assertNull("Key without dash should return null", noDashResult);
- }
-
- @Test
- public void testAddMetaKeyPrefix() throws Exception {
- // Test the static addMetaKeyPrefix method using reflection
- Method addMetaKeyPrefixMethod = MethodUtils.getMatchingMethod(AzureBlobStoreBackend.class, "addMetaKeyPrefix", String.class);
- addMetaKeyPrefixMethod.setAccessible(true);
- String result = (String) addMetaKeyPrefixMethod.invoke(null, "test-key");
- assertTrue("Result should contain META prefix", result.startsWith("META/"));
- assertTrue("Result should contain original key", result.endsWith("test-key"));
- }
-
- @Test
- public void testStripMetaKeyPrefix() throws Exception {
- // Test the static stripMetaKeyPrefix method using reflection
- Method stripMetaKeyPrefixMethod = MethodUtils.getMatchingMethod(AzureBlobStoreBackend.class, "stripMetaKeyPrefix", String.class);
- stripMetaKeyPrefixMethod.setAccessible(true);
-
- String withPrefix = "META/test-key";
- String result = (String) stripMetaKeyPrefixMethod.invoke(null, withPrefix);
- assertEquals("Should strip META prefix", "test-key", result);
-
- String withoutPrefix = "regular-key";
- String result2 = (String) stripMetaKeyPrefixMethod.invoke(null, withoutPrefix);
- assertEquals("Should return original key if no prefix", withoutPrefix, result2);
- }
-
- @Test
- public void testGetOrCreateReferenceKey() throws Exception {
- // Enable reference key creation on init
- Properties propsWithRef = createTestProperties();
- propsWithRef.setProperty(AZURE_REF_ON_INIT, "true");
-
- AzureBlobStoreBackend refBackend = new AzureBlobStoreBackend();
- refBackend.setProperties(propsWithRef);
- refBackend.init();
-
- try {
- // Get reference key
- byte[] key1 = refBackend.getOrCreateReferenceKey();
- assertNotNull("Reference key should not be null", key1);
- assertTrue("Reference key should have length > 0", key1.length > 0);
-
- // Get reference key again - should be same
- byte[] key2 = refBackend.getOrCreateReferenceKey();
- assertArrayEquals("Reference key should be consistent", key1, key2);
-
- // Verify reference key is stored as metadata
- DataRecord refRecord = refBackend.getMetadataRecord(AZURE_BLOB_REF_KEY);
- assertNotNull("Reference key metadata record should exist", refRecord);
- assertTrue("Reference key record should have length > 0", refRecord.getLength() > 0);
- } finally {
- refBackend.close();
- }
- }
-
- @Test
- public void testReadMetadataBytes() throws Exception {
- backend.init();
-
- String metadataName = "read-bytes-test";
- String content = "test bytes content";
-
- // Add metadata record
- backend.addMetadataRecord(new ByteArrayInputStream(content.getBytes()), metadataName);
-
- try {
- // Read metadata bytes using reflection
- byte[] bytes = (byte[]) MethodUtils.invokeMethod(backend, true, "readMetadataBytes", metadataName);
- assertNotNull("Bytes should not be null", bytes);
- assertEquals("Content should match", content, new String(bytes));
-
- // Test with non-existent metadata
- byte[] nullBytes = (byte[]) MethodUtils.invokeMethod(backend, true, "readMetadataBytes", "non-existent");
- assertNull("Non-existent metadata should return null", nullBytes);
- } finally {
- backend.deleteMetadataRecord(metadataName);
- }
- }
-
- // ========== DIRECT ACCESS FUNCTIONALITY TESTS ==========
-
- @Test
- public void testSetHttpDownloadURIExpirySeconds() throws Exception {
- // Test setting download URI expiry using reflection
- MethodUtils.invokeMethod(backend, true, "setHttpDownloadURIExpirySeconds", 3600);
-
- // Verify the field was set
- Field expiryField = AzureBlobStoreBackend.class.getDeclaredField("httpDownloadURIExpirySeconds");
- expiryField.setAccessible(true);
- int expiry = (int) expiryField.get(backend);
- assertEquals("Expiry should be set", 3600, expiry);
- }
-
- @Test
- public void testSetHttpUploadURIExpirySeconds() throws Exception {
- // Test setting upload URI expiry using reflection
- MethodUtils.invokeMethod(backend, true, "setHttpUploadURIExpirySeconds", 1800);
-
- // Verify the field was set
- Field expiryField = AzureBlobStoreBackend.class.getDeclaredField("httpUploadURIExpirySeconds");
- expiryField.setAccessible(true);
- int expiry = (int) expiryField.get(backend);
- assertEquals("Expiry should be set", 1800, expiry);
- }
-
- @Test
- public void testSetHttpDownloadURICacheSize() throws Exception {
- // Test setting cache size using reflection
- // Test with positive cache size
- MethodUtils.invokeMethod(backend, true, "setHttpDownloadURICacheSize", 100);
-
- Field cacheField = AzureBlobStoreBackend.class.getDeclaredField("httpDownloadURICache");
- cacheField.setAccessible(true);
- Cache<String, URI> cache = (Cache<String, URI>) cacheField.get(backend);
- assertNotNull("Cache should be created for positive size", cache);
-
- // Test with zero cache size (disabled)
- MethodUtils.invokeMethod(backend, true, "setHttpDownloadURICacheSize", 0);
- cache = (Cache<String, URI>) cacheField.get(backend);
- assertNull("Cache should be null for zero size", cache);
- }
-
- @Test
- public void testCreateHttpDownloadURI() throws Exception {
- backend.init();
-
- // Set up download URI configuration
- Properties propsWithDownload = createTestProperties();
- propsWithDownload.setProperty(PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "3600");
-
- AzureBlobStoreBackend downloadBackend = new AzureBlobStoreBackend();
- downloadBackend.setProperties(propsWithDownload);
- downloadBackend.init();
-
- try {
- // Create a test blob first
- File testFile = createTempFile("download-test");
- DataIdentifier identifier = new DataIdentifier("downloadtestblob");
- downloadBackend.write(identifier, testFile);
-
- // Create download URI using reflection
- DataRecordDownloadOptions options = DataRecordDownloadOptions.DEFAULT;
-
- MethodUtils.invokeMethod(downloadBackend, true, "createHttpDownloadURI",
- new Object[]{identifier, options},
- new Class<?>[]{DataIdentifier.class, DataRecordDownloadOptions.class});
- // Note: This may return null if the backend doesn't support presigned URIs in test environment
- // The important thing is that it doesn't throw an exception
-
- testFile.delete();
-
- //No exception should be thrown
- assertTrue("Should not throw exception", true);
- } finally {
- downloadBackend.close();
- }
-
-
- }
-
- @Test
- public void testCreateHttpDownloadURIWithNullIdentifier() throws Exception {
- backend.init();
-
- DataRecordDownloadOptions options = DataRecordDownloadOptions.DEFAULT;
-
- try {
- MethodUtils.invokeMethod(backend, true, "createHttpDownloadURI",
- new Object[]{null, options},
- new Class<?>[]{DataIdentifier.class, DataRecordDownloadOptions.class});
- fail("Expected NullPointerException for null identifier");
- } catch (InvocationTargetException e) {
- Throwable targetException = e.getTargetException();
- assertEquals("Exception should be NullPointerException", NullPointerException.class, targetException.getClass());
- assertEquals("Message should match","identifier must not be null", targetException.getMessage());
- }
- }
-
- @Test
- public void testCreateHttpDownloadURIWithNullOptions() throws Exception {
- backend.init();
-
- DataIdentifier identifier = new DataIdentifier("test");
-
- try {
- MethodUtils.invokeMethod(backend, true, "createHttpDownloadURI",
- new Object[]{identifier, null},
- new Class<?>[]{DataIdentifier.class, DataRecordDownloadOptions.class});
- fail("Expected NullPointerException for null options");
- } catch (InvocationTargetException e) {
- Throwable targetException = e.getTargetException();
- assertEquals("Exception should be NullPointerException", NullPointerException.class, targetException.getClass());
- assertEquals("Message should match","downloadOptions must not be null", targetException.getMessage());
- }
- }
-
- // ========== AZUREBLOBSTOREDATARECORD INNER CLASS TESTS ==========
-
- @Test
- public void testAzureBlobStoreDataRecordRegular() throws Exception {
- backend.init();
-
- // Create test file and write it
- File testFile = createTempFile("data-record-test");
- DataIdentifier identifier = new DataIdentifier("datarecordtest123");
-
- try {
- backend.write(identifier, testFile);
-
- // Get the data record
- DataRecord record = backend.getRecord(identifier);
- assertNotNull("Record should not be null", record);
-
- // Test getLength()
- assertEquals("Length should match file length", testFile.length(), record.getLength());
-
- // Test getLastModified()
- assertTrue("Last modified should be positive", record.getLastModified() > 0);
-
- // Test getIdentifier()
- assertEquals("Identifier should match", identifier, record.getIdentifier());
-
- // Test getStream()
- try (InputStream stream = record.getStream()) {
- String content = IOUtils.toString(stream, StandardCharsets.UTF_8);
- assertEquals("Content should match", "data-record-test", content);
- }
-
- // Test toString()
- String toString = record.toString();
- assertNotNull("toString should not be null", toString);
- assertTrue("toString should contain identifier", toString.contains(identifier.toString()));
- assertTrue("toString should contain length", toString.contains(String.valueOf(testFile.length())));
- assertTrue("toString should contain container name", toString.contains(CONTAINER_NAME));
- } finally {
- testFile.delete();
- }
- }
-
- @Test
- public void testAzureBlobStoreDataRecordMetadata() throws Exception {
- backend.init();
-
- String metadataName = "data-record-metadata-test";
- String content = "metadata record content";
-
- // Add metadata record
- backend.addMetadataRecord(new ByteArrayInputStream(content.getBytes()), metadataName);
-
- try {
- // Get the metadata record
- DataRecord record = backend.getMetadataRecord(metadataName);
- assertNotNull("Metadata record should not be null", record);
-
- // Test getLength()
- assertEquals("Length should match content length", content.length(), record.getLength());
-
- // Test getLastModified()
- assertTrue("Last modified should be positive", record.getLastModified() > 0);
-
- // Test getIdentifier()
- assertEquals("Identifier should match metadata name", metadataName, record.getIdentifier().toString());
-
- // Test getStream()
- try (InputStream stream = record.getStream()) {
- String readContent = IOUtils.toString(stream, StandardCharsets.UTF_8);
- assertEquals("Content should match", content, readContent);
- }
-
- // Test toString()
- String toString = record.toString();
- assertNotNull("toString should not be null", toString);
- assertTrue("toString should contain identifier", toString.contains(metadataName));
- assertTrue("toString should contain length", toString.contains(String.valueOf(content.length())));
- } finally {
- backend.deleteMetadataRecord(metadataName);
- }
- }
-
- // ========== CLOSE AND CLEANUP TESTS ==========
-
- @Test
- public void testClose() throws Exception {
- backend.init();
-
- // Should not throw exception
- backend.close();
-
- // Should be able to call close multiple times
- backend.close();
- backend.close();
-
- //No exception should be thrown
- assertTrue("Should not throw exception", true);
-
- // Should be able to use backend after close (since close() is empty)
- assertNotNull("Backend should still be usable", backend.getAzureContainer());
- }
-
- // ========== ERROR HANDLING AND EDGE CASES ==========
-
- @Test
- public void testInitWithInvalidConnectionString() {
- AzureBlobStoreBackend invalidBackend = new AzureBlobStoreBackend();
- Properties invalidProps = new Properties();
- invalidProps.setProperty(AZURE_CONNECTION_STRING, "invalid-connection-string");
- invalidProps.setProperty(AZURE_BLOB_CONTAINER_NAME, "test-container");
- invalidBackend.setProperties(invalidProps);
-
- try {
- invalidBackend.init();
- fail("Expected exception with invalid connection string");
- } catch (Exception e) {
- // Expected - can be DataStoreException or IllegalArgumentException
- assertNotNull("Exception should not be null", e);
- assertTrue("Should be DataStoreException or IllegalArgumentException",
- e instanceof DataStoreException || e instanceof IllegalArgumentException);
- }
- }
-
- @Test
- public void testInitWithMissingContainer() {
- Properties propsNoContainer = createTestProperties();
- propsNoContainer.remove(AZURE_BLOB_CONTAINER_NAME);
-
- AzureBlobStoreBackend noContainerBackend = new AzureBlobStoreBackend();
- noContainerBackend.setProperties(propsNoContainer);
-
- try {
- noContainerBackend.init();
- // If no exception is thrown, the backend might use a default container name
- // This is acceptable behavior
- } catch (Exception e) {
- // Exception is also acceptable - depends on implementation
- assertNotNull("Exception should not be null", e);
- }
- }
-
- @Test
- public void testInitWithCreateContainerDisabled() throws Exception {
- // Create container first
- container = azurite.getContainer(CONTAINER_NAME + "-nocreate", getConnectionString());
-
- Properties propsNoCreate = createTestProperties();
- propsNoCreate.setProperty(AZURE_BLOB_CONTAINER_NAME, CONTAINER_NAME + "-nocreate");
- propsNoCreate.setProperty(AZURE_CREATE_CONTAINER, "false");
-
- AzureBlobStoreBackend noCreateBackend = new AzureBlobStoreBackend();
- noCreateBackend.setProperties(propsNoCreate);
- noCreateBackend.init();
-
- assertNotNull("Backend should initialize with existing container", noCreateBackend);
- noCreateBackend.close();
- }
-
- // ========== HELPER METHODS ==========
-
- @Test
- public void testLargeFileHandling() throws Exception {
- backend.init();
-
- // Create a larger test file (1MB)
- File largeFile = File.createTempFile("large-test", ".tmp");
- try (FileWriter writer = new FileWriter(largeFile)) {
- StringBuilder sb = new StringBuilder();
- for (int i = 0; i < 10000; i++) {
- sb.append("This is line ").append(i).append(" of the large test file.\n");
- }
- writer.write(sb.toString());
- }
-
- DataIdentifier identifier = new DataIdentifier("largefiletest123");
-
- try {
- // Write large file
- backend.write(identifier, largeFile);
-
- // Verify it exists
- assertTrue("Large file should exist", backend.exists(identifier));
-
- // Read and verify content
- try (InputStream inputStream = backend.read(identifier)) {
- byte[] readBytes = IOUtils.toByteArray(inputStream);
- assertEquals("Content length should match", largeFile.length(), readBytes.length);
- }
-
- // Get record and verify
- DataRecord record = backend.getRecord(identifier);
- assertEquals("Record length should match file length", largeFile.length(), record.getLength());
- } finally {
- largeFile.delete();
- }
- }
-
- @Test
- public void testEmptyFileHandling() throws Exception {
- backend.init();
-
- // Create empty file
- File emptyFile = File.createTempFile("empty-test", ".tmp");
- DataIdentifier identifier = new DataIdentifier("emptyfiletest123");
-
- try {
- // Azure SDK doesn't support zero-length block sizes, so this should throw an exception
- backend.write(identifier, emptyFile);
- fail("Expected IllegalArgumentException for empty file");
- } catch (IllegalArgumentException e) {
- // Expected - Azure SDK doesn't allow zero-length block sizes
- assertTrue("Should mention block size", e.getMessage().contains("blockSize"));
- } catch (Exception e) {
- // Also acceptable if wrapped in another exception
- assertTrue("Should be related to empty file handling",
- e.getMessage().contains("blockSize") || e.getCause() instanceof IllegalArgumentException);
- } finally {
- emptyFile.delete();
- }
- }
-
- @Test
- public void testSpecialCharactersInIdentifier() throws Exception {
- backend.init();
-
- File testFile = createTempFile("special-chars-content");
- // Use identifier with special characters that are valid in blob names
- DataIdentifier identifier = new DataIdentifier("testfile123data");
-
- try {
- // Write file
- backend.write(identifier, testFile);
-
- // Verify operations work with special characters
- assertTrue("File with special chars should exist", backend.exists(identifier));
-
- DataRecord record = backend.getRecord(identifier);
- assertEquals("Identifier should match", identifier, record.getIdentifier());
-
- // Read content
- try (InputStream inputStream = backend.read(identifier)) {
- String content = IOUtils.toString(inputStream, StandardCharsets.UTF_8);
- assertEquals("Content should match", "special-chars-content", content);
- }
- } finally {
- testFile.delete();
- }
- }
-
- @Test
- public void testConcurrentOperations() throws Exception {
- backend.init();
-
- int threadCount = 5;
- int operationsPerThread = 10;
- ExecutorService executor = Executors.newFixedThreadPool(threadCount);
- CountDownLatch latch = new CountDownLatch(threadCount);
- AtomicInteger successCount = new AtomicInteger(0);
-
- List<Future<Void>> futures = new ArrayList<>();
-
- for (int t = 0; t < threadCount; t++) {
- final int threadId = t;
- futures.add(executor.submit(() -> {
- try {
- latch.countDown();
- latch.await(); // Wait for all threads to be ready
-
- for (int i = 0; i < operationsPerThread; i++) {
- String content = "Thread " + threadId + " operation " + i;
- File testFile = createTempFile(content);
- DataIdentifier identifier = new DataIdentifier("concurrent" + threadId + "op" + i);
-
- try {
- // Write
- backend.write(identifier, testFile);
-
- // Verify exists
- if (backend.exists(identifier)) {
- // Read back
- try (InputStream inputStream = backend.read(identifier)) {
- String readContent = IOUtils.toString(inputStream, StandardCharsets.UTF_8);
- if (content.equals(readContent)) {
- successCount.incrementAndGet();
- }
- }
- }
- } finally {
- testFile.delete();
- }
- }
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- return null;
- }));
- }
-
- // Wait for all operations to complete
- for (Future<Void> future : futures) {
- future.get(30, TimeUnit.SECONDS);
- }
-
- executor.shutdown();
-
- // Verify that most operations succeeded (allowing for some potential race conditions)
- int expectedSuccesses = threadCount * operationsPerThread;
- assertTrue("Most concurrent operations should succeed",
- successCount.get() >= expectedSuccesses * 0.8); // Allow 20% failure rate for race conditions
- }
-
- @Test
- public void testMetadataDirectoryStructure() throws Exception {
- backend.init();
-
- String metadataName = "directory-structure-test";
- String content = "directory test content";
-
- // Add metadata record
- backend.addMetadataRecord(new ByteArrayInputStream(content.getBytes()), metadataName);
-
- try {
- // Verify the record is stored with correct path prefix
- BlobContainerClient azureContainer = backend.getAzureContainer();
- String expectedBlobName = AZURE_BlOB_META_DIR_NAME + "/" + metadataName;
-
- BlobClient blobClient = azureContainer.getBlobClient(expectedBlobName);
- assertTrue("Blob should exist at expected path", blobClient.exists());
-
- // Verify the blob is in the META directory
- ListBlobsOptions listOptions = new ListBlobsOptions();
- listOptions.setPrefix(AZURE_BlOB_META_DIR_NAME);
-
- boolean foundBlob = false;
- for (BlobItem blobItem : azureContainer.listBlobs(listOptions, null)) {
- if (blobItem.getName().equals(expectedBlobName)) {
- foundBlob = true;
- break;
- }
- }
- assertTrue("Blob should be found in META directory listing", foundBlob);
- } finally {
- backend.deleteMetadataRecord(metadataName);
- }
- }
-
- // ========== ADDITIONAL COVERAGE TESTS ==========
-
- @Test
- public void testReadWithDebugLoggingEnabled() throws Exception {
- backend.init();
-
- // Create test file
- File testFile = createTempFile("debug-logging-test");
- DataIdentifier identifier = new DataIdentifier("debuglogtest123");
-
- try {
- // Write file first
- backend.write(identifier, testFile);
-
- // Set up logging capture
- ch.qos.logback.classic.Logger streamLogger =
- (ch.qos.logback.classic.Logger) LoggerFactory.getLogger("oak.datastore.download.streams");
- ch.qos.logback.classic.Level originalLevel = streamLogger.getLevel();
-
- // Create a list appender to capture log messages
- ListAppender<ILoggingEvent> listAppender = new ListAppender<>();
- listAppender.start();
- streamLogger.addAppender(listAppender);
- streamLogger.setLevel(ch.qos.logback.classic.Level.DEBUG);
-
- try {
- // Call read method to trigger debug logging - this should cover lines 253-255
- // Note: Due to a bug in the implementation, the InputStream is closed before being returned
- // But the debug logging happens before the stream is returned, so it will be executed
- try {
- InputStream inputStream = backend.read(identifier);
- // We don't actually need to use the stream, just calling read() is enough
- // to trigger the debug logging on lines 253-255
- assertNotNull("InputStream should not be null", inputStream);
- } catch (RuntimeException e) {
- // Expected due to the stream being closed prematurely in the implementation
- // But the debug logging should have been executed before this exception
- assertTrue("Should be stream closed error", e.getMessage().contains("Stream is already closed"));
- }
-
- // Verify that debug logging was captured
- boolean foundDebugLog = listAppender.list.stream()
- .anyMatch(event -> event.getMessage().contains("Binary downloaded from Azure Blob Storage"));
- assertTrue("Debug logging should have been executed for lines 253-255", foundDebugLog);
-
- } finally {
- // Clean up logging
- streamLogger.detachAppender(listAppender);
- streamLogger.setLevel(originalLevel);
- }
- } finally {
- testFile.delete();
- }
- }
-
- @Test
- public void testConcurrentRequestCountTooLow() throws Exception {
- Properties props = createTestProperties();
- props.setProperty(AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION, "1"); // Below minimum
-
- AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend();
- testBackend.setProperties(props);
- testBackend.init();
-
- // Should have been reset to default minimum
- // We can't directly access the field, but the init should complete successfully
- assertNotNull("Backend should initialize successfully", testBackend);
- }
-
- @Test
- public void testConcurrentRequestCountTooHigh() throws Exception {
- Properties props = createTestProperties();
- props.setProperty(AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION, "1000"); // Above maximum
-
- AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend();
- testBackend.setProperties(props);
- testBackend.init();
-
- // Should have been reset to default maximum
- assertNotNull("Backend should initialize successfully", testBackend);
- }
-
- @Test
- public void testRequestTimeoutConfiguration() throws Exception {
- Properties props = createTestProperties();
- props.setProperty(AzureConstants.AZURE_BLOB_REQUEST_TIMEOUT, "30000");
-
- AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend();
- testBackend.setProperties(props);
- testBackend.init();
-
- assertNotNull("Backend should initialize with request timeout", testBackend);
- }
-
- @Test
- public void testPresignedDownloadURIVerifyExistsDisabled() throws Exception {
- Properties props = createTestProperties();
- props.setProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_VERIFY_EXISTS, "false");
-
- AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend();
- testBackend.setProperties(props);
- testBackend.init();
-
- assertNotNull("Backend should initialize with verify exists disabled", testBackend);
- }
-
- @Test
- public void testCreateContainerDisabled() throws Exception {
- // First ensure container exists
- backend.init();
-
- Properties props = createTestProperties();
- props.setProperty(AZURE_CREATE_CONTAINER, "false");
-
- AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend();
- testBackend.setProperties(props);
- testBackend.init();
-
- assertNotNull("Backend should initialize without creating container", testBackend);
- }
-
- @Test
- public void testReferenceKeyInitializationDisabled() throws Exception {
- Properties props = createTestProperties();
- props.setProperty(AZURE_REF_ON_INIT, "false");
-
- AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend();
- testBackend.setProperties(props);
- testBackend.init();
-
- assertNotNull("Backend should initialize without reference key", testBackend);
- }
-
- @Test
- public void testHttpDownloadURICacheConfiguration() throws Exception {
- Properties props = createTestProperties();
- props.setProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "3600");
- props.setProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_CACHE_MAX_SIZE, "100");
-
- AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend();
- testBackend.setProperties(props);
- testBackend.init();
-
- assertNotNull("Backend should initialize with download URI cache", testBackend);
- }
-
- @Test
- public void testHttpDownloadURICacheDisabled() throws Exception {
- Properties props = createTestProperties();
- props.setProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "3600");
- // No cache max size property - should default to 0 (disabled)
-
- AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend();
- testBackend.setProperties(props);
- testBackend.init();
-
- assertNotNull("Backend should initialize with download URI cache disabled", testBackend);
- }
-
- @Test
- public void testUploadDomainOverride() throws Exception {
- Properties props = createTestProperties();
- props.setProperty(AzureConstants.PRESIGNED_HTTP_UPLOAD_URI_DOMAIN_OVERRIDE, "custom-upload.example.com");
-
- AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend();
- testBackend.setProperties(props);
- testBackend.init();
-
- assertNotNull("Backend should initialize with upload domain override", testBackend);
- }
-
- @Test
- public void testDownloadDomainOverride() throws Exception {
- Properties props = createTestProperties();
- props.setProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_DOMAIN_OVERRIDE, "custom-download.example.com");
-
- AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend();
- testBackend.setProperties(props);
- testBackend.init();
-
- assertNotNull("Backend should initialize with download domain override", testBackend);
- }
-
- @Test
- public void testGetLastModifiedWithoutMetadata() throws Exception {
- backend.init();
-
- // Create a blob without custom metadata
- File testFile = createTempFile("test content for last modified");
- DataIdentifier identifier = new DataIdentifier("lastmodifiedtest");
-
- try {
- backend.write(identifier, testFile);
-
- // Get the record - this should use the blob's native lastModified property
- DataRecord record = backend.getRecord(identifier);
- assertNotNull("Record should exist", record);
- assertTrue("Last modified should be positive", record.getLastModified() > 0);
- } finally {
- testFile.delete();
- try {
- backend.deleteRecord(identifier);
- } catch (Exception e) {
- // Ignore cleanup errors
- }
- }
- }
-
- @Test
- public void testInitiateHttpUploadInvalidParameters() throws Exception {
- backend.init();
-
- DataRecordUploadOptions options = DataRecordUploadOptions.DEFAULT;
-
- // Test maxUploadSizeInBytes <= 0
- try {
- backend.initiateHttpUpload(0L, 5, options);
- fail("Should throw IllegalArgumentException for maxUploadSizeInBytes <= 0");
- } catch (IllegalArgumentException e) {
- assertTrue("Should contain size error: " + e.getMessage(), e.getMessage().contains("maxUploadSizeInBytes must be > 0"));
- }
-
- // Test maxNumberOfURIs == 0
- try {
- backend.initiateHttpUpload(1000L, 0, options);
- fail("Should throw IllegalArgumentException for maxNumberOfURIs == 0");
- } catch (IllegalArgumentException e) {
- assertTrue("Should contain URI count error: " + e.getMessage(), e.getMessage().contains("maxNumberOfURIs must either be > 0 or -1"));
- }
-
- // Test maxNumberOfURIs < -1
- try {
- backend.initiateHttpUpload(1000L, -2, options);
- fail("Should throw IllegalArgumentException for maxNumberOfURIs < -1");
- } catch (IllegalArgumentException e) {
- assertTrue("Should contain URI count error: " + e.getMessage(), e.getMessage().contains("maxNumberOfURIs must either be > 0 or -1"));
- }
- }
-
- @Test
- public void testInitiateHttpUploadSinglePutTooLarge() throws Exception {
- Properties props = createTestProperties();
- props.setProperty(AzureConstants.PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS, "3600");
-
- AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend();
- testBackend.setProperties(props);
- testBackend.init();
-
- DataRecordUploadOptions options = DataRecordUploadOptions.DEFAULT;
-
- // Test single-put upload that exceeds max single-put size
- long tooLargeSize = 5L * 1024 * 1024 * 1024; // 5GB - exceeds Azure single-put limit
- try {
- testBackend.initiateHttpUpload(tooLargeSize, 1, options);
- fail("Should throw IllegalArgumentException for single-put upload too large");
- } catch (IllegalArgumentException e) {
- assertTrue("Should contain single-put size error: " + e.getMessage(),
- e.getMessage().contains("Cannot do single-put upload with file size"));
- }
- }
-
- @Test
- public void testInitiateHttpUploadWithValidParameters() throws Exception {
- Properties props = createTestProperties();
- props.setProperty(AzureConstants.PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS, "3600");
-
- AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend();
- testBackend.setProperties(props);
- testBackend.init();
-
- DataRecordUploadOptions options = DataRecordUploadOptions.DEFAULT;
-
- // Test with reasonable upload size that should work
- long reasonableSize = 100L * 1024 * 1024; // 100MB
- try {
- DataRecordUpload upload = testBackend.initiateHttpUpload(reasonableSize, 5, options);
- assertNotNull("Should successfully initiate upload with reasonable parameters", upload);
- } catch (Exception e) {
- // If upload initiation fails, it might be due to missing reference key or other setup issues
- // This is acceptable as we're mainly testing the parameter validation logic
- assertNotNull("Exception should have a message", e.getMessage());
- }
- }
-
- @Test
- public void testInitiateHttpUploadPartSizeTooLarge() throws Exception {
- Properties props = createTestProperties();
- props.setProperty(AzureConstants.PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS, "3600");
-
- AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend();
- testBackend.setProperties(props);
- testBackend.init();
-
- DataRecordUploadOptions options = DataRecordUploadOptions.DEFAULT;
-
- // Test with requested part size that's too large
- long uploadSize = 10L * 1024 * 1024 * 1024; // 10GB
- int maxURIs = 1; // This would create a part size > max allowed
- try {
- testBackend.initiateHttpUpload(uploadSize, maxURIs, options);
- fail("Should throw IllegalArgumentException for part size too large");
- } catch (IllegalArgumentException e) {
- assertTrue("Should contain part size error: " + e.getMessage(),
- e.getMessage().contains("Cannot do multi-part upload with requested part size") ||
- e.getMessage().contains("Cannot do single-put upload with file size"));
- } catch (Exception e) {
- // If the validation happens at a different level, accept other exceptions
- // as long as the upload is rejected
- assertNotNull("Should reject upload with invalid part size", e.getMessage());
- }
- }
-
- @Test
- public void testCreateHttpDownloadURINullIdentifier() throws Exception {
- Properties props = createTestProperties();
- props.setProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "3600");
-
- AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend();
- testBackend.setProperties(props);
- testBackend.init();
-
- DataRecordDownloadOptions options = DataRecordDownloadOptions.DEFAULT;
-
- try {
- testBackend.createHttpDownloadURI(null, options);
- fail("Should throw NullPointerException for null identifier");
- } catch (NullPointerException e) {
- assertEquals("identifier must not be null", e.getMessage());
- }
- }
-
- @Test
- public void testCreateHttpDownloadURINullOptions() throws Exception {
- Properties props = createTestProperties();
- props.setProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "3600");
-
- AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend();
- testBackend.setProperties(props);
- testBackend.init();
-
- DataIdentifier identifier = new DataIdentifier("test123");
-
- try {
- testBackend.createHttpDownloadURI(identifier, null);
- fail("Should throw NullPointerException for null options");
- } catch (NullPointerException e) {
- assertEquals("downloadOptions must not be null", e.getMessage());
- }
- }
-
- @Test
- public void testCreateHttpDownloadURIForNonExistentBlob() throws Exception {
- Properties props = createTestProperties();
- props.setProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "3600");
- props.setProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_VERIFY_EXISTS, "true");
-
- AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend();
- testBackend.setProperties(props);
- testBackend.init();
-
- DataIdentifier nonExistentId = new DataIdentifier("nonexistent123");
- DataRecordDownloadOptions options = DataRecordDownloadOptions.DEFAULT;
-
- URI downloadURI = testBackend.createHttpDownloadURI(nonExistentId, options);
- assertNull("Should return null for non-existent blob when verify exists is enabled", downloadURI);
- }
-
- @Test
- public void testWriteBlobWithLengthCollision() throws Exception {
- backend.init();
-
- // Create initial blob
- String content1 = "initial content";
- File testFile1 = createTempFile(content1);
- DataIdentifier identifier = new DataIdentifier("lengthcollisiontest");
-
- try {
- backend.write(identifier, testFile1);
-
- // Try to write different content with different length using same identifier
- String content2 = "different content with different length";
- File testFile2 = createTempFile(content2);
-
- try {
- backend.write(identifier, testFile2);
- fail("Should throw DataStoreException for length collision");
- } catch (DataStoreException e) {
- assertTrue("Should contain length collision error",
- e.getMessage().contains("Length Collision"));
- } finally {
- testFile2.delete();
- }
- } finally {
- testFile1.delete();
- try {
- backend.deleteRecord(identifier);
- } catch (Exception e) {
- // Ignore cleanup errors
- }
- }
- }
-
- @Test
- public void testBlobStorageExceptionHandling() {
- // Test with invalid connection string to trigger exception handling
- Properties invalidProps = new Properties();
- invalidProps.setProperty(AZURE_BLOB_CONTAINER_NAME, CONTAINER_NAME);
- invalidProps.setProperty(AZURE_CONNECTION_STRING, "invalid-connection-string");
-
- AzureBlobStoreBackend invalidBackend = new AzureBlobStoreBackend();
- invalidBackend.setProperties(invalidProps);
-
- try {
- invalidBackend.init();
- fail("Should throw exception for invalid connection string");
- } catch (Exception e) {
- // Expected - invalid connection string should cause initialization to fail
- // Could be DataStoreException or IllegalArgumentException depending on validation level
- assertNotNull("Exception should have a message", e.getMessage());
- assertTrue("Should be a relevant exception type",
- e instanceof DataStoreException || e instanceof IllegalArgumentException);
- }
- }
-
- @Test
- public void testHttpDownloadURICacheHit() throws Exception {
- Properties props = createTestProperties();
- props.setProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "3600");
- props.setProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_CACHE_MAX_SIZE, "10");
- props.setProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_VERIFY_EXISTS, "false");
-
- AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend();
- testBackend.setProperties(props);
- testBackend.init();
-
- // Create a blob first
- File testFile = createTempFile("cache test content");
- DataIdentifier identifier = new DataIdentifier("cachetest");
-
- try {
- testBackend.write(identifier, testFile);
-
- DataRecordDownloadOptions options = DataRecordDownloadOptions.DEFAULT;
-
- // First call should create and cache the URI
- URI uri1 = testBackend.createHttpDownloadURI(identifier, options);
- assertNotNull("First URI should not be null", uri1);
-
- // Second call should hit the cache and return the same URI
- URI uri2 = testBackend.createHttpDownloadURI(identifier, options);
- assertNotNull("Second URI should not be null", uri2);
-
- // URIs should be the same (cache hit)
- assertEquals("URIs should be identical (cache hit)", uri1, uri2);
- } finally {
- testFile.delete();
- try {
- testBackend.deleteRecord(identifier);
- } catch (Exception e) {
- // Ignore cleanup errors
- }
- }
- }
-
- @Test
- public void testHttpDownloadURIWithoutExpiry() throws Exception {
- Properties props = createTestProperties();
- // Don't set expiry seconds - should default to 0 (disabled)
-
- AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend();
- testBackend.setProperties(props);
- testBackend.init();
+import com.microsoft.azure.storage.StorageException;
+import com.microsoft.azure.storage.blob.CloudBlobContainer;
+import com.microsoft.azure.storage.blob.SharedAccessBlobPermissions;
+import com.microsoft.azure.storage.blob.SharedAccessBlobPolicy;
+import org.apache.jackrabbit.core.data.DataRecord;
+import org.apache.jackrabbit.core.data.DataStoreException;
+import org.jetbrains.annotations.NotNull;
+import org.junit.After;
+import org.junit.ClassRule;
+import org.junit.Test;
- DataIdentifier identifier = new DataIdentifier("noexpirytest");
- DataRecordDownloadOptions options = DataRecordDownloadOptions.DEFAULT;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Date;
+import java.util.EnumSet;
+import java.util.Properties;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
- URI downloadURI = testBackend.createHttpDownloadURI(identifier, options);
- assertNull("Should return null when download URI expiry is disabled", downloadURI);
- }
+import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.ADD;
+import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.CREATE;
+import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.LIST;
+import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.READ;
+import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.WRITE;
+import static java.util.stream.Collectors.toSet;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.junit.Assume.assumeNotNull;
- @Test
- public void testCompleteHttpUploadWithMissingRecord() throws Exception {
- Properties props = createTestProperties();
- props.setProperty(AzureConstants.PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS, "3600");
+public class AzureBlobStoreBackendTest {
+ private static final String AZURE_ACCOUNT_NAME = "AZURE_ACCOUNT_NAME";
+ private static final String AZURE_TENANT_ID = "AZURE_TENANT_ID";
+ private static final String AZURE_CLIENT_ID = "AZURE_CLIENT_ID";
+ private static final String AZURE_CLIENT_SECRET = "AZURE_CLIENT_SECRET";
+ @ClassRule
+ public static AzuriteDockerRule azurite = new AzuriteDockerRule();
- AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend();
- testBackend.setProperties(props);
- testBackend.init();
+ private static final String CONTAINER_NAME = "blobstore";
+ private static final EnumSet<SharedAccessBlobPermissions> READ_ONLY = EnumSet.of(READ, LIST);
+ private static final EnumSet<SharedAccessBlobPermissions> READ_WRITE = EnumSet.of(READ, LIST, CREATE, WRITE, ADD);
+ private static final Set<String> BLOBS = Set.of("blob1", "blob2");
- // Create a fake upload token for a non-existent blob
- String fakeToken = "fake-upload-token-for-nonexistent-blob";
+ private CloudBlobContainer container;
- try {
- testBackend.completeHttpUpload(fakeToken);
- fail("Should throw exception for invalid token");
- } catch (Exception e) {
- // Expected - invalid token should cause completion to fail
- // Could be various exception types depending on where validation fails
- assertNotNull("Should reject invalid upload token", e.getMessage());
+ @After
+ public void tearDown() throws Exception {
+ if (container != null) {
+ container.deleteIfExists();
}
}
- // ========== INTEGRATION TESTS (from AzureBlobStoreBackendIT) ==========
-
@Test
public void initWithSharedAccessSignature_readOnly() throws Exception {
- BlobContainerClient container = createBlobContainer();
- OffsetDateTime expiryTime = OffsetDateTime.now().plusDays(7);
- BlobSasPermission permissions = new BlobSasPermission().setReadPermission(true)
- .setWritePermission(false)
- .setListPermission(true);
-
- BlobServiceSasSignatureValues sasValues = new BlobServiceSasSignatureValues(expiryTime, permissions);
- String sasToken = container.generateSas(sasValues);
+ CloudBlobContainer container = createBlobContainer();
+ String sasToken = container.generateSharedAccessSignature(policy(READ_ONLY), null);
AzureBlobStoreBackend azureBlobStoreBackend = new AzureBlobStoreBackend();
azureBlobStoreBackend.setProperties(getConfigurationWithSasToken(sasToken));
@@ -2108,16 +91,8 @@ public class AzureBlobStoreBackendTest {
@Test
public void initWithSharedAccessSignature_readWrite() throws Exception {
- BlobContainerClient container = createBlobContainer();
- OffsetDateTime expiryTime = OffsetDateTime.now().plusDays(7);
- BlobSasPermission permissions = new BlobSasPermission().setReadPermission(true)
- .setListPermission(true)
- .setAddPermission(true)
- .setCreatePermission(true)
- .setWritePermission(true);
-
- BlobServiceSasSignatureValues sasValues = new BlobServiceSasSignatureValues(expiryTime, permissions);
- String sasToken = container.generateSas(sasValues);
+ CloudBlobContainer container = createBlobContainer();
+ String sasToken = container.generateSharedAccessSignature(policy(READ_WRITE), null);
AzureBlobStoreBackend azureBlobStoreBackend = new AzureBlobStoreBackend();
azureBlobStoreBackend.setProperties(getConfigurationWithSasToken(sasToken));
@@ -2131,14 +106,9 @@ public class AzureBlobStoreBackendTest {
@Test
public void connectWithSharedAccessSignatureURL_expired() throws Exception {
- BlobContainerClient container = createBlobContainer();
-
- OffsetDateTime expiryTime = OffsetDateTime.now().minusDays(1);
- BlobSasPermission permissions = new BlobSasPermission().setReadPermission(true)
- .setWritePermission(true);
-
- BlobServiceSasSignatureValues sasValues = new BlobServiceSasSignatureValues(expiryTime, permissions);
- String sasToken = container.generateSas(sasValues);
+ CloudBlobContainer container = createBlobContainer();
+ SharedAccessBlobPolicy expiredPolicy = policy(READ_WRITE, yesterday());
+ String sasToken = container.generateSharedAccessSignature(expiredPolicy, null);
AzureBlobStoreBackend azureBlobStoreBackend = new AzureBlobStoreBackend();
azureBlobStoreBackend.setProperties(getConfigurationWithSasToken(sasToken));
@@ -2180,13 +150,18 @@ public class AzureBlobStoreBackendTest {
assertReferenceSecret(azureBlobStoreBackend);
}
+ /* make sure that blob1.txt and blob2.txt are uploaded to AZURE_ACCOUNT_NAME/blobstore container before
+ * executing this test
+ * */
@Test
public void initWithServicePrincipals() throws Exception {
- // Create blob container with test blobs using Azurite
- createBlobContainer();
+ assumeNotNull(getEnvironmentVariable(AZURE_ACCOUNT_NAME));
+ assumeNotNull(getEnvironmentVariable(AZURE_TENANT_ID));
+ assumeNotNull(getEnvironmentVariable(AZURE_CLIENT_ID));
+ assumeNotNull(getEnvironmentVariable(AZURE_CLIENT_SECRET));
AzureBlobStoreBackend azureBlobStoreBackend = new AzureBlobStoreBackend();
- azureBlobStoreBackend.setProperties(getConfigurationWithConnectionString());
+ azureBlobStoreBackend.setProperties(getPropertiesWithServicePrincipals());
azureBlobStoreBackend.init();
@@ -2194,57 +169,29 @@ public class AzureBlobStoreBackendTest {
assertReadAccessGranted(azureBlobStoreBackend, concat(BLOBS, "test"));
}
- @Test
- public void testMetadataOperationsWithRenamedConstants() throws Exception {
- createBlobContainer();
-
- AzureBlobStoreBackend azureBlobStoreBackend = new AzureBlobStoreBackend();
- azureBlobStoreBackend.setProperties(getConfigurationWithConnectionString());
- azureBlobStoreBackend.init();
-
- // Test that metadata operations work correctly with the renamed constants
- String testMetadataName = "test-metadata-record";
- String testContent = "test metadata content";
-
- // Add a metadata record
- azureBlobStoreBackend.addMetadataRecord(new ByteArrayInputStream(testContent.getBytes()), testMetadataName);
+ private Properties getPropertiesWithServicePrincipals() {
+ final String accountName = getEnvironmentVariable(AZURE_ACCOUNT_NAME);
+ final String tenantId = getEnvironmentVariable(AZURE_TENANT_ID);
+ final String clientId = getEnvironmentVariable(AZURE_CLIENT_ID);
+ final String clientSecret = getEnvironmentVariable(AZURE_CLIENT_SECRET);
- // Verify the record exists
- assertTrue("Metadata record should exist", azureBlobStoreBackend.metadataRecordExists(testMetadataName));
-
- // Retrieve the record
- DataRecord retrievedRecord = azureBlobStoreBackend.getMetadataRecord(testMetadataName);
- assertNotNull("Retrieved metadata record should not be null", retrievedRecord);
- assertEquals("Retrieved record should have correct length", testContent.length(), retrievedRecord.getLength());
-
- // Verify the record appears in getAllMetadataRecords
- List<DataRecord> allRecords = azureBlobStoreBackend.getAllMetadataRecords("");
- boolean foundTestRecord = allRecords.stream()
- .anyMatch(record -> record.getIdentifier().toString().equals(testMetadataName));
- assertTrue("Test metadata record should be found in getAllMetadataRecords", foundTestRecord);
-
- // Clean up - delete the test record
- azureBlobStoreBackend.deleteMetadataRecord(testMetadataName);
- assertFalse("Metadata record should be deleted", azureBlobStoreBackend.metadataRecordExists(testMetadataName));
+ Properties properties = new Properties();
+ properties.setProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, accountName);
+ properties.setProperty(AzureConstants.AZURE_TENANT_ID, tenantId);
+ properties.setProperty(AzureConstants.AZURE_CLIENT_ID, clientId);
+ properties.setProperty(AzureConstants.AZURE_CLIENT_SECRET, clientSecret);
+ properties.setProperty(AzureConstants.AZURE_BLOB_CONTAINER_NAME, CONTAINER_NAME);
+ return properties;
}
- // ========== HELPER METHODS ==========
-
- private File createTempFile(String content) throws IOException {
- File tempFile = File.createTempFile("azure-test", ".tmp");
- try (FileWriter writer = new FileWriter(tempFile)) {
- writer.write(content);
- }
- return tempFile;
+ private String getEnvironmentVariable(String variableName) {
+ return System.getenv(variableName);
}
- private BlobContainerClient createBlobContainer() {
- container = azurite.getContainer(CONTAINER_NAME, getConnectionString());
+ private CloudBlobContainer createBlobContainer() throws Exception {
+ container = azurite.getContainer("blobstore");
for (String blob : BLOBS) {
- InputStream blobStream = new BufferedInputStream(new ByteArrayInputStream(blob.getBytes()));
- BlobClient blobClient = container.getBlobClient(blob + ".txt");
- long length = blob.getBytes().length;
- blobClient.upload(blobStream, length, true);
+ container.getBlockBlobReference(blob + ".txt").uploadText(blob);
}
return container;
}
@@ -2280,11 +227,25 @@ public class AzureBlobStoreBackendTest {
return properties;
}
+ @NotNull
+ private static SharedAccessBlobPolicy policy(EnumSet<SharedAccessBlobPermissions> permissions, Instant expirationTime) {
+ SharedAccessBlobPolicy sharedAccessBlobPolicy = new SharedAccessBlobPolicy();
+ sharedAccessBlobPolicy.setPermissions(permissions);
+ sharedAccessBlobPolicy.setSharedAccessExpiryTime(Date.from(expirationTime));
+ return sharedAccessBlobPolicy;
+ }
+
+ @NotNull
+ private static SharedAccessBlobPolicy policy(EnumSet<SharedAccessBlobPermissions> permissions) {
+ return policy(permissions, Instant.now().plus(Duration.ofDays(7)));
+ }
+
private static void assertReadAccessGranted(AzureBlobStoreBackend backend, Set<String> expectedBlobs) throws Exception {
- BlobContainerClient container = backend.getAzureContainer();
+ CloudBlobContainer container = backend.getAzureContainer();
Set<String> actualBlobNames = StreamSupport.stream(container.listBlobs().spliterator(), false)
- .map(blobItem -> container.getBlobClient(blobItem.getName()).getBlobName())
- .filter(name -> !name.contains(AZURE_BlOB_META_DIR_NAME))
+ .map(blob -> blob.getUri().getPath())
+ .map(path -> path.substring(path.lastIndexOf('/') + 1))
+ .filter(path -> !path.isEmpty())
.collect(toSet());
Set<String> expectedBlobNames = expectedBlobs.stream().map(name -> name + ".txt").collect(toSet());
@@ -2294,8 +255,8 @@ public class AzureBlobStoreBackendTest {
Set<String> actualBlobContent = actualBlobNames.stream()
.map(name -> {
try {
- return container.getBlobClient(name).getBlockBlobClient().downloadContent().toString();
- } catch (Exception e) {
+ return container.getBlockBlobReference(name).downloadText();
+ } catch (StorageException | IOException | URISyntaxException e) {
throw new RuntimeException("Error while reading blob " + name, e);
}
})
@@ -2304,10 +265,8 @@ public class AzureBlobStoreBackendTest {
}
private static void assertWriteAccessGranted(AzureBlobStoreBackend backend, String blob) throws Exception {
- InputStream blobStream = new ByteArrayInputStream(blob.getBytes());
backend.getAzureContainer()
- .getBlobClient(blob + ".txt")
- .upload(blobStream, blob.getBytes().length, true);
+ .getBlockBlobReference(blob + ".txt").uploadText(blob);
}
private static void assertWriteAccessNotGranted(AzureBlobStoreBackend backend) {
@@ -2328,159 +287,23 @@ public class AzureBlobStoreBackendTest {
}
}
+ private static Instant yesterday() {
+ return Instant.now().minus(Duration.ofDays(1));
+ }
+
private static Set<String> concat(Set<String> set, String element) {
return Stream.concat(set.stream(), Stream.of(element)).collect(Collectors.toSet());
}
- private static void assertReferenceSecret(AzureBlobStoreBackend AzureBlobStoreBackend)
- throws DataStoreException {
+ private static String getConnectionString() {
+ return Utils.getConnectionString(AzuriteDockerRule.ACCOUNT_NAME, AzuriteDockerRule.ACCOUNT_KEY, azurite.getBlobEndpoint());
+ }
+
+ private static void assertReferenceSecret(AzureBlobStoreBackend azureBlobStoreBackend)
+ throws DataStoreException, IOException {
// assert secret already created on init
- DataRecord refRec = AzureBlobStoreBackend.getMetadataRecord("reference.key");
+ DataRecord refRec = azureBlobStoreBackend.getMetadataRecord("reference.key");
assertNotNull("Reference data record null", refRec);
assertTrue("reference key is empty", refRec.getLength() > 0);
}
-
- /**
- * Test that headers are properly included in presigned download URIs.
- * This test verifies the fix for the critical issue where headers were being ignored.
- */
- @Test
- public void testCreateHttpDownloadURIWithHeaders() throws Exception {
- // Set up download URI configuration
- Properties propsWithDownload = createTestProperties();
- propsWithDownload.setProperty(PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "3600");
-
- AzureBlobStoreBackend downloadBackend = new AzureBlobStoreBackend();
- downloadBackend.setProperties(propsWithDownload);
- downloadBackend.init();
-
- try {
- // Create a test blob first
- File testFile = createTempFile("header-test");
- DataIdentifier identifier = new DataIdentifier("headertestblob");
- downloadBackend.write(identifier, testFile);
-
- // Create download options with custom headers
- String expectedContentType = "image/png";
- String expectedFileName = "test-image.png";
- DataRecordDownloadOptions options = DataRecordDownloadOptions.fromBlobDownloadOptions(
- new BlobDownloadOptions(
- expectedContentType,
- null,
- expectedFileName,
- "attachment"
- )
- );
-
- // Create download URI using reflection
- URI uri = (URI) MethodUtils.invokeMethod(downloadBackend, true, "createHttpDownloadURI",
- new Object[]{identifier, options},
- new Class<?>[]{DataIdentifier.class, DataRecordDownloadOptions.class});
-
- // Verify URI was created
- assertNotNull("Download URI should not be null", uri);
-
- // Verify the URI contains SAS parameters
- String uriString = uri.toString();
- assertTrue("URI should contain SAS signature", uriString.contains("sig="));
- assertTrue("URI should contain expiry", uriString.contains("se="));
- assertTrue("URI should contain permissions", uriString.contains("sp="));
-
- // Verify headers are encoded in the SAS token
- // The Azure SDK encodes headers in the SAS signature
- // We verify by checking that the rscc (cache-control), rsct (content-type),
- // and rscd (content-disposition) parameters are present
- assertTrue("URI should contain cache-control parameter (rscc)",
- uriString.contains("rscc=") || uriString.contains("&rscc") || uriString.contains("?rscc"));
- assertTrue("URI should contain content-type parameter (rsct)",
- uriString.contains("rsct=") || uriString.contains("&rsct") || uriString.contains("?rsct"));
- assertTrue("URI should contain content-disposition parameter (rscd)",
- uriString.contains("rscd=") || uriString.contains("&rscd") || uriString.contains("?rscd"));
-
- testFile.delete();
- } finally {
- downloadBackend.close();
- }
- }
-
- /**
- * Test that default headers (cache-control) are included even without custom content headers.
- */
- @Test
- public void testCreateHttpDownloadURIWithDefaultHeaders() throws Exception {
- Properties propsWithDownload = createTestProperties();
- propsWithDownload.setProperty(PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "3600");
-
- AzureBlobStoreBackend downloadBackend = new AzureBlobStoreBackend();
- downloadBackend.setProperties(propsWithDownload);
- downloadBackend.init();
-
- try {
- File testFile = createTempFile("default-header-test");
- DataIdentifier identifier = new DataIdentifier("defaultheadertestblob");
- downloadBackend.write(identifier, testFile);
-
- // Use default options (no custom headers)
- DataRecordDownloadOptions options = DataRecordDownloadOptions.DEFAULT;
-
- URI uri = (URI) MethodUtils.invokeMethod(downloadBackend, true, "createHttpDownloadURI",
- new Object[]{identifier, options},
- new Class<?>[]{DataIdentifier.class, DataRecordDownloadOptions.class});
-
- assertNotNull("Download URI should not be null", uri);
-
- String uriString = uri.toString();
- // Cache-control should always be present
- assertTrue("URI should contain cache-control parameter",
- uriString.contains("rscc=") || uriString.contains("&rscc") || uriString.contains("?rscc"));
-
- testFile.delete();
- } finally {
- downloadBackend.close();
- }
- }
-
- /**
- * Test that content-disposition header is properly formatted with filename.
- */
- @Test
- public void testCreateHttpDownloadURIWithContentDisposition() throws Exception {
- Properties propsWithDownload = createTestProperties();
- propsWithDownload.setProperty(PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "3600");
-
- AzureBlobStoreBackend downloadBackend = new AzureBlobStoreBackend();
- downloadBackend.setProperties(propsWithDownload);
- downloadBackend.init();
-
- try {
- File testFile = createTempFile("disposition-test");
- DataIdentifier identifier = new DataIdentifier("dispositiontestblob");
- downloadBackend.write(identifier, testFile);
-
- // Create options with filename
- String fileName = "my-document.pdf";
- DataRecordDownloadOptions options = DataRecordDownloadOptions.fromBlobDownloadOptions(
- new BlobDownloadOptions(
- "application/pdf",
- null,
- fileName,
- "attachment"
- )
- );
-
- URI uri = (URI) MethodUtils.invokeMethod(downloadBackend, true, "createHttpDownloadURI",
- new Object[]{identifier, options},
- new Class<?>[]{DataIdentifier.class, DataRecordDownloadOptions.class});
-
- assertNotNull("Download URI should not be null", uri);
-
- String uriString = uri.toString();
- assertTrue("URI should contain content-disposition parameter",
- uriString.contains("rscd=") || uriString.contains("&rscd") || uriString.contains("?rscd"));
-
- testFile.delete();
- } finally {
- downloadBackend.close();
- }
- }
}
diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureConstantsTest.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureConstantsTest.java
deleted file mode 100644
index f259b91ca2..0000000000
--- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureConstantsTest.java
+++ /dev/null
@@ -1,243 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;
-
-import org.junit.Test;
-
-import static org.junit.Assert.*;
-
-/**
- * Test class for AzureConstants to ensure constant values are correct
- * and that changes don't break existing functionality.
- */
-public class AzureConstantsTest {
-
- @Test
- public void testMetadataDirectoryConstant() {
- // Test that the metadata directory name constant has the expected value
- assertEquals("META directory name should be 'META'", "META", AzureConstants.AZURE_BlOB_META_DIR_NAME);
-
- // Ensure the constant is not null or empty
- assertNotNull("META directory name should not be null", AzureConstants.AZURE_BlOB_META_DIR_NAME);
- assertFalse("META directory name should not be empty", AzureConstants.AZURE_BlOB_META_DIR_NAME.isEmpty());
- }
-
- @Test
- public void testMetadataKeyPrefixConstant() {
- // Test that the metadata key prefix is correctly constructed
- String expectedPrefix = AzureConstants.AZURE_BlOB_META_DIR_NAME + "/";
- assertEquals("META key prefix should be constructed correctly", expectedPrefix, AzureConstants.AZURE_BLOB_META_KEY_PREFIX);
-
- // Verify it ends with a slash for directory structure
- assertTrue("META key prefix should end with '/'", AzureConstants.AZURE_BLOB_META_KEY_PREFIX.endsWith("/"));
-
- // Verify it starts with the directory name
- assertTrue("META key prefix should start with directory name",
- AzureConstants.AZURE_BLOB_META_KEY_PREFIX.startsWith(AzureConstants.AZURE_BlOB_META_DIR_NAME));
- }
-
- @Test
- public void testMetadataKeyPrefixValue() {
- // Test the exact expected value
- assertEquals("META key prefix should be 'META/'", "META/", AzureConstants.AZURE_BLOB_META_KEY_PREFIX);
-
- // Ensure the constant is not null or empty
- assertNotNull("META key prefix should not be null", AzureConstants.AZURE_BLOB_META_KEY_PREFIX);
- assertFalse("META key prefix should not be empty", AzureConstants.AZURE_BLOB_META_KEY_PREFIX.isEmpty());
- }
-
- @Test
- public void testBlobReferenceKeyConstant() {
- // Test that the blob reference key constant has the expected value
- assertEquals("Blob reference key should be 'reference.key'", "reference.key", AzureConstants.AZURE_BLOB_REF_KEY);
-
- // Ensure the constant is not null or empty
- assertNotNull("Blob reference key should not be null", AzureConstants.AZURE_BLOB_REF_KEY);
- assertFalse("Blob reference key should not be empty", AzureConstants.AZURE_BLOB_REF_KEY.isEmpty());
- }
-
- @Test
- public void testConstantConsistency() {
- // Test that the constants are consistent with each other
- String expectedKeyPrefix = AzureConstants.AZURE_BlOB_META_DIR_NAME + "/";
- assertEquals("Key prefix should be consistent with directory name",
- expectedKeyPrefix, AzureConstants.AZURE_BLOB_META_KEY_PREFIX);
- }
-
- @Test
- public void testConstantImmutability() {
- // Test that constants are final and cannot be changed (compile-time check)
- // This test verifies that the constants exist and have expected types
- assertTrue("META directory name should be a String", AzureConstants.AZURE_BlOB_META_DIR_NAME instanceof String);
- assertTrue("META key prefix should be a String", AzureConstants.AZURE_BLOB_META_KEY_PREFIX instanceof String);
- assertTrue("Blob reference key should be a String", AzureConstants.AZURE_BLOB_REF_KEY instanceof String);
- }
-
- @Test
- public void testMetadataPathConstruction() {
- // Test that metadata paths can be constructed correctly using the constants
- String testFileName = "test-metadata.json";
- String expectedPath = AzureConstants.AZURE_BLOB_META_KEY_PREFIX + testFileName;
- String actualPath = AzureConstants.AZURE_BlOB_META_DIR_NAME + "/" + testFileName;
-
- assertEquals("Metadata path construction should be consistent", expectedPath, actualPath);
- assertEquals("Metadata path should start with META/", "META/" + testFileName, expectedPath);
- }
-
- @Test
- public void testConstantNamingConvention() {
- // Test that the constant names follow expected naming conventions
-
- // These tests verify that the constants exist by accessing them
- // If the constant names were changed incorrectly, these would fail at compile time
- assertNotNull("Directory constant should exist", AzureConstants.AZURE_BlOB_META_DIR_NAME);
- assertNotNull("Prefix constant should exist", AzureConstants.AZURE_BLOB_META_KEY_PREFIX);
- assertNotNull("Reference key constant should exist", AzureConstants.AZURE_BLOB_REF_KEY);
- }
-
- @Test
- public void testBackwardCompatibility() {
- // Test that the constant values maintain backward compatibility
- // These values should not change as they affect stored data structure
- assertEquals("META directory name must remain 'META' for backward compatibility",
- "META", AzureConstants.AZURE_BlOB_META_DIR_NAME);
- assertEquals("Reference key must remain 'reference.key' for backward compatibility",
- "reference.key", AzureConstants.AZURE_BLOB_REF_KEY);
- }
-
- @Test
- public void testAllStringConstants() {
- // Test all string constants have expected values
- assertEquals("accessKey", AzureConstants.AZURE_STORAGE_ACCOUNT_NAME);
- assertEquals("secretKey", AzureConstants.AZURE_STORAGE_ACCOUNT_KEY);
- assertEquals("azureConnectionString", AzureConstants.AZURE_CONNECTION_STRING);
- assertEquals("azureSas", AzureConstants.AZURE_SAS);
- assertEquals("tenantId", AzureConstants.AZURE_TENANT_ID);
- assertEquals("clientId", AzureConstants.AZURE_CLIENT_ID);
- assertEquals("clientSecret", AzureConstants.AZURE_CLIENT_SECRET);
- assertEquals("azureBlobEndpoint", AzureConstants.AZURE_BLOB_ENDPOINT);
- assertEquals("container", AzureConstants.AZURE_BLOB_CONTAINER_NAME);
- assertEquals("azureCreateContainer", AzureConstants.AZURE_CREATE_CONTAINER);
- assertEquals("socketTimeout", AzureConstants.AZURE_BLOB_REQUEST_TIMEOUT);
- assertEquals("maxErrorRetry", AzureConstants.AZURE_BLOB_MAX_REQUEST_RETRY);
- assertEquals("maxConnections", AzureConstants.AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION);
- assertEquals("enableSecondaryLocation", AzureConstants.AZURE_BLOB_ENABLE_SECONDARY_LOCATION_NAME);
- assertEquals("proxyHost", AzureConstants.PROXY_HOST);
- assertEquals("proxyPort", AzureConstants.PROXY_PORT);
- assertEquals("lastModified", AzureConstants.AZURE_BLOB_LAST_MODIFIED_KEY);
- assertEquals("refOnInit", AzureConstants.AZURE_REF_ON_INIT);
- }
-
- @Test
- public void testPresignedURIConstants() {
- // Test presigned URI related constants
- assertEquals("presignedHttpUploadURIExpirySeconds", AzureConstants.PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS);
- assertEquals("presignedHttpDownloadURIExpirySeconds", AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS);
- assertEquals("presignedHttpDownloadURICacheMaxSize", AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_CACHE_MAX_SIZE);
- assertEquals("presignedHttpDownloadURIVerifyExists", AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_VERIFY_EXISTS);
- assertEquals("presignedHttpDownloadURIDomainOverride", AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_DOMAIN_OVERRIDE);
- assertEquals("presignedHttpUploadURIDomainOverride", AzureConstants.PRESIGNED_HTTP_UPLOAD_URI_DOMAIN_OVERRIDE);
- }
-
- @Test
- public void testNumericConstants() {
- // Test all numeric constants have expected values
- assertFalse("Secondary location should be disabled by default", AzureConstants.AZURE_BLOB_ENABLE_SECONDARY_LOCATION_DEFAULT);
- assertEquals("Buffered stream threshold should be 8MB", 8L * 1024L * 1024L, AzureConstants.AZURE_BLOB_BUFFERED_STREAM_THRESHOLD);
- assertEquals("Min multipart upload part size should be 256KB", 256L * 1024L, AzureConstants.AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE);
- assertEquals("Max multipart upload part size should be 4000MB", 4000L * 1024L * 1024L, AzureConstants.AZURE_BLOB_MAX_MULTIPART_UPLOAD_PART_SIZE);
- assertEquals("Max single PUT upload size should be 256MB", 256L * 1024L * 1024L, AzureConstants.AZURE_BLOB_MAX_SINGLE_PUT_UPLOAD_SIZE);
- assertEquals("Max binary upload size should be ~190.7TB", 190L * 1024L * 1024L * 1024L * 1024L, AzureConstants.AZURE_BLOB_MAX_BINARY_UPLOAD_SIZE);
- assertEquals("Max allowable upload URIs should be 50000", 50000, AzureConstants.AZURE_BLOB_MAX_ALLOWABLE_UPLOAD_URIS);
- assertEquals("Max unique record tries should be 10", 10, AzureConstants.AZURE_BLOB_MAX_UNIQUE_RECORD_TRIES);
- assertEquals("Default concurrent request count should be 5", 5, AzureConstants.AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT);
- assertEquals("Max concurrent request count should be 10", 10, AzureConstants.AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT);
- assertEquals("Max block size should be 100MB", 100L * 1024L * 1024L, AzureConstants.AZURE_BLOB_MAX_BLOCK_SIZE);
- assertEquals("Max retry requests should be 4", 4, AzureConstants.AZURE_BLOB_MAX_RETRY_REQUESTS);
- assertEquals("Default timeout should be 60 seconds", 60, AzureConstants.AZURE_BLOB_DEFAULT_TIMEOUT_SECONDS);
- }
-
- @Test
- public void testMetadataKeyPrefixConstruction() {
- // Test that the metadata key prefix is correctly constructed
- String expectedPrefix = AzureConstants.AZURE_BlOB_META_DIR_NAME + "/";
- assertEquals("Metadata key prefix should be META/", expectedPrefix, AzureConstants.AZURE_BLOB_META_KEY_PREFIX);
- }
-
- @Test
- public void testConstantsAreNotNull() {
- // Ensure no constants are null
- assertNotNull("AZURE_STORAGE_ACCOUNT_NAME should not be null", AzureConstants.AZURE_STORAGE_ACCOUNT_NAME);
- assertNotNull("AZURE_STORAGE_ACCOUNT_KEY should not be null", AzureConstants.AZURE_STORAGE_ACCOUNT_KEY);
- assertNotNull("AZURE_CONNECTION_STRING should not be null", AzureConstants.AZURE_CONNECTION_STRING);
- assertNotNull("AZURE_SAS should not be null", AzureConstants.AZURE_SAS);
- assertNotNull("AZURE_TENANT_ID should not be null", AzureConstants.AZURE_TENANT_ID);
- assertNotNull("AZURE_CLIENT_ID should not be null", AzureConstants.AZURE_CLIENT_ID);
- assertNotNull("AZURE_CLIENT_SECRET should not be null", AzureConstants.AZURE_CLIENT_SECRET);
- assertNotNull("AZURE_BLOB_ENDPOINT should not be null", AzureConstants.AZURE_BLOB_ENDPOINT);
- assertNotNull("AZURE_BLOB_CONTAINER_NAME should not be null", AzureConstants.AZURE_BLOB_CONTAINER_NAME);
- assertNotNull("AZURE_BlOB_META_DIR_NAME should not be null", AzureConstants.AZURE_BlOB_META_DIR_NAME);
- assertNotNull("AZURE_BLOB_META_KEY_PREFIX should not be null", AzureConstants.AZURE_BLOB_META_KEY_PREFIX);
- assertNotNull("AZURE_BLOB_REF_KEY should not be null", AzureConstants.AZURE_BLOB_REF_KEY);
- assertNotNull("AZURE_BLOB_LAST_MODIFIED_KEY should not be null", AzureConstants.AZURE_BLOB_LAST_MODIFIED_KEY);
- }
-
- @Test
- public void testConstantsAreNotEmpty() {
- // Ensure no string constants are empty
- assertFalse("AZURE_STORAGE_ACCOUNT_NAME should not be empty", AzureConstants.AZURE_STORAGE_ACCOUNT_NAME.isEmpty());
- assertFalse("AZURE_STORAGE_ACCOUNT_KEY should not be empty", AzureConstants.AZURE_STORAGE_ACCOUNT_KEY.isEmpty());
- assertFalse("AZURE_CONNECTION_STRING should not be empty", AzureConstants.AZURE_CONNECTION_STRING.isEmpty());
- assertFalse("AZURE_SAS should not be empty", AzureConstants.AZURE_SAS.isEmpty());
- assertFalse("AZURE_TENANT_ID should not be empty", AzureConstants.AZURE_TENANT_ID.isEmpty());
- assertFalse("AZURE_CLIENT_ID should not be empty", AzureConstants.AZURE_CLIENT_ID.isEmpty());
- assertFalse("AZURE_CLIENT_SECRET should not be empty", AzureConstants.AZURE_CLIENT_SECRET.isEmpty());
- assertFalse("AZURE_BLOB_ENDPOINT should not be empty", AzureConstants.AZURE_BLOB_ENDPOINT.isEmpty());
- assertFalse("AZURE_BLOB_CONTAINER_NAME should not be empty", AzureConstants.AZURE_BLOB_CONTAINER_NAME.isEmpty());
- assertFalse("AZURE_BlOB_META_DIR_NAME should not be empty", AzureConstants.AZURE_BlOB_META_DIR_NAME.isEmpty());
- assertFalse("AZURE_BLOB_META_KEY_PREFIX should not be empty", AzureConstants.AZURE_BLOB_META_KEY_PREFIX.isEmpty());
- assertFalse("AZURE_BLOB_REF_KEY should not be empty", AzureConstants.AZURE_BLOB_REF_KEY.isEmpty());
- assertFalse("AZURE_BLOB_LAST_MODIFIED_KEY should not be empty", AzureConstants.AZURE_BLOB_LAST_MODIFIED_KEY.isEmpty());
- }
-
- @Test
- public void testSizeConstants() {
- // Test that size constants are reasonable and in correct order
- assertTrue("Min part size should be less than max part size",
- AzureConstants.AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE < AzureConstants.AZURE_BLOB_MAX_MULTIPART_UPLOAD_PART_SIZE);
- assertTrue("Max single PUT size should be less than max binary size",
- AzureConstants.AZURE_BLOB_MAX_SINGLE_PUT_UPLOAD_SIZE < AzureConstants.AZURE_BLOB_MAX_BINARY_UPLOAD_SIZE);
- assertTrue("Buffered stream threshold should be reasonable",
- AzureConstants.AZURE_BLOB_BUFFERED_STREAM_THRESHOLD > 0 && AzureConstants.AZURE_BLOB_BUFFERED_STREAM_THRESHOLD < 100L * 1024L * 1024L);
- assertTrue("Max block size should be reasonable",
- AzureConstants.AZURE_BLOB_MAX_BLOCK_SIZE > 0 && AzureConstants.AZURE_BLOB_MAX_BLOCK_SIZE <= 1024L * 1024L * 1024L);
- }
-
- @Test
- public void testConcurrencyConstants() {
- // Test concurrency-related constants
- assertTrue("Default concurrent request count should be positive", AzureConstants.AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT > 0);
- assertTrue("Max concurrent request count should be greater than or equal to default",
- AzureConstants.AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT >= AzureConstants.AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT);
- assertTrue("Max allowable upload URIs should be positive", AzureConstants.AZURE_BLOB_MAX_ALLOWABLE_UPLOAD_URIS > 0);
- assertTrue("Max unique record tries should be positive", AzureConstants.AZURE_BLOB_MAX_UNIQUE_RECORD_TRIES > 0);
- assertTrue("Max retry requests should be positive", AzureConstants.AZURE_BLOB_MAX_RETRY_REQUESTS > 0);
- assertTrue("Default timeout should be positive", AzureConstants.AZURE_BLOB_DEFAULT_TIMEOUT_SECONDS > 0);
- }
-}
diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataRecordAccessProviderCDNTest.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataRecordAccessProviderCDNTest.java
index 3f5c54cc10..1a81f9f278 100644
--- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataRecordAccessProviderCDNTest.java
+++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataRecordAccessProviderCDNTest.java
@@ -33,7 +33,7 @@ import java.net.URI;
import java.util.Properties;
import org.apache.commons.lang3.StringUtils;
-import org.apache.jackrabbit.oak.spi.blob.data.DataIdentifier;
+import org.apache.jackrabbit.core.data.DataIdentifier;
import org.apache.jackrabbit.oak.api.blob.BlobDownloadOptions;
import org.apache.jackrabbit.oak.api.blob.BlobUploadOptions;
import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.ConfigurableDataRecordAccessProvider;
diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataRecordAccessProviderIT.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataRecordAccessProviderIT.java
index 46b9379948..203d783188 100644
--- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataRecordAccessProviderIT.java
+++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataRecordAccessProviderIT.java
@@ -25,10 +25,10 @@ import java.net.URI;
import javax.net.ssl.HttpsURLConnection;
import org.apache.commons.lang3.StringUtils;
-import org.apache.jackrabbit.oak.spi.blob.data.DataIdentifier;
-import org.apache.jackrabbit.oak.spi.blob.data.DataRecord;
-import org.apache.jackrabbit.oak.spi.blob.data.DataStore;
-import org.apache.jackrabbit.oak.spi.blob.data.DataStoreException;
+import org.apache.jackrabbit.core.data.DataIdentifier;
+import org.apache.jackrabbit.core.data.DataRecord;
+import org.apache.jackrabbit.core.data.DataStore;
+import org.apache.jackrabbit.core.data.DataStoreException;
import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.AbstractDataRecordAccessProviderIT;
import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.ConfigurableDataRecordAccessProvider;
import org.junit.BeforeClass;
@@ -76,7 +76,7 @@ public class AzureDataRecordAccessProviderIT extends AbstractDataRecordAccessPro
@Override
protected long getProviderMaxPartSize() {
- return AzureConstants.AZURE_BLOB_MAX_MULTIPART_UPLOAD_PART_SIZE;
+ return AzureBlobStoreBackend.MAX_MULTIPART_UPLOAD_PART_SIZE;
}
@Override
diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataRecordAccessProviderTest.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataRecordAccessProviderTest.java
index 1e5243778b..85b0ede09f 100644
--- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataRecordAccessProviderTest.java
+++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataRecordAccessProviderTest.java
@@ -20,7 +20,6 @@ package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertNotNull;
import java.io.File;
import java.io.IOException;
@@ -33,10 +32,10 @@ import java.util.Properties;
import javax.net.ssl.HttpsURLConnection;
-import org.apache.jackrabbit.oak.spi.blob.data.DataIdentifier;
-import org.apache.jackrabbit.oak.spi.blob.data.DataRecord;
-import org.apache.jackrabbit.oak.spi.blob.data.DataStore;
-import org.apache.jackrabbit.oak.spi.blob.data.DataStoreException;
+import org.apache.jackrabbit.core.data.DataIdentifier;
+import org.apache.jackrabbit.core.data.DataRecord;
+import org.apache.jackrabbit.core.data.DataStore;
+import org.apache.jackrabbit.core.data.DataStoreException;
import org.apache.jackrabbit.oak.api.blob.BlobDownloadOptions;
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreUtils;
import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.AbstractDataRecordAccessProviderTest;
@@ -46,6 +45,7 @@ import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordU
import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadException;
import org.apache.jackrabbit.oak.spi.blob.BlobOptions;
import org.jetbrains.annotations.NotNull;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Test;
@@ -93,19 +93,19 @@ public class AzureDataRecordAccessProviderTest extends AbstractDataRecordAccessP
@Override
protected long getProviderMinPartSize() {
- return Math.max(0L, AzureConstants.AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE);
+ return Math.max(0L, AzureBlobStoreBackend.MIN_MULTIPART_UPLOAD_PART_SIZE);
}
@Override
protected long getProviderMaxPartSize() {
- return AzureConstants.AZURE_BLOB_MAX_MULTIPART_UPLOAD_PART_SIZE;
+ return AzureBlobStoreBackend.MAX_MULTIPART_UPLOAD_PART_SIZE;
}
@Override
- protected long getProviderMaxSinglePutSize() { return AzureConstants.AZURE_BLOB_MAX_SINGLE_PUT_UPLOAD_SIZE; }
+ protected long getProviderMaxSinglePutSize() { return AzureBlobStoreBackend.MAX_SINGLE_PUT_UPLOAD_SIZE; }
@Override
- protected long getProviderMaxBinaryUploadSize() { return AzureConstants.AZURE_BLOB_MAX_BINARY_UPLOAD_SIZE; }
+ protected long getProviderMaxBinaryUploadSize() { return AzureBlobStoreBackend.MAX_BINARY_UPLOAD_SIZE; }
@Override
protected boolean isSinglePutURI(URI uri) {
@@ -127,7 +127,6 @@ public class AzureDataRecordAccessProviderTest extends AbstractDataRecordAccessP
Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS);
ds.setDirectUploadURIExpirySeconds(60);
DataRecordUpload uploadContext = ds.initiateDataRecordUpload(ONE_MB, 1);
- assertNotNull("The upload context should not be null", uploadContext);
URI uploadURI = uploadContext.getUploadURIs().iterator().next();
Map<String, String> params = parseQueryString(uploadURI);
String expiryDateStr = params.get("se");
@@ -145,19 +144,16 @@ public class AzureDataRecordAccessProviderTest extends AbstractDataRecordAccessP
long uploadSize = ONE_GB * 100;
int expectedNumURIs = 10000;
DataRecordUpload upload = ds.initiateDataRecordUpload(uploadSize, -1);
- assertNotNull("The upload context should not be null", upload);
assertEquals(expectedNumURIs, upload.getUploadURIs().size());
uploadSize = ONE_GB * 500;
expectedNumURIs = 50000;
upload = ds.initiateDataRecordUpload(uploadSize, -1);
- assertNotNull("The upload context should not be null", upload);
assertEquals(expectedNumURIs, upload.getUploadURIs().size());
uploadSize = ONE_GB * 1000;
// expectedNumURIs still 50000, Azure limit
upload = ds.initiateDataRecordUpload(uploadSize, -1);
- assertNotNull("The upload context should not be null", upload);
assertEquals(expectedNumURIs, upload.getUploadURIs().size());
}
@@ -174,14 +170,14 @@ public class AzureDataRecordAccessProviderTest extends AbstractDataRecordAccessP
record = this.doSynchronousAddRecord((DataStore) dataStore, testStream);
DataIdentifier id = record.getIdentifier();
URI uri = dataStore.getDownloadURI(id, downloadOptionsWithMimeType(null));
- assertNotNull(uri);
+ Assert.assertNotNull(uri);
URI uriWithContentType = dataStore.getDownloadURI(id, downloadOptionsWithMimeType("application/octet-stream"));
- assertNotNull(uriWithContentType);
+ Assert.assertNotNull(uriWithContentType);
// must generate different download URIs
assertNotEquals(uri.toString(), uriWithContentType.toString());
} finally {
dataStore.setDirectDownloadURICacheSize(0);
- if (record != null) {
+ if (null != record) {
this.doDeleteRecord((DataStore) dataStore, record.getIdentifier());
}
}
diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreTest.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreTest.java
index 2f42b527d6..0f5e68e95b 100644
--- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreTest.java
+++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreTest.java
@@ -18,608 +18,107 @@ package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;
import static org.apache.commons.codec.binary.Hex.encodeHexString;
import static org.apache.commons.io.FileUtils.copyInputStreamToFile;
-import static org.junit.Assert.*;
-import static org.mockito.Mockito.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.junit.Assume.assumeTrue;
-import com.azure.storage.blob.BlobContainerClient;
+import org.apache.commons.lang3.StringUtils;
+import com.microsoft.azure.storage.StorageException;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.output.NullOutputStream;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.jackrabbit.oak.spi.blob.data.DataIdentifier;
-import org.apache.jackrabbit.oak.spi.blob.data.DataRecord;
-import org.apache.jackrabbit.oak.spi.blob.data.DataStoreException;
-import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v8.AzureBlobStoreBackendV8;
+import org.apache.jackrabbit.core.data.DataIdentifier;
+import org.apache.jackrabbit.core.data.DataRecord;
+import org.apache.jackrabbit.core.data.DataStoreException;
import org.apache.jackrabbit.oak.commons.collections.IteratorUtils;
-import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordDownloadOptions;
-import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadException;
-import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadOptions;
-import org.apache.jackrabbit.oak.spi.blob.AbstractSharedBackend;
import org.apache.jackrabbit.oak.spi.blob.SharedBackend;
-import org.junit.*;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
import org.junit.rules.TemporaryFolder;
-import org.junit.runner.RunWith;
-import org.mockito.Mock;
-import org.mockito.junit.MockitoJUnitRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.crypto.Mac;
-import javax.crypto.spec.SecretKeySpec;
-import javax.jcr.RepositoryException;
-import java.io.*;
-import java.net.URI;
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.StringWriter;
+import java.net.URISyntaxException;
import java.security.DigestOutputStream;
+import java.security.InvalidKeyException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Random;
+import java.util.Set;
+
+import javax.crypto.Mac;
+import javax.crypto.spec.SecretKeySpec;
+import javax.jcr.RepositoryException;
/**
- * Combined unit and integration tests for AzureDataStore class.
- * Unit tests use Mockito and don't require Azure configuration.
- * Integration tests use Azurite (Azure Storage emulator) for local testing.
+ * Test {@link AzureDataStore} with AzureDataStore and local cache on.
+ * It requires to pass azure config file via system property or system properties by prefixing with 'ds.'.
+ * See details @ {@link AzureDataStoreUtils}.
+ * For e.g. -Dconfig=/opt/cq/azure.properties. Sample azure properties located at
+ * src/test/resources/azure.properties
*/
-@RunWith(MockitoJUnitRunner.class)
public class AzureDataStoreTest {
protected static final Logger LOG = LoggerFactory.getLogger(AzureDataStoreTest.class);
- // Azurite Docker container for integration tests
- @ClassRule
- public static AzuriteDockerRule azurite = new AzuriteDockerRule();
-
- // Unit test fields
- private AzureDataStore azureDataStore;
-
- @Mock
- private DataIdentifier mockDataIdentifier;
-
- // Integration test fields
@Rule
public TemporaryFolder folder = new TemporaryFolder(new File("target"));
private Properties props;
private static byte[] testBuffer = "test".getBytes();
private AzureDataStore ds;
- private AbstractAzureBlobStoreBackend backend;
+ private AzureBlobStoreBackend backend;
private String container;
- private BlobContainerClient azuriteContainer;
Random randomGen = new Random();
- @Before
- public void setUp() {
- azureDataStore = new AzureDataStore();
- }
-
- ////
- // Unit Tests - Testing AzureDataStore logic without Azure backend
- ////
-
- @Test
- public void testDefaultConstructor() {
- AzureDataStore ds = new AzureDataStore();
- assertEquals(16 * 1024, ds.getMinRecordLength());
- assertNull(ds.getBackend()); // Backend not created until createBackend() is called
- }
-
- @Test
- public void testSetAndGetProperties() {
- Properties props = new Properties();
- props.setProperty("test.key", "test.value");
-
- azureDataStore.setProperties(props);
-
- // Verify properties are stored by testing behavior when backend is created
- assertNotNull(props);
- }
-
- @Test
- public void testSetPropertiesWithNull() {
- azureDataStore.setProperties(null);
-
- // Should not throw exception
- assertTrue("Should not throw exception", true);
- }
-
- @Test
- public void testSetAndGetMinRecordLength() {
- int newMinRecordLength = 32 * 1024;
-
- azureDataStore.setMinRecordLength(newMinRecordLength);
-
- assertEquals(newMinRecordLength, azureDataStore.getMinRecordLength());
- }
-
- @Test
- public void testMinRecordLengthBoundaryValues() {
- // Test with zero
- azureDataStore.setMinRecordLength(0);
- assertEquals(0, azureDataStore.getMinRecordLength());
-
- // Test with negative value
- azureDataStore.setMinRecordLength(-1);
- assertEquals(-1, azureDataStore.getMinRecordLength());
-
- // Test with large value
- azureDataStore.setMinRecordLength(Integer.MAX_VALUE);
- assertEquals(Integer.MAX_VALUE, azureDataStore.getMinRecordLength());
- }
-
- @Test
- public void testCreateBackendReturnsNonNull() {
- AbstractSharedBackend result = azureDataStore.createBackend();
- assertNotNull(result);
- assertTrue(result instanceof AbstractAzureBlobStoreBackend);
- }
-
- @Test
- public void testCreateBackendWithProperties() {
- Properties props = new Properties();
- props.setProperty("test.key", "test.value");
- azureDataStore.setProperties(props);
-
- AbstractSharedBackend result = azureDataStore.createBackend();
- assertNotNull(result);
- assertTrue(result instanceof AbstractAzureBlobStoreBackend);
- }
-
- @Test
- public void testGetBackendBeforeInit() {
- // Initially null before init
- assertNull(azureDataStore.getBackend());
- }
-
- @Test
- public void testSetBinaryTransferAccelerationEnabled() {
- // This method is a NOOP for Azure, so just verify it doesn't throw
- azureDataStore.setBinaryTransferAccelerationEnabled(true);
- azureDataStore.setBinaryTransferAccelerationEnabled(false);
- // No exception should be thrown
- assertTrue("Should not throw exception", true);
- }
-
- @Test
- public void testSetDirectUploadURIExpirySecondsWithoutBackend() {
- // Should not throw exception when backend is null
- azureDataStore.setDirectUploadURIExpirySeconds(3600);
- azureDataStore.setDirectUploadURIExpirySeconds(0);
- azureDataStore.setDirectUploadURIExpirySeconds(-1);
- // No exception should be thrown
- assertTrue("Should not throw exception", true);
- }
-
- @Test
- public void testSetDirectDownloadURIExpirySecondsWithoutBackend() {
- // Should not throw exception when backend is null
- azureDataStore.setDirectDownloadURIExpirySeconds(7200);
- azureDataStore.setDirectDownloadURIExpirySeconds(0);
- azureDataStore.setDirectDownloadURIExpirySeconds(-1);
-
- // No exception should be thrown
- assertTrue("Should not throw exception", true);
- }
-
- @Test(expected = DataRecordUploadException.class)
- public void testInitiateDataRecordUploadTwoParamsWithoutBackendThrowsException() throws DataRecordUploadException {
- azureDataStore.initiateDataRecordUpload(1000L, 5);
- }
-
- @Test(expected = DataRecordUploadException.class)
- public void testInitiateDataRecordUploadThreeParamsWithoutBackendThrowsException() throws DataRecordUploadException {
- azureDataStore.initiateDataRecordUpload(1000L, 5, DataRecordUploadOptions.DEFAULT);
- }
-
- @Test(expected = DataRecordUploadException.class)
- public void testCompleteDataRecordUploadWithoutBackendThrowsException()
- throws DataRecordUploadException, DataStoreException {
- azureDataStore.completeDataRecordUpload("test-token");
- }
-
- @Test
- public void testGetDownloadURIWithoutBackend() {
- URI result = azureDataStore.getDownloadURI(mockDataIdentifier, DataRecordDownloadOptions.DEFAULT);
- assertNull(result);
- }
-
- @Test
- public void testSetDirectDownloadURICacheSizeWithoutBackend() {
- // This should call the method on null backend, which will cause NPE
- // But looking at the implementation, it doesn't check for null like the other methods
- try {
- azureDataStore.setDirectDownloadURICacheSize(100);
- fail("Expected NullPointerException");
- } catch (NullPointerException e) {
- // Expected behavior since the method doesn't check for null backend
- }
- }
-
- @Test
- public void testInitiateDataRecordUploadTwoParamsCallsThreeParamsVersion() throws DataRecordUploadException {
- // Create a spy to verify the method delegation
- AzureDataStore spyDataStore = spy(azureDataStore);
-
- // Mock the three-parameter version to avoid backend initialization
- doThrow(new DataRecordUploadException("Backend not initialized"))
- .when(spyDataStore).initiateDataRecordUpload(anyLong(), anyInt(), any(DataRecordUploadOptions.class));
-
- try {
- spyDataStore.initiateDataRecordUpload(1000L, 5);
- fail("Expected DataRecordUploadException");
- } catch (DataRecordUploadException e) {
- // Verify that the three-parameter version was called with DEFAULT options
- verify(spyDataStore).initiateDataRecordUpload(1000L, 5, DataRecordUploadOptions.DEFAULT);
- }
- }
-
- @Test
- public void testExceptionMessagesAreCorrect() {
- try {
- azureDataStore.initiateDataRecordUpload(1000L, 5);
- fail("Expected DataRecordUploadException");
- } catch (DataRecordUploadException e) {
- assertEquals("Backend not initialized", e.getMessage());
- }
-
- try {
- azureDataStore.completeDataRecordUpload("test-token");
- fail("Expected DataRecordUploadException");
- } catch (DataRecordUploadException | DataStoreException e) {
- assertEquals("Backend not initialized", e.getMessage());
- }
- }
-
- @Test
- public void testSetDirectUploadURIExpirySecondsWithBackend() {
- // Create backend first and set it to the azureBlobStoreBackend field
- azureDataStore.createBackend();
-
- // These should not throw exceptions
- azureDataStore.setDirectUploadURIExpirySeconds(3600);
- azureDataStore.setDirectUploadURIExpirySeconds(0);
- azureDataStore.setDirectUploadURIExpirySeconds(-1);
-
- // Should not throw exceptions
- assertTrue("Should not throw exception", true);
- }
-
- @Test
- public void testSetDirectDownloadURIExpirySecondsWithBackend() {
- // Create backend first and set it to the azureBlobStoreBackend field
- azureDataStore.createBackend();
-
- // These should not throw exceptions
- azureDataStore.setDirectDownloadURIExpirySeconds(7200);
- azureDataStore.setDirectDownloadURIExpirySeconds(0);
- azureDataStore.setDirectDownloadURIExpirySeconds(-1);
-
- // Should not throw exceptions
- assertTrue("Should not throw exception", true);
- }
-
- @Test
- public void testSetDirectDownloadURICacheSizeWithBackend() {
- // Create backend first and set it to the azureBlobStoreBackend field
- azureDataStore.createBackend();
-
- // These should not throw exceptions
- azureDataStore.setDirectDownloadURICacheSize(100);
- azureDataStore.setDirectDownloadURICacheSize(0);
- azureDataStore.setDirectDownloadURICacheSize(-1);
-
- // Should not throw exceptions
- assertTrue("Should not throw exception", true);
- }
-
- @Test(expected = NullPointerException.class)
- public void testGetDownloadURIWithBackendButNullIdentifier() {
- // Create backend first and initialize it
- azureDataStore.createBackend();
-
- // This should throw NPE for null identifier
- azureDataStore.getDownloadURI(null, DataRecordDownloadOptions.DEFAULT);
-
- fail("Expected NullPointerException for null identifier");
- }
-
- @Test(expected = NullPointerException.class)
- public void testGetDownloadURIWithBackendButNullOptions() {
- // Create backend first and initialize it
- azureDataStore.createBackend();
-
- // This should throw NPE for null options
- azureDataStore.getDownloadURI(mockDataIdentifier, null);
- }
-
- @Test
- public void testCreateBackendMultipleTimes() {
- // Creating backend multiple times should work
- AbstractSharedBackend backend1 = azureDataStore.createBackend();
- AbstractSharedBackend backend2 = azureDataStore.createBackend();
-
- assertNotNull(backend1);
- assertNotNull(backend2);
- // They should be different instances
- assertNotSame(backend1, backend2);
- }
-
- @Test
- public void testPropertiesArePassedToBackend() {
- Properties props = new Properties();
- props.setProperty("azure.accountName", "testaccount");
- props.setProperty("azure.accountKey", "testkey");
-
- azureDataStore.setProperties(props);
- AbstractSharedBackend backend = azureDataStore.createBackend();
-
- assertNotNull(backend);
- // The backend should have been created and properties should have been set
- // We can't directly verify this without accessing private fields, but we can
- // verify that no exception was thrown during creation
- }
-
- @Test
- public void testNullPropertiesDoNotCauseException() {
- azureDataStore.setProperties(null);
- AbstractSharedBackend backend = azureDataStore.createBackend();
-
- assertNotNull(backend);
- // Should not throw exception even with null properties
- }
-
- @Test
- public void testEmptyPropertiesDoNotCauseException() {
- azureDataStore.setProperties(new Properties());
- AbstractSharedBackend backend = azureDataStore.createBackend();
-
- assertNotNull(backend);
- // Should not throw exception even with empty properties
- }
-
- @Test
- public void testCreateBackendWithDifferentSDKVersions() {
- // Test that createBackend works regardless of SDK version
- // The actual SDK version is determined by system property, but we can test that
- // the method doesn't fail
- AbstractSharedBackend backend1 = azureDataStore.createBackend();
- assertNotNull(backend1);
-
- // Create another instance to test consistency
- AzureDataStore anotherDataStore = new AzureDataStore();
- AbstractSharedBackend backend2 = anotherDataStore.createBackend();
- assertNotNull(backend2);
-
- // Both should be the same type (determined by system property)
- assertEquals(backend1.getClass(), backend2.getClass());
- }
-
- @Test
- public void testBackendInstantiationWithAzureSdk12Enabled() {
- String originalProperty = System.getProperty("blob.azure.v12.enabled");
- try {
- // Set system property to enable Azure SDK 12
- System.setProperty("blob.azure.v12.enabled", "true");
-
- // Create a new AzureDataStore instance
- AzureDataStore dataStore = new AzureDataStore();
-
- // Call createBackend which should instantiate AzureBlobStoreBackend
- AbstractSharedBackend backend = dataStore.createBackend();
-
- // Verify that the backend is an instance of AzureBlobStoreBackend (SDK 12)
- assertNotNull("Backend should not be null", backend);
- assertTrue("Backend should be an instance of AzureBlobStoreBackend when SDK 12 is enabled",
- backend instanceof AzureBlobStoreBackend);
- assertFalse("Backend should not be an instance of AzureBlobStoreBackendV8 when SDK 12 is enabled",
- backend instanceof AzureBlobStoreBackendV8);
- } finally {
- // Restore original system property
- if (originalProperty != null) {
- System.setProperty("blob.azure.v12.enabled", originalProperty);
- } else {
- System.clearProperty("blob.azure.v12.enabled");
- }
- }
- }
-
- @Test
- public void testBackendInstantiationWithAzureSdk12Disabled() {
- String originalProperty = System.getProperty("blob.azure.v12.enabled");
- try {
- // Set system property to disable Azure SDK 12
- System.setProperty("blob.azure.v12.enabled", "false");
-
- // Create a new AzureDataStore instance
- AzureDataStore dataStore = new AzureDataStore();
-
- // Call createBackend which should instantiate AzureBlobStoreBackendV8
- AbstractSharedBackend backend = dataStore.createBackend();
-
- // Verify that the backend is an instance of AzureBlobStoreBackendV8 (SDK 8)
- assertNotNull("Backend should not be null", backend);
- assertTrue("Backend should be an instance of AzureBlobStoreBackendV8 when SDK 12 is disabled",
- backend instanceof AzureBlobStoreBackendV8);
- } finally {
- // Restore original system property
- if (originalProperty != null) {
- System.setProperty("blob.azure.v12.enabled", originalProperty);
- } else {
- System.clearProperty("blob.azure.v12.enabled");
- }
- }
+ @BeforeClass
+ public static void assumptions() {
+ assumeTrue(AzureDataStoreUtils.isAzureConfigured());
}
- @Test
- public void testBackendInstantiationWithAzureSdk12NotSet() {
- String originalProperty = System.getProperty("blob.azure.v12.enabled");
- try {
- // Clear system property to test default behavior
- System.clearProperty("blob.azure.v12.enabled");
-
- // Create a new AzureDataStore instance
- AzureDataStore dataStore = new AzureDataStore();
-
- // Call createBackend which should instantiate AzureBlobStoreBackendV8 (default)
- AbstractSharedBackend backend = dataStore.createBackend();
-
- // Verify that the backend is an instance of AzureBlobStoreBackendV8 (SDK 8) by default
- assertNotNull("Backend should not be null", backend);
- assertTrue("Backend should be an instance of AzureBlobStoreBackendV8 when SDK 12 property is not set",
- backend instanceof AzureBlobStoreBackendV8);
- } finally {
- // Restore original system property
- if (originalProperty != null) {
- System.setProperty("blob.azure.v12.enabled", originalProperty);
- } else {
- System.clearProperty("blob.azure.v12.enabled");
- }
- }
- }
-
- @Test
- public void testConfigurableDataRecordAccessProviderMethods() {
- // Test all ConfigurableDataRecordAccessProvider methods without backend
- azureDataStore.setDirectUploadURIExpirySeconds(1800);
- azureDataStore.setDirectDownloadURIExpirySeconds(3600);
- azureDataStore.setBinaryTransferAccelerationEnabled(true);
- azureDataStore.setBinaryTransferAccelerationEnabled(false);
-
- // These should not throw exceptions even without backend
- assertTrue("Should not throw exception", true);
- }
-
- @Test
- public void testGetDownloadURIWithNullBackend() {
- // Ensure getDownloadURI returns null when backend is not initialized
- URI result = azureDataStore.getDownloadURI(mockDataIdentifier, DataRecordDownloadOptions.DEFAULT);
- assertNull(result);
- }
-
- @Test
- public void testMethodCallsWithVariousParameterValues() {
- // Test boundary values for various methods
- azureDataStore.setMinRecordLength(0);
- assertEquals(0, azureDataStore.getMinRecordLength());
-
- azureDataStore.setMinRecordLength(1);
- assertEquals(1, azureDataStore.getMinRecordLength());
-
- azureDataStore.setMinRecordLength(1024 * 1024); // 1MB
- assertEquals(1024 * 1024, azureDataStore.getMinRecordLength());
-
- // Test with negative values
- azureDataStore.setDirectUploadURIExpirySeconds(-100);
- azureDataStore.setDirectDownloadURIExpirySeconds(-200);
-
- // Should not throw exceptions
- }
-
- @Test
- public void testDataRecordUploadExceptionMessages() {
- // Test that exception messages are consistent
- try {
- azureDataStore.initiateDataRecordUpload(1000L, 5);
- fail("Expected DataRecordUploadException");
- } catch (DataRecordUploadException e) {
- assertEquals("Backend not initialized", e.getMessage());
- }
-
- try {
- azureDataStore.initiateDataRecordUpload(1000L, 5, DataRecordUploadOptions.DEFAULT);
- fail("Expected DataRecordUploadException");
- } catch (DataRecordUploadException e) {
- assertEquals("Backend not initialized", e.getMessage());
- }
-
- try {
- azureDataStore.completeDataRecordUpload("test-token");
- fail("Expected DataRecordUploadException");
- } catch (DataRecordUploadException | DataStoreException e) {
- assertEquals("Backend not initialized", e.getMessage());
- }
- }
-
- @Test
- public void testCreateBackendSetsAzureBlobStoreBackendField() {
- // Verify that createBackend() properly sets the azureBlobStoreBackend field
- // by testing that subsequent calls to methods that depend on it work
- azureDataStore.createBackend();
-
- // These methods should not throw exceptions after createBackend() is called
- azureDataStore.setDirectUploadURIExpirySeconds(3600);
- azureDataStore.setDirectDownloadURIExpirySeconds(7200);
- azureDataStore.setDirectDownloadURICacheSize(100);
-
- // No exceptions should be thrown
- assertTrue("Should not throw exception", true);
- }
-
- ////
- // Integration Tests - Testing with Azurite (Azure Storage emulator)
- // These tests use Azurite Docker container for local testing without requiring real Azure credentials
- ////
+ @Before
+ public void setup() throws IOException, RepositoryException, URISyntaxException, InvalidKeyException, StorageException {
- private void setupIntegrationTest() throws IOException, RepositoryException {
- // Generate unique container name for this test run
+ props = AzureDataStoreUtils.getAzureConfig();
container = String.valueOf(randomGen.nextInt(9999)) + "-" + String.valueOf(randomGen.nextInt(9999))
+ "-test";
-
- // Create Azurite container
- azuriteContainer = azurite.getContainer(container, getConnectionString());
-
- // Setup properties for Azurite
- props = createAzuriteProperties();
+ props.setProperty(AzureConstants.AZURE_BLOB_CONTAINER_NAME, container);
ds = new AzureDataStore();
ds.setProperties(props);
ds.setCacheSize(0); // Turn caching off so we don't get weird test results due to caching
ds.init(folder.newFolder().getAbsolutePath());
- backend = (AbstractAzureBlobStoreBackend) ds.getBackend();
+ backend = (AzureBlobStoreBackend) ds.getBackend();
}
- private void teardownIntegrationTest() {
- if (ds != null) {
- try {
- ds.close();
- } catch (Exception ignore) {
- LOG.warn("Error closing data store", ignore);
- }
- ds = null;
- }
-
- if (azuriteContainer != null) {
- try {
- azuriteContainer.deleteIfExists();
- } catch (Exception ignore) {
- LOG.warn("Error deleting Azurite container: {}", container, ignore);
- }
- azuriteContainer = null;
- }
-
- backend = null;
- }
-
- /**
- * Creates properties configured for Azurite local testing.
- */
- private Properties createAzuriteProperties() {
- Properties properties = new Properties();
- properties.setProperty(AzureConstants.AZURE_BLOB_CONTAINER_NAME, container);
- properties.setProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, AzuriteDockerRule.ACCOUNT_NAME);
- properties.setProperty(AzureConstants.AZURE_BLOB_ENDPOINT, azurite.getBlobEndpoint());
- properties.setProperty(AzureConstants.AZURE_CONNECTION_STRING, getConnectionString());
- properties.setProperty(AzureConstants.AZURE_CREATE_CONTAINER, "true");
- properties.setProperty(AzureConstants.AZURE_REF_ON_INIT, "true"); // Enable reference key creation
- return properties;
- }
-
- /**
- * Gets the Azurite connection string.
- */
- private static String getConnectionString() {
- return Utils.getConnectionString(
- AzuriteDockerRule.ACCOUNT_NAME,
- AzuriteDockerRule.ACCOUNT_KEY,
- azurite.getBlobEndpoint()
- );
+ @After
+ public void teardown() throws InvalidKeyException, URISyntaxException, StorageException {
+ ds = null;
+ try {
+ AzureDataStoreUtils.deleteContainer(container);
+ } catch (Exception ignore) {}
}
private void validateRecord(final DataRecord record,
@@ -651,10 +150,10 @@ public class AzureDataStoreTest {
} else {
assertTrue(record.getLastModified() > lastModified);
}
- assertEquals(record.getIdentifier().toString(), identifier.toString());
+ assertTrue(record.getIdentifier().toString().equals(identifier.toString()));
StringWriter writer = new StringWriter();
org.apache.commons.io.IOUtils.copy(record.getStream(), writer, "utf-8");
- assertEquals(writer.toString(), contents);
+ assertTrue(writer.toString().equals(contents));
}
private static InputStream randomStream(int seed, int size) {
@@ -667,7 +166,7 @@ public class AzureDataStoreTest {
private static String getIdForInputStream(final InputStream in)
throws NoSuchAlgorithmException, IOException {
MessageDigest digest = MessageDigest.getInstance("SHA-1");
- OutputStream output = new DigestOutputStream(NullOutputStream.INSTANCE, digest);
+ OutputStream output = new DigestOutputStream(new NullOutputStream(), digest);
try {
IOUtils.copyLarge(in, output);
} finally {
@@ -677,710 +176,572 @@ public class AzureDataStoreTest {
return encodeHexString(digest.digest());
}
- private void validateRecordData(final SharedBackend backend,
- final DataIdentifier identifier,
- int expectedSize,
- final InputStream expected) throws IOException, DataStoreException {
- byte[] blobData = new byte[expectedSize];
- backend.read(identifier).read(blobData);
- byte[] expectedData = new byte[expectedSize];
- expected.read(expectedData);
- for (int i=0; i<expectedSize; i++) {
- assertEquals(expectedData[i], blobData[i]);
- }
- }
@Test
- public void testCreateAndDeleteBlobHappyPath() throws Exception {
- setupIntegrationTest();
- try {
- final DataRecord uploadedRecord = ds.addRecord(new ByteArrayInputStream(testBuffer));
- DataIdentifier identifier = uploadedRecord.getIdentifier();
- assertTrue(backend.exists(identifier));
- assertTrue(0 != uploadedRecord.getLastModified());
- assertEquals(testBuffer.length, uploadedRecord.getLength());
+ public void testCreateAndDeleteBlobHappyPath() throws DataStoreException, IOException {
+ final DataRecord uploadedRecord = ds.addRecord(new ByteArrayInputStream(testBuffer));
+ DataIdentifier identifier = uploadedRecord.getIdentifier();
+ assertTrue(backend.exists(identifier));
+ assertTrue(0 != uploadedRecord.getLastModified());
+ assertEquals(testBuffer.length, uploadedRecord.getLength());
- final DataRecord retrievedRecord = ds.getRecord(identifier);
- validateRecord(retrievedRecord, new String(testBuffer), uploadedRecord);
+ final DataRecord retrievedRecord = ds.getRecord(identifier);
+ validateRecord(retrievedRecord, new String(testBuffer), uploadedRecord);
- ds.deleteRecord(identifier);
- assertFalse(backend.exists(uploadedRecord.getIdentifier()));
- } finally {
- teardownIntegrationTest();
- }
+ ds.deleteRecord(identifier);
+ assertFalse(backend.exists(uploadedRecord.getIdentifier()));
}
+
@Test
- public void testCreateAndReUploadBlob() throws Exception {
- setupIntegrationTest();
- try {
- final DataRecord createdRecord = ds.addRecord(new ByteArrayInputStream(testBuffer));
- DataIdentifier identifier1 = createdRecord.getIdentifier();
- assertTrue(backend.exists(identifier1));
+ public void testCreateAndReUploadBlob() throws DataStoreException, IOException {
+ final DataRecord createdRecord = ds.addRecord(new ByteArrayInputStream(testBuffer));
+ DataIdentifier identifier1 = createdRecord.getIdentifier();
+ assertTrue(backend.exists(identifier1));
- final DataRecord record1 = ds.getRecord(identifier1);
- validateRecord(record1, new String(testBuffer), createdRecord);
+ final DataRecord record1 = ds.getRecord(identifier1);
+ validateRecord(record1, new String(testBuffer), createdRecord);
- try { Thread.sleep(1001); } catch (InterruptedException e) { }
+ try { Thread.sleep(1001); } catch (InterruptedException e) { }
- final DataRecord updatedRecord = ds.addRecord(new ByteArrayInputStream(testBuffer));
- DataIdentifier identifier2 = updatedRecord.getIdentifier();
- assertTrue(backend.exists(identifier2));
+ final DataRecord updatedRecord = ds.addRecord(new ByteArrayInputStream(testBuffer));
+ DataIdentifier identifier2 = updatedRecord.getIdentifier();
+ assertTrue(backend.exists(identifier2));
- assertTrue(identifier1.toString().equals(identifier2.toString()));
- validateRecord(record1, new String(testBuffer), createdRecord);
+ assertTrue(identifier1.toString().equals(identifier2.toString()));
+ validateRecord(record1, new String(testBuffer), createdRecord);
- ds.deleteRecord(identifier1);
- assertFalse(backend.exists(createdRecord.getIdentifier()));
- } finally {
- teardownIntegrationTest();
- }
+ ds.deleteRecord(identifier1);
+ assertFalse(backend.exists(createdRecord.getIdentifier()));
}
@Test
- public void testListBlobs() throws Exception {
- setupIntegrationTest();
- try {
- final Set<DataIdentifier> identifiers = new HashSet<>();
- final Set<String> testStrings = Set.of("test1", "test2", "test3");
+ public void testListBlobs() throws DataStoreException, IOException {
+ final Set<DataIdentifier> identifiers = new HashSet<>();
+ final Set<String> testStrings = Set.of("test1", "test2", "test3");
- for (String s : testStrings) {
- identifiers.add(ds.addRecord(new ByteArrayInputStream(s.getBytes())).getIdentifier());
- }
+ for (String s : testStrings) {
+ identifiers.add(ds.addRecord(new ByteArrayInputStream(s.getBytes())).getIdentifier());
+ }
- Iterator<DataIdentifier> iter = ds.getAllIdentifiers();
- while (iter.hasNext()) {
- DataIdentifier identifier = iter.next();
- assertTrue(identifiers.contains(identifier));
- ds.deleteRecord(identifier);
- }
- } finally {
- teardownIntegrationTest();
+ Iterator<DataIdentifier> iter = ds.getAllIdentifiers();
+ while (iter.hasNext()) {
+ DataIdentifier identifier = iter.next();
+ assertTrue(identifiers.contains(identifier));
+ ds.deleteRecord(identifier);
}
}
+ ////
// Backend Tests
+ ////
- @Test
- public void testBackendWriteDifferentSizedRecords() throws Exception {
- setupIntegrationTest();
- try {
- // Sizes are chosen as follows:
- // 0 - explicitly test zero-size file
- // 10 - very small file
- // 1000 - under 4K (a reasonably expected stream buffer size)
- // 4100 - over 4K but under 8K and 16K (other reasonably expected stream buffer sizes)
- // 16500 - over 8K and 16K but under 64K (another reasonably expected stream buffer size)
- // 66000 - over 64K but under 128K (probably the largest reasonably expected stream buffer size)
- // 132000 - over 128K
- for (int size : List.of(0, 10, 1000, 4100, 16500, 66000, 132000)) {
- File testFile = folder.newFile();
- copyInputStreamToFile(randomStream(size, size), testFile);
- DataIdentifier identifier = new DataIdentifier(getIdForInputStream(new FileInputStream(testFile)));
- backend.write(identifier, testFile);
- assertTrue(backend.exists(identifier));
-
- validateRecordData(backend, identifier, size, new FileInputStream(testFile));
-
- backend.deleteRecord(identifier);
- assertFalse(backend.exists(identifier));
- }
- } finally {
- teardownIntegrationTest();
+ private void validateRecordData(final SharedBackend backend,
+ final DataIdentifier identifier,
+ int expectedSize,
+ final InputStream expected) throws IOException, DataStoreException {
+ byte[] blobData = new byte[expectedSize];
+ backend.read(identifier).read(blobData);
+ byte[] expectedData = new byte[expectedSize];
+ expected.read(expectedData);
+ for (int i=0; i<expectedSize; i++) {
+ assertEquals(expectedData[i], blobData[i]);
}
}
+ // Write (Backend)
+
@Test
- public void testBackendWriteRecordNullIdentifierThrowsNullPointerException() throws Exception {
- setupIntegrationTest();
- try {
- DataIdentifier identifier = null;
+ public void testBackendWriteDifferentSizedRecords() throws IOException, NoSuchAlgorithmException, DataStoreException {
+ // Sizes are chosen as follows:
+ // 0 - explicitly test zero-size file
+ // 10 - very small file
+ // 1000 - under 4K (a reasonably expected stream buffer size)
+ // 4100 - over 4K but under 8K and 16K (other reasonably expected stream buffer sizes)
+ // 16500 - over 8K and 16K but under 64K (another reasonably expected stream buffer size)
+ // 66000 - over 64K but under 128K (probably the largest reasonably expected stream buffer size)
+ // 132000 - over 128K
+ for (int size : List.of(0, 10, 1000, 4100, 16500, 66000, 132000)) {
File testFile = folder.newFile();
- copyInputStreamToFile(randomStream(0, 10), testFile);
- try {
- backend.write(identifier, testFile);
- fail();
- } catch (NullPointerException e) {
- assertEquals("identifier must not be null", e.getMessage());
- }
- } finally {
- teardownIntegrationTest();
+ copyInputStreamToFile(randomStream(size, size), testFile);
+ DataIdentifier identifier = new DataIdentifier(getIdForInputStream(new FileInputStream(testFile)));
+ backend.write(identifier, testFile);
+ assertTrue(backend.exists(identifier));
+
+ validateRecordData(backend, identifier, size, new FileInputStream(testFile));
+
+ backend.deleteRecord(identifier);
+ assertFalse(backend.exists(identifier));
}
}
@Test
- public void testBackendWriteRecordNullFileThrowsNullPointerException() throws Exception {
- setupIntegrationTest();
+ public void testBackendWriteRecordNullIdentifierThrowsNullPointerException() throws IOException, DataStoreException{
+ DataIdentifier identifier = null;
+ File testFile = folder.newFile();
+ copyInputStreamToFile(randomStream(0, 10), testFile);
try {
- File testFile = null;
- DataIdentifier identifier = new DataIdentifier("fake");
- try {
- backend.write(identifier, testFile);
- fail();
- }
- catch (NullPointerException e) {
- assertTrue("file must not be null".equals(e.getMessage()));
- }
- } finally {
- teardownIntegrationTest();
+ backend.write(identifier, testFile);
+ fail();
+ } catch (NullPointerException e) {
+ assertEquals("identifier", e.getMessage());
}
}
@Test
- public void testBackendWriteRecordFileNotFoundThrowsException() throws Exception {
- setupIntegrationTest();
+ public void testBackendWriteRecordNullFileThrowsNullPointerException() throws DataStoreException {
+ File testFile = null;
+ DataIdentifier identifier = new DataIdentifier("fake");
try {
- File testFile = folder.newFile();
- copyInputStreamToFile(randomStream(0, 10), testFile);
- DataIdentifier identifier = new DataIdentifier(getIdForInputStream(new FileInputStream(testFile)));
- assertTrue(testFile.delete());
- try {
- backend.write(identifier, testFile);
- fail();
- } catch (DataStoreException e) {
- assertTrue(e.getCause() instanceof FileNotFoundException);
- }
- } finally {
- teardownIntegrationTest();
+ backend.write(identifier, testFile);
+ fail();
+ }
+ catch (NullPointerException e) {
+ assertTrue("file".equals(e.getMessage()));
}
}
@Test
- public void testBackendReadRecordNullIdentifier() throws Exception {
- setupIntegrationTest();
+ public void testBackendWriteRecordFileNotFoundThrowsException() throws IOException, NoSuchAlgorithmException {
+ File testFile = folder.newFile();
+ copyInputStreamToFile(randomStream(0, 10), testFile);
+ DataIdentifier identifier = new DataIdentifier(getIdForInputStream(new FileInputStream(testFile)));
+ assertTrue(testFile.delete());
try {
- DataIdentifier identifier = null;
- try {
- backend.read(identifier);
- fail();
- }
- catch (NullPointerException e) {
- assert("identifier must not be null".equals(e.getMessage()));
- }
- } finally {
- teardownIntegrationTest();
+ backend.write(identifier, testFile);
+ fail();
+ } catch (DataStoreException e) {
+ assertTrue(e.getCause() instanceof FileNotFoundException);
}
}
- @Test(expected = DataStoreException.class)
- public void testBackendReadRecordInvalidIdentifier() throws Exception {
- setupIntegrationTest();
+ // Read (Backend)
+
+ @Test
+ public void testBackendReadRecordNullIdentifier() throws DataStoreException {
+ DataIdentifier identifier = null;
try {
- DataIdentifier identifier = new DataIdentifier("fake");
- try (var is = backend.read(identifier);){
- fail("Expected DataStoreException for invalid identifier");
- }
- catch (DataStoreException e) {
- assertTrue("Should contain missing blob error",
- e.getMessage().contains("Trying to read missing blob"));
- throw e;
- }
- } finally {
- teardownIntegrationTest();
+ backend.read(identifier);
+ fail();
+ }
+ catch (NullPointerException e) {
+ assert("identifier".equals(e.getMessage()));
}
}
@Test
- public void testBackendDeleteRecordNullIdentifier() throws Exception {
- setupIntegrationTest();
+ public void testBackendReadRecordInvalidIdentifier() {
+ DataIdentifier identifier = new DataIdentifier("fake");
try {
- DataIdentifier identifier = null;
- try {
- backend.deleteRecord(identifier);
- fail();
- }
- catch (NullPointerException e) {
- assert("identifier must not be null".equals(e.getMessage()));
- }
- } finally {
- teardownIntegrationTest();
+ backend.read(identifier);
+ fail();
}
+ catch (DataStoreException e) { }
}
+ // Delete (Backend)
+
@Test
- public void testBackendDeleteRecordInvalidIdentifier() throws Exception {
- setupIntegrationTest();
+ public void testBackendDeleteRecordNullIdentifier() throws DataStoreException {
+ DataIdentifier identifier = null;
try {
- DataIdentifier identifier = new DataIdentifier("fake");
- backend.deleteRecord(identifier); // We don't care if the identifier is invalid; this is a noop
- } finally {
- teardownIntegrationTest();
+ backend.deleteRecord(identifier);
+ fail();
+ }
+ catch (NullPointerException e) {
+ assert("identifier".equals(e.getMessage()));
}
}
@Test
- public void testBackendNotCreatedRecordDoesNotExist() throws Exception {
- setupIntegrationTest();
- try {
- assertFalse(backend.exists(new DataIdentifier(("fake"))));
- } finally {
- teardownIntegrationTest();
- }
+ public void testBackendDeleteRecordInvalidIdentifier() throws DataStoreException {
+ DataIdentifier identifier = new DataIdentifier("fake");
+ backend.deleteRecord(identifier); // We don't care if the identifier is invalid; this is a noop
}
+ // Exists (Backend)
+
@Test
- public void testBackendRecordExistsNullIdentifierThrowsNullPointerException() throws Exception {
- setupIntegrationTest();
- try {
- try {
- DataIdentifier nullIdentifier = null;
- backend.exists(nullIdentifier);
- fail();
- }
- catch (NullPointerException e) { }
- } finally {
- teardownIntegrationTest();
- }
+ public void testBackendNotCreatedRecordDoesNotExist() throws DataStoreException {
+ assertFalse(backend.exists(new DataIdentifier(("fake"))));
}
@Test
- public void testBackendGetAllIdentifiersNoRecordsReturnsNone() throws Exception {
- setupIntegrationTest();
+ public void testBackendRecordExistsNullIdentifierThrowsNullPointerException() throws DataStoreException {
try {
- Iterator<DataIdentifier> allIdentifiers = backend.getAllIdentifiers();
- assertFalse(allIdentifiers.hasNext());
- } finally {
- teardownIntegrationTest();
+ DataIdentifier nullIdentifier = null;
+ backend.exists(nullIdentifier);
+ fail();
}
+ catch (NullPointerException e) { }
}
+ // GetAllIdentifiers (Backend)
+
@Test
- public void testBackendGetAllIdentifiers() throws Exception {
- setupIntegrationTest();
- try {
- for (int expectedRecCount : List.of(1, 2, 5)) {
- final List<DataIdentifier> ids = new ArrayList<>();
- for (int i=0; i<expectedRecCount; i++) {
- File testfile = folder.newFile();
- copyInputStreamToFile(randomStream(i, 10), testfile);
- DataIdentifier identifier = new DataIdentifier(getIdForInputStream(new FileInputStream(testfile)));
- backend.write(identifier, testfile);
- ids.add(identifier);
- }
+ public void testBackendGetAllIdentifiersNoRecordsReturnsNone() throws DataStoreException {
+ Iterator<DataIdentifier> allIdentifiers = backend.getAllIdentifiers();
+ assertFalse(allIdentifiers.hasNext());
+ }
- int actualRecCount = IteratorUtils.size(backend.getAllIdentifiers());
+ @Test
+ public void testBackendGetAllIdentifiers() throws DataStoreException, IOException, NoSuchAlgorithmException {
+ for (int expectedRecCount : List.of(1, 2, 5)) {
+ final List<DataIdentifier> ids = new ArrayList<>();
+ for (int i=0; i<expectedRecCount; i++) {
+ File testfile = folder.newFile();
+ copyInputStreamToFile(randomStream(i, 10), testfile);
+ DataIdentifier identifier = new DataIdentifier(getIdForInputStream(new FileInputStream(testfile)));
+ backend.write(identifier, testfile);
+ ids.add(identifier);
+ }
- for (DataIdentifier identifier : ids) {
- backend.deleteRecord(identifier);
- }
+ int actualRecCount = IteratorUtils.size(backend.getAllIdentifiers());
- assertEquals(expectedRecCount, actualRecCount);
+ for (DataIdentifier identifier : ids) {
+ backend.deleteRecord(identifier);
}
- } finally {
- teardownIntegrationTest();
+
+ assertEquals(expectedRecCount, actualRecCount);
}
}
+ // GetRecord (Backend)
+
@Test
- public void testBackendGetRecord() throws Exception {
- setupIntegrationTest();
- try {
- String recordData = "testData";
- DataRecord record = ds.addRecord(new ByteArrayInputStream(recordData.getBytes()));
- DataRecord retrievedRecord = backend.getRecord(record.getIdentifier());
- validateRecord(record, recordData, retrievedRecord);
- } finally {
- teardownIntegrationTest();
- }
+ public void testBackendGetRecord() throws IOException, DataStoreException {
+ String recordData = "testData";
+ DataRecord record = ds.addRecord(new ByteArrayInputStream(recordData.getBytes()));
+ DataRecord retrievedRecord = backend.getRecord(record.getIdentifier());
+ validateRecord(record, recordData, retrievedRecord);
}
@Test
- public void testBackendGetRecordNullIdentifierThrowsNullPointerException() throws Exception {
- setupIntegrationTest();
+ public void testBackendGetRecordNullIdentifierThrowsNullPointerException() throws DataStoreException {
try {
- try {
- DataIdentifier identifier = null;
- backend.getRecord(identifier);
- fail();
- }
- catch (NullPointerException e) {
- assertTrue("identifier must not be null".equals(e.getMessage()));
- }
- } finally {
- teardownIntegrationTest();
+ DataIdentifier identifier = null;
+ backend.getRecord(identifier);
+ fail();
+ }
+ catch (NullPointerException e) {
+ assertTrue("identifier".equals(e.getMessage()));
}
}
@Test
- public void testBackendGetRecordInvalidIdentifierThrowsDataStoreException() throws Exception {
- setupIntegrationTest();
+ public void testBackendGetRecordInvalidIdentifierThrowsDataStoreException() {
try {
- try {
- backend.getRecord(new DataIdentifier("invalid"));
- fail();
- }
- catch (DataStoreException e) {
+ backend.getRecord(new DataIdentifier("invalid"));
+ fail();
+ }
+ catch (DataStoreException e) {
- }
- } finally {
- teardownIntegrationTest();
}
}
- @Test
- public void testBackendGetAllRecordsReturnsAll() throws Exception {
- setupIntegrationTest();
- try {
- for (int recCount : List.of(0, 1, 2, 5)) {
- Map<DataIdentifier, String> addedRecords = new HashMap<>();
- if (0 < recCount) {
- for (int i = 0; i < recCount; i++) {
- String data = String.format("testData%d", i);
- DataRecord record = ds.addRecord(new ByteArrayInputStream(data.getBytes()));
- addedRecords.put(record.getIdentifier(), data);
- }
- }
+ // GetAllRecords (Backend)
... 8273 lines suppressed ...