You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ma...@apache.org on 2020/07/21 18:41:32 UTC
svn commit: r1880114 - in
/jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage:
AzureBlobStoreBackend.java Utils.java
Author: mattryan
Date: Tue Jul 21 18:41:31 2020
New Revision: 1880114
URL: http://svn.apache.org/viewvc?rev=1880114&view=rev
Log:
OAK-9142: Use concurrent request limit option for all Azure interactions
Modified:
jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/Utils.java
Modified: jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java?rev=1880114&r1=1880113&r2=1880114&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java (original)
+++ jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java Tue Jul 21 18:41:31 2020
@@ -54,7 +54,6 @@ import com.google.common.collect.Abstrac
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.microsoft.azure.storage.AccessCondition;
-import com.microsoft.azure.storage.RequestOptions;
import com.microsoft.azure.storage.ResultContinuation;
import com.microsoft.azure.storage.ResultSegment;
import com.microsoft.azure.storage.RetryPolicy;
@@ -77,11 +76,11 @@ import org.apache.jackrabbit.core.data.D
import org.apache.jackrabbit.core.data.DataRecord;
import org.apache.jackrabbit.core.data.DataStoreException;
import org.apache.jackrabbit.oak.commons.PropertiesUtil;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordDownloadOptions;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUpload;
import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadException;
import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadOptions;
import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadToken;
-import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordDownloadOptions;
-import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUpload;
import org.apache.jackrabbit.oak.spi.blob.AbstractDataRecord;
import org.apache.jackrabbit.oak.spi.blob.AbstractSharedBackend;
import org.apache.jackrabbit.util.Base64;
@@ -108,11 +107,13 @@ public class AzureBlobStoreBackend exten
static final long MAX_BINARY_UPLOAD_SIZE = (long) Math.floor(1024L * 1024L * 1024L * 1024L * 4.75); // 4.75TB, Azure limit
private static final int MAX_ALLOWABLE_UPLOAD_URIS = 50000; // Azure limit
private static final int MAX_UNIQUE_RECORD_TRIES = 10;
+ private static final int DEFAULT_CONCURRENT_REQUEST_COUNT = 2;
+ private static final int MAX_CONCURRENT_REQUEST_COUNT = 50;
private Properties properties;
private String containerName;
private String connectionString;
- private int concurrentRequestCount = 1;
+ private int concurrentRequestCount = DEFAULT_CONCURRENT_REQUEST_COUNT;
private RetryPolicy retryPolicy;
private Integer requestTimeout;
private int httpDownloadURIExpirySeconds = 0; // disabled by default
@@ -131,15 +132,16 @@ public class AzureBlobStoreBackend exten
}
protected CloudBlobContainer getAzureContainer() throws DataStoreException {
- CloudBlobContainer container = Utils.getBlobContainer(connectionString, containerName);
- RequestOptions requestOptions = container.getServiceClient().getDefaultRequestOptions();
- if (retryPolicy != null) {
+ BlobRequestOptions requestOptions = new BlobRequestOptions();
+ if (null != retryPolicy) {
requestOptions.setRetryPolicyFactory(retryPolicy);
}
- if (requestTimeout != null) {
+ if (null != requestTimeout) {
requestOptions.setTimeoutIntervalInMs(requestTimeout);
}
- return container;
+ requestOptions.setConcurrentRequestCount(concurrentRequestCount);
+
+ return Utils.getBlobContainer(connectionString, containerName, requestOptions);
}
@Override
@@ -164,8 +166,23 @@ public class AzureBlobStoreBackend exten
containerName = (String) properties.get(AzureConstants.AZURE_BLOB_CONTAINER_NAME);
createBlobContainer = PropertiesUtil.toBoolean(properties.getProperty(AzureConstants.AZURE_CREATE_CONTAINER), true);
connectionString = Utils.getConnectionStringFromProperties(properties);
- concurrentRequestCount = PropertiesUtil.toInteger(properties.get(AzureConstants.AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION), 1);
+
+ concurrentRequestCount = PropertiesUtil.toInteger(
+ properties.get(AzureConstants.AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION),
+ DEFAULT_CONCURRENT_REQUEST_COUNT);
+ if (concurrentRequestCount < DEFAULT_CONCURRENT_REQUEST_COUNT) {
+ LOG.warn("Invalid setting [{}] for concurrentRequestsPerOperation (too low); resetting to {}",
+ concurrentRequestCount,
+ DEFAULT_CONCURRENT_REQUEST_COUNT);
+ concurrentRequestCount = DEFAULT_CONCURRENT_REQUEST_COUNT;
+ } else if (concurrentRequestCount > MAX_CONCURRENT_REQUEST_COUNT) {
+ LOG.warn("Invalid setting [{}] for concurrentRequestsPerOperation (too high); resetting to {}",
+ concurrentRequestCount,
+ MAX_CONCURRENT_REQUEST_COUNT);
+ concurrentRequestCount = MAX_CONCURRENT_REQUEST_COUNT;
+ }
LOG.info("Using concurrentRequestsPerOperation={}", concurrentRequestCount);
+
retryPolicy = Utils.getRetryPolicy((String)properties.get(AzureConstants.AZURE_BLOB_MAX_REQUEST_RETRY));
if (properties.getProperty(AzureConstants.AZURE_BLOB_REQUEST_TIMEOUT) != null) {
requestTimeout = PropertiesUtil.toInteger(properties.getProperty(AzureConstants.AZURE_BLOB_REQUEST_TIMEOUT), RetryPolicy.DEFAULT_CLIENT_RETRY_COUNT);
Modified: jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/Utils.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/Utils.java?rev=1880114&r1=1880113&r2=1880114&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/Utils.java (original)
+++ jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/Utils.java Tue Jul 21 18:41:31 2020
@@ -19,6 +19,17 @@
package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.InetSocketAddress;
+import java.net.Proxy;
+import java.net.SocketAddress;
+import java.net.URISyntaxException;
+import java.security.InvalidKeyException;
+import java.util.Properties;
+
import com.google.common.base.Strings;
import com.microsoft.azure.storage.CloudStorageAccount;
import com.microsoft.azure.storage.OperationContext;
@@ -26,21 +37,13 @@ import com.microsoft.azure.storage.Retry
import com.microsoft.azure.storage.RetryNoRetry;
import com.microsoft.azure.storage.RetryPolicy;
import com.microsoft.azure.storage.StorageException;
+import com.microsoft.azure.storage.blob.BlobRequestOptions;
import com.microsoft.azure.storage.blob.CloudBlobClient;
import com.microsoft.azure.storage.blob.CloudBlobContainer;
import org.apache.jackrabbit.core.data.DataStoreException;
import org.apache.jackrabbit.oak.commons.PropertiesUtil;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.InetSocketAddress;
-import java.net.Proxy;
-import java.net.SocketAddress;
-import java.net.URISyntaxException;
-import java.security.InvalidKeyException;
-import java.util.Properties;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
public final class Utils {
@@ -60,15 +63,34 @@ public final class Utils {
* @param connectionString connectionString to configure @link {@link CloudBlobClient}
* @return {@link CloudBlobClient}
*/
- public static CloudBlobClient getBlobClient(final String connectionString) throws URISyntaxException, InvalidKeyException {
+ public static CloudBlobClient getBlobClient(@NotNull final String connectionString) throws URISyntaxException, InvalidKeyException {
+ return getBlobClient(connectionString, null);
+ }
+
+ public static CloudBlobClient getBlobClient(@NotNull final String connectionString,
+ @Nullable final BlobRequestOptions requestOptions) throws URISyntaxException, InvalidKeyException {
CloudStorageAccount account = CloudStorageAccount.parse(connectionString);
CloudBlobClient client = account.createCloudBlobClient();
+ if (null != requestOptions) {
+ client.setDefaultRequestOptions(requestOptions);
+ }
return client;
}
- public static CloudBlobContainer getBlobContainer(final String connectionString, final String containerName) throws DataStoreException {
+ public static CloudBlobContainer getBlobContainer(@NotNull final String connectionString,
+ @NotNull final String containerName) throws DataStoreException {
+ return getBlobContainer(connectionString, containerName, null);
+ }
+
+ public static CloudBlobContainer getBlobContainer(@NotNull final String connectionString,
+ @NotNull final String containerName,
+ @Nullable final BlobRequestOptions requestOptions) throws DataStoreException {
try {
- CloudBlobClient client = Utils.getBlobClient(connectionString);
+ CloudBlobClient client = (
+ (null == requestOptions)
+ ? Utils.getBlobClient(connectionString)
+ : Utils.getBlobClient(connectionString, requestOptions)
+ );
return client.getContainerReference(containerName);
} catch (InvalidKeyException | URISyntaxException | StorageException e) {
throw new DataStoreException(e);