You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by mi...@apache.org on 2023/11/29 16:58:04 UTC
(jackrabbit-oak) branch issue/OAK-10573_azure_write_timeout created (now f0665d8450)
This is an automated email from the ASF dual-hosted git repository.
miroslav pushed a change to branch issue/OAK-10573_azure_write_timeout
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
at f0665d8450 OAK-10573 request options optimised for write operations
This branch includes the following new commits:
new f0665d8450 OAK-10573 request options optimised for write operations
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(jackrabbit-oak) 01/01: OAK-10573 request options optimised for write operations
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
miroslav pushed a commit to branch issue/OAK-10573_azure_write_timeout
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit f0665d84507bac6fa42c9ab41419be33907e961e
Author: Miroslav Smiljanic <mi...@apache.com>
AuthorDate: Wed Nov 29 17:57:23 2023 +0100
OAK-10573 request options optimised for write operations
---
.../oak/segment/azure/AzureJournalFile.java | 15 ++++--
.../oak/segment/azure/AzurePersistence.java | 22 +-------
.../segment/azure/AzureSegmentArchiveWriter.java | 13 +++--
.../segment/azure/util/AzureRequestOptions.java | 63 ++++++++++++++++++++++
4 files changed, 85 insertions(+), 28 deletions(-)
diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureJournalFile.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureJournalFile.java
index b468e42b44..76b52de113 100644
--- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureJournalFile.java
+++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureJournalFile.java
@@ -22,6 +22,9 @@ import com.microsoft.azure.storage.blob.CloudAppendBlob;
import com.microsoft.azure.storage.blob.CloudBlob;
import com.microsoft.azure.storage.blob.CloudBlobDirectory;
import com.microsoft.azure.storage.blob.ListBlobItem;
+import com.microsoft.azure.storage.blob.DeleteSnapshotsOption;
+import com.microsoft.azure.storage.blob.BlobRequestOptions;
+import org.apache.jackrabbit.oak.segment.azure.util.AzureRequestOptions;
import org.apache.jackrabbit.oak.segment.azure.util.CaseInsensitiveKeysMapAccess;
import org.apache.jackrabbit.oak.segment.remote.WriteAccessController;
import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFile;
@@ -162,7 +165,11 @@ public class AzureJournalFile implements JournalFile {
private int lineCount;
+ private final BlobRequestOptions writeOptimisedBlobRequestOptions;
+
public AzureJournalWriter() throws IOException {
+ writeOptimisedBlobRequestOptions = AzureRequestOptions.optimiseForWriteOperations(directory.getServiceClient().getDefaultRequestOptions());
+
List<CloudAppendBlob> blobs = getJournalBlobs();
if (blobs.isEmpty()) {
try {
@@ -190,7 +197,7 @@ public class AzureJournalFile implements JournalFile {
writeAccessController.checkWritingAllowed();
for (CloudAppendBlob cloudAppendBlob : getJournalBlobs()) {
- cloudAppendBlob.delete();
+ cloudAppendBlob.delete(DeleteSnapshotsOption.NONE, null, writeOptimisedBlobRequestOptions, null);
}
createNextFile(0);
@@ -229,11 +236,11 @@ public class AzureJournalFile implements JournalFile {
text.append(line).append("\n");
}
try {
- currentBlob.appendText(text.toString());
+ currentBlob.appendText(text.toString(), null, null, writeOptimisedBlobRequestOptions, null);
currentBlob.getMetadata().put("lastEntry", entries.get(entries.size() - 1));
lineCount += entries.size();
currentBlob.getMetadata().put("lineCount", Integer.toString(lineCount));
- currentBlob.uploadMetadata();
+ currentBlob.uploadMetadata(null, writeOptimisedBlobRequestOptions, null);
} catch (StorageException e) {
throw new IOException(e);
}
@@ -243,7 +250,7 @@ public class AzureJournalFile implements JournalFile {
private void createNextFile(int suffix) throws IOException {
try {
currentBlob = directory.getAppendBlobReference(getJournalFileName(suffix + 1));
- currentBlob.createOrReplace();
+ currentBlob.createOrReplace(null, writeOptimisedBlobRequestOptions, null);
lineCount = 0;
} catch (URISyntaxException | StorageException e) {
throw new IOException(e);
diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzurePersistence.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzurePersistence.java
index eed07004e5..392a352915 100644
--- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzurePersistence.java
+++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzurePersistence.java
@@ -34,6 +34,7 @@ import com.microsoft.azure.storage.blob.CloudAppendBlob;
import com.microsoft.azure.storage.blob.CloudBlobDirectory;
import com.microsoft.azure.storage.blob.CloudBlockBlob;
import com.microsoft.azure.storage.blob.ListBlobItem;
+import org.apache.jackrabbit.oak.segment.azure.util.AzureRequestOptions;
import org.apache.jackrabbit.oak.segment.remote.WriteAccessController;
import org.apache.jackrabbit.oak.segment.spi.monitor.FileStoreMonitor;
import org.apache.jackrabbit.oak.segment.spi.monitor.IOMonitor;
@@ -70,26 +71,7 @@ public class AzurePersistence implements SegmentNodeStorePersistence {
public AzurePersistence(CloudBlobDirectory segmentStoreDirectory) {
this.segmentstoreDirectory = segmentStoreDirectory;
- BlobRequestOptions defaultRequestOptions = segmentStoreDirectory.getServiceClient().getDefaultRequestOptions();
- if (defaultRequestOptions.getRetryPolicyFactory() == null) {
- int retryAttempts = Integer.getInteger(RETRY_ATTEMPTS_PROP, DEFAULT_RETRY_ATTEMPTS);
- if (retryAttempts > 0) {
- Integer retryBackoffSeconds = Integer.getInteger(RETRY_BACKOFF_PROP, DEFAULT_RETRY_BACKOFF_SECONDS);
- defaultRequestOptions.setRetryPolicyFactory(new RetryLinearRetry((int) TimeUnit.SECONDS.toMillis(retryBackoffSeconds), retryAttempts));
- }
- }
- if (defaultRequestOptions.getMaximumExecutionTimeInMs() == null) {
- int timeoutExecution = Integer.getInteger(TIMEOUT_EXECUTION_PROP, DEFAULT_TIMEOUT_EXECUTION);
- if (timeoutExecution > 0) {
- defaultRequestOptions.setMaximumExecutionTimeInMs((int) TimeUnit.SECONDS.toMillis(timeoutExecution));
- }
- }
- if (defaultRequestOptions.getTimeoutIntervalInMs() == null) {
- int timeoutInterval = Integer.getInteger(TIMEOUT_INTERVAL_PROP, DEFAULT_TIMEOUT_INTERVAL);
- if (timeoutInterval > 0) {
- defaultRequestOptions.setTimeoutIntervalInMs((int) TimeUnit.SECONDS.toMillis(timeoutInterval));
- }
- }
+ AzureRequestOptions.applyDefaultRequestOptions(segmentStoreDirectory.getServiceClient().getDefaultRequestOptions());
}
@Override
diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentArchiveWriter.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentArchiveWriter.java
index 3df0eaafa6..e9bbbb322a 100644
--- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentArchiveWriter.java
+++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentArchiveWriter.java
@@ -25,12 +25,14 @@ import java.io.IOException;
import java.net.URISyntaxException;
import java.util.concurrent.TimeUnit;
+import com.microsoft.azure.storage.blob.BlobRequestOptions;
import org.apache.jackrabbit.guava.common.base.Stopwatch;
import com.microsoft.azure.storage.StorageException;
import com.microsoft.azure.storage.blob.CloudBlobDirectory;
import com.microsoft.azure.storage.blob.CloudBlockBlob;
import org.apache.jackrabbit.oak.commons.Buffer;
+import org.apache.jackrabbit.oak.segment.azure.util.AzureRequestOptions;
import org.apache.jackrabbit.oak.segment.remote.WriteAccessController;
import org.apache.jackrabbit.oak.segment.azure.util.Retrier;
import org.apache.jackrabbit.oak.segment.remote.AbstractRemoteSegmentArchiveWriter;
@@ -47,10 +49,13 @@ public class AzureSegmentArchiveWriter extends AbstractRemoteSegmentArchiveWrite
Integer.getInteger("azure.segment.archive.writer.retries.intervalMs", 5000)
);
+ private final BlobRequestOptions writeOptimisedBlobRequestOptions;
+
public AzureSegmentArchiveWriter(CloudBlobDirectory archiveDirectory, IOMonitor ioMonitor, FileStoreMonitor monitor, WriteAccessController writeAccessController) {
super(ioMonitor, monitor);
this.archiveDirectory = archiveDirectory;
this.writeAccessController = writeAccessController;
+ this.writeOptimisedBlobRequestOptions = AzureRequestOptions.optimiseForWriteOperations(archiveDirectory.getServiceClient().getDefaultRequestOptions());
}
@Override
@@ -71,8 +76,8 @@ public class AzureSegmentArchiveWriter extends AbstractRemoteSegmentArchiveWrite
Stopwatch stopwatch = Stopwatch.createStarted();
try {
blob.setMetadata(AzureBlobMetadata.toSegmentMetadata(indexEntry));
- blob.uploadFromByteArray(data, offset, size);
- blob.uploadMetadata();
+ blob.uploadFromByteArray(data, offset, size, null, writeOptimisedBlobRequestOptions, null);
+ blob.uploadMetadata(null, writeOptimisedBlobRequestOptions, null);
} catch (StorageException e) {
throw new IOException(e);
}
@@ -97,7 +102,7 @@ public class AzureSegmentArchiveWriter extends AbstractRemoteSegmentArchiveWrite
try {
writeAccessController.checkWritingAllowed();
- getBlob(getName() + extension).uploadFromByteArray(data, 0, data.length);
+ getBlob(getName() + extension).uploadFromByteArray(data, 0, data.length, null, writeOptimisedBlobRequestOptions, null);
} catch (StorageException e) {
throw new IOException(e);
}
@@ -110,7 +115,7 @@ public class AzureSegmentArchiveWriter extends AbstractRemoteSegmentArchiveWrite
try {
writeAccessController.checkWritingAllowed();
- getBlob("closed").uploadFromByteArray(new byte[0], 0, 0);
+ getBlob("closed").uploadFromByteArray(new byte[0], 0, 0, null, writeOptimisedBlobRequestOptions, null);
} catch (StorageException e) {
throw new IOException(e);
}
diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/util/AzureRequestOptions.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/util/AzureRequestOptions.java
new file mode 100644
index 0000000000..485612371d
--- /dev/null
+++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/util/AzureRequestOptions.java
@@ -0,0 +1,63 @@
+package org.apache.jackrabbit.oak.segment.azure.util;
+
+import com.microsoft.azure.storage.RetryLinearRetry;
+import com.microsoft.azure.storage.blob.BlobRequestOptions;
+
+import java.util.concurrent.TimeUnit;
+
+public class AzureRequestOptions {
+
+ private static final String RETRY_ATTEMPTS_PROP = "segment.azure.retry.attempts";
+ private static final int DEFAULT_RETRY_ATTEMPTS = 5;
+
+ private static final String RETRY_BACKOFF_PROP = "segment.azure.retry.backoff";
+ private static final int DEFAULT_RETRY_BACKOFF_SECONDS = 5;
+
+ private static final String TIMEOUT_EXECUTION_PROP = "segment.timeout.execution";
+ private static final int DEFAULT_TIMEOUT_EXECUTION = 30;
+
+ private static final String TIMEOUT_INTERVAL_PROP = "segment.timeout.interval";
+ private static final int DEFAULT_TIMEOUT_INTERVAL = 1;
+
+ private static final String WRITE_TIMEOUT_EXECUTION_PROP = "segment.write.timeout.execution";
+
+ private static final String WRITE_TIMEOUT_INTERVAL_PROP = "segment.write.timeout.interval";
+
+ public static void applyDefaultRequestOptions(BlobRequestOptions blobRequestOptions) {
+ if (blobRequestOptions.getRetryPolicyFactory() == null) {
+ int retryAttempts = Integer.getInteger(RETRY_ATTEMPTS_PROP, DEFAULT_RETRY_ATTEMPTS);
+ if (retryAttempts > 0) {
+ Integer retryBackoffSeconds = Integer.getInteger(RETRY_BACKOFF_PROP, DEFAULT_RETRY_BACKOFF_SECONDS);
+ blobRequestOptions.setRetryPolicyFactory(new RetryLinearRetry((int) TimeUnit.SECONDS.toMillis(retryBackoffSeconds), retryAttempts));
+ }
+ }
+ if (blobRequestOptions.getMaximumExecutionTimeInMs() == null) {
+ int timeoutExecution = Integer.getInteger(TIMEOUT_EXECUTION_PROP, DEFAULT_TIMEOUT_EXECUTION);
+ if (timeoutExecution > 0) {
+ blobRequestOptions.setMaximumExecutionTimeInMs((int) TimeUnit.SECONDS.toMillis(timeoutExecution));
+ }
+ }
+ if (blobRequestOptions.getTimeoutIntervalInMs() == null) {
+ int timeoutInterval = Integer.getInteger(TIMEOUT_INTERVAL_PROP, DEFAULT_TIMEOUT_INTERVAL);
+ if (timeoutInterval > 0) {
+ blobRequestOptions.setTimeoutIntervalInMs((int) TimeUnit.SECONDS.toMillis(timeoutInterval));
+ }
+ }
+ }
+
+ public static BlobRequestOptions optimiseForWriteOperations(BlobRequestOptions blobRequestOptions) {
+ BlobRequestOptions writeOptimisedBlobRequestOptions = new BlobRequestOptions(blobRequestOptions);
+
+ Integer writeTimeoutExecution = Integer.getInteger(WRITE_TIMEOUT_EXECUTION_PROP);
+ if (writeTimeoutExecution != null) {
+ writeOptimisedBlobRequestOptions.setMaximumExecutionTimeInMs(writeTimeoutExecution);
+ }
+
+ Integer writeTimeoutInterval = Integer.getInteger(WRITE_TIMEOUT_INTERVAL_PROP);
+ if (writeTimeoutInterval != null) {
+ writeOptimisedBlobRequestOptions.setTimeoutIntervalInMs(writeTimeoutInterval);
+ }
+
+ return writeOptimisedBlobRequestOptions;
+ }
+}