You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by js...@apache.org on 2026/09/28 13:15:18 UTC
(jackrabbit-oak) branch issue/OAK-12422-optimized-remote-segment-recovery created (now 2c3bf30445)
This is an automated email from the ASF dual-hosted git repository.
jsedding pushed a change to branch issue/OAK-12422-optimized-remote-segment-recovery
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
at 2c3bf30445 OAK-12422: Optimize remote segment archive recovery
This branch includes the following new commits:
new 2c3bf30445 OAK-12422: Optimize remote segment archive recovery
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(jackrabbit-oak) 01/01: OAK-12422: Optimize remote segment archive recovery
Posted by js...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jsedding pushed a commit to branch issue/OAK-12422-optimized-remote-segment-recovery
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit 2c3bf30445c205fe68da2f8f08b69106e3d53e0e
Author: Julian Sedding <js...@apache.org>
AuthorDate: Mon Sep 28 14:35:39 2026 +0200
OAK-12422: Optimize remote segment archive recovery
---
.../oak/segment/aws/AwsArchiveManager.java | 32 +++---
.../oak/segment/aws/AwsSegmentArchiveWriter.java | 5 +
.../oak/segment/aws/AwsArchiveManagerTest.java | 47 +++++++-
.../oak/segment/azure/AzureArchiveManager.java | 30 ++----
.../segment/azure/AzureSegmentArchiveWriter.java | 21 +++-
.../segment/azure/v8/AzureArchiveManagerV8.java | 30 ++----
.../azure/v8/AzureSegmentArchiveWriterV8.java | 22 ++++
.../oak/segment/azure/AzureArchiveManagerTest.java | 16 ++-
.../azure/v8/AzureArchiveManagerV8Test.java | 20 +++-
.../remote/AbstractRemoteSegmentArchiveWriter.java | 43 +++++++-
.../oak/segment/remote/package-info.java | 2 +-
.../AbstractRemoteSegmentArchiveWriterTest.java | 118 +++++++++++++++++++++
.../oak/segment/SegmentNodeStoreRegistrar.java | 6 ++
.../jackrabbit/oak/segment/file/tar/TarReader.java | 8 +-
.../jackrabbit/oak/segment/file/tar/TarWriter.java | 17 ++-
.../spi/persistence/SegmentArchiveManager.java | 15 ++-
.../spi/persistence/SegmentArchiveWriter.java | 23 +++-
.../oak/segment/file/tar/TarWriterTest.java | 71 +++++++++++++
18 files changed, 444 insertions(+), 82 deletions(-)
diff --git a/oak-segment-aws/src/main/java/org/apache/jackrabbit/oak/segment/aws/AwsArchiveManager.java b/oak-segment-aws/src/main/java/org/apache/jackrabbit/oak/segment/aws/AwsArchiveManager.java
index 95a95743b9..3e7de24f95 100644
--- a/oak-segment-aws/src/main/java/org/apache/jackrabbit/oak/segment/aws/AwsArchiveManager.java
+++ b/oak-segment-aws/src/main/java/org/apache/jackrabbit/oak/segment/aws/AwsArchiveManager.java
@@ -17,6 +17,7 @@
package org.apache.jackrabbit.oak.segment.aws;
import static org.apache.jackrabbit.oak.segment.remote.RemoteUtilities.getSegmentUUID;
+import static org.apache.jackrabbit.oak.segment.remote.RemoteUtilities.isSegmentName;
import java.io.IOException;
import java.nio.file.Path;
@@ -187,29 +188,26 @@ public class AwsArchiveManager implements SegmentArchiveManager {
}
/**
- * Avoids deleting segments from the directory given with {@code archiveName},
- * if they are in the set of recovered segments. Reason for that is because
- * during execution of this method, remote repository can be accessed by another
- * application, and deleting a valid segment can cause consistency issues there.
+ * Moves only discarded segments to the backup archive. Recovered segments
+ * remain in place so that recovery can reuse their data.
*/
@Override
public void backup(String archiveName, String backupArchiveName, Set<UUID> recoveredEntries) throws IOException {
- copyFile(archiveName, backupArchiveName);
- delete(archiveName, recoveredEntries);
- }
-
- private void delete(String archiveName, Set<UUID> recoveredEntries) throws IOException {
- List<KeyVersion> keys = new ArrayList<>();
-
- for (S3ObjectSummary b : directory.withDirectory(archiveName).listObjects("")) {
- String name = Paths.get(b.getKey()).getFileName().toString();
- UUID uuid = getSegmentUUID(name);
- if (!recoveredEntries.contains(uuid)) {
- keys.add(new KeyVersion(b.getKey()));
+ S3Directory archiveDirectory = directory.withDirectory(archiveName);
+ S3Directory backupDirectory = directory.withDirectory(backupArchiveName);
+ List<KeyVersion> discarded = new ArrayList<>();
+
+ for (S3ObjectSummary object : archiveDirectory.listObjects("")) {
+ String name = Paths.get(object.getKey()).getFileName().toString();
+ if (isSegmentName(name) && !recoveredEntries.contains(getSegmentUUID(name))) {
+ backupDirectory.copyObject(archiveDirectory, object.getKey());
+ discarded.add(new KeyVersion(object.getKey()));
}
}
- directory.withDirectory(archiveName).deleteObjects(keys);
+ if (!discarded.isEmpty() && !archiveDirectory.deleteObjects(discarded)) {
+ throw new IOException("Can't remove discarded segments from " + archiveName);
+ }
}
private static class RecoveredEntry implements Comparable<RecoveredEntry> {
diff --git a/oak-segment-aws/src/main/java/org/apache/jackrabbit/oak/segment/aws/AwsSegmentArchiveWriter.java b/oak-segment-aws/src/main/java/org/apache/jackrabbit/oak/segment/aws/AwsSegmentArchiveWriter.java
index b1b3e32d18..80a131184a 100644
--- a/oak-segment-aws/src/main/java/org/apache/jackrabbit/oak/segment/aws/AwsSegmentArchiveWriter.java
+++ b/oak-segment-aws/src/main/java/org/apache/jackrabbit/oak/segment/aws/AwsSegmentArchiveWriter.java
@@ -48,6 +48,11 @@ public class AwsSegmentArchiveWriter extends AbstractRemoteSegmentArchiveWriter
return archiveName;
}
+ @Override
+ protected boolean doTryReuseArchiveEntry(RemoteSegmentArchiveEntry indexEntry) {
+ return directory.doesObjectExist(getSegmentFileName(indexEntry));
+ }
+
@Override
protected void doWriteArchiveEntry(RemoteSegmentArchiveEntry indexEntry, byte[] data, int offset, int size) throws IOException {
long msb = indexEntry.getMsb();
diff --git a/oak-segment-aws/src/test/java/org/apache/jackrabbit/oak/segment/aws/AwsArchiveManagerTest.java b/oak-segment-aws/src/test/java/org/apache/jackrabbit/oak/segment/aws/AwsArchiveManagerTest.java
index 18d55b4ac6..16648fbf08 100644
--- a/oak-segment-aws/src/test/java/org/apache/jackrabbit/oak/segment/aws/AwsArchiveManagerTest.java
+++ b/oak-segment-aws/src/test/java/org/apache/jackrabbit/oak/segment/aws/AwsArchiveManagerTest.java
@@ -17,6 +17,11 @@
package org.apache.jackrabbit.oak.segment.aws;
import static org.junit.Assert.assertEquals;
+import static org.mockito.ArgumentMatchers.argThat;
+import static org.mockito.Mockito.clearInvocations;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
import java.io.File;
import java.io.IOException;
@@ -64,7 +69,7 @@ public class AwsArchiveManagerTest {
@Before
public void setup() throws IOException {
- s3 = s3Mock.createClient();
+ s3 = spy(s3Mock.createClient());
AmazonDynamoDB ddb = DynamoDBEmbedded.create().amazonDynamoDB();
long time = new Date().getTime();
bucketName = "bucket-" + time;
@@ -95,6 +100,41 @@ public class AwsArchiveManagerTest {
assertEquals(uuids.subList(0, 5), new ArrayList<>(recovered.keySet()));
}
+ @Test
+ public void testBackupWithRecoveredEntries() throws IOException {
+ SegmentArchiveManager manager = new AwsPersistence(awsContext).createArchiveManager(false, false,
+ new IOMonitorAdapter(), new FileStoreMonitorAdapter(), new RemoteStoreMonitorAdapter());
+ SegmentArchiveWriter writer = manager.create("data00000a.tar");
+
+ List<UUID> uuids = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ UUID uuid = UUID.randomUUID();
+ writer.writeSegment(uuid.getMostSignificantBits(), uuid.getLeastSignificantBits(),
+ new byte[10], 0, 10, 0, 0, false);
+ uuids.add(uuid);
+ }
+ writer.flush();
+ writer.close();
+
+ String archivePrefix = rootDirectory + "/data00000a.tar/";
+ String backupPrefix = rootDirectory + "/data00000a.tar.bak/";
+ s3.deleteObject(bucketName, archivePrefix + "0005." + uuids.get(5));
+
+ LinkedHashMap<UUID, byte[]> recovered = new LinkedHashMap<>();
+ manager.recoverEntries("data00000a.tar", recovered);
+ manager.backup("data00000a.tar", "data00000a.tar.bak", recovered.keySet());
+
+ for (int i = 0; i <= 4; i++) {
+ Assert.assertTrue(s3.doesObjectExist(bucketName, archivePrefix + "000" + i + "." + uuids.get(i)));
+ Assert.assertFalse(s3.doesObjectExist(bucketName, backupPrefix + "000" + i + "." + uuids.get(i)));
+ }
+ Assert.assertFalse(s3.doesObjectExist(bucketName, backupPrefix + "0005." + uuids.get(5)));
+ for (int i = 6; i <= 9; i++) {
+ Assert.assertFalse(s3.doesObjectExist(bucketName, archivePrefix + "000" + i + "." + uuids.get(i)));
+ Assert.assertTrue(s3.doesObjectExist(bucketName, backupPrefix + "000" + i + "." + uuids.get(i)));
+ }
+ }
+
@Test
public void testUncleanStop() throws InvalidFileStoreVersionException, IOException, CommitFailedException {
SegmentNodeStorePersistence p = new AwsPersistence(awsContext);
@@ -108,11 +148,16 @@ public class AwsArchiveManagerTest {
s3.deleteObject(bucketName, rootDirectory + "/data00000a.tar/closed");
s3.deleteObject(bucketName, rootDirectory + "/data00000a.tar/data00000a.tar.brf");
s3.deleteObject(bucketName, rootDirectory + "/data00000a.tar/data00000a.tar.gph");
+ clearInvocations(s3);
fs = FileStoreBuilder.fileStoreBuilder(new File("target")).withCustomPersistence(p).build();
segmentNodeStore = SegmentNodeStoreBuilders.builder(fs).build();
assertEquals("bar", segmentNodeStore.getRoot().getString("foo"));
fs.close();
+
+ verify(s3, never()).putObject(argThat(request -> request.getKey()
+ .matches("oak/data00000a\\.tar/[0-9a-f]{4}\\..*")));
+ Assert.assertTrue(awsContext.directory.listObjects("data00000a.tar.bak/").isEmpty());
}
@Test
diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManager.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManager.java
index fe1426a617..55b36e72d6 100644
--- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManager.java
+++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManager.java
@@ -269,29 +269,21 @@ public class AzureArchiveManager implements SegmentArchiveManager {
}
}
- private void delete(List<BlobItem> from, Set<UUID> recoveredEntries) {
- from.forEach(blobItem -> {
- String name = getName(blobItem);
- if (RemoteUtilities.isSegmentName(name) && !recoveredEntries.contains(RemoteUtilities.getSegmentUUID(name))) {
- try {
- writeBlobContainerClient.getBlobClient(blobItem.getName()).delete();
- } catch (BlobStorageException e) {
- log.error("Can't delete segment {}", blobItem.getName(), e);
- }
- }
- });
- }
-
/**
- * Method is not deleting segments from the directory given with {@code archiveName}, if they are in the set of recovered segments.
- * Reason for that is because during execution of this method, remote repository can be accessed by another application, and deleting a valid segment can
- * cause consistency issues there.
+ * Moves only discarded segments to the backup archive. Recovered segments
+ * remain in place so that recovery can reuse their data.
*/
@Override
public void backup(@NotNull String archiveName, @NotNull String backupArchiveName, @NotNull Set<UUID> recoveredEntries) throws IOException {
- List<BlobItem> blobItems = getBlobs(archiveName);
- batchCopyBlobs(blobItems, backupArchiveName);
- delete(blobItems, recoveredEntries);
+ String backupDirectory = getDirectory(backupArchiveName);
+ for (BlobItem blobItem : getBlobs(archiveName)) {
+ String name = getName(blobItem);
+ if (RemoteUtilities.isSegmentName(name)
+ && !recoveredEntries.contains(RemoteUtilities.getSegmentUUID(name))) {
+ writeAccessController.checkWritingAllowed();
+ renameBlob(blobItem, backupDirectory);
+ }
+ }
}
/**
diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentArchiveWriter.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentArchiveWriter.java
index 9f28315bc9..d831f5ddbb 100644
--- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentArchiveWriter.java
+++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentArchiveWriter.java
@@ -34,7 +34,7 @@ import org.apache.jackrabbit.oak.segment.spi.monitor.IOMonitor;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
-import java.util.Arrays;
+import java.util.Map;
import java.util.concurrent.TimeUnit;
import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.readBufferFully;
@@ -68,6 +68,25 @@ public class AzureSegmentArchiveWriter extends AbstractRemoteSegmentArchiveWrite
return archiveName;
}
+ @Override
+ protected boolean doTryReuseArchiveEntry(RemoteSegmentArchiveEntry indexEntry) throws IOException {
+ BlockBlobClient blob = getBlockBlobClient(getSegmentFileName(indexEntry));
+ Map<String, String> expectedMetadata = AzureBlobMetadata.toSegmentMetadata(indexEntry);
+ try {
+ Map<String, String> metadata = blob.getProperties().getMetadata();
+ if (!metadata.equals(expectedMetadata)) {
+ writeAccessController.checkWritingAllowed();
+ blob.setMetadata(expectedMetadata);
+ }
+ return true;
+ } catch (BlobStorageException e) {
+ if (e.getStatusCode() == 404) {
+ return false;
+ }
+ throw new IOException(e);
+ }
+ }
+
@Override
protected void doWriteArchiveEntry(RemoteSegmentArchiveEntry indexEntry, byte[] data, int offset, int size) throws IOException {
diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/v8/AzureArchiveManagerV8.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/v8/AzureArchiveManagerV8.java
index d63f6144a3..0bbc18f589 100644
--- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/v8/AzureArchiveManagerV8.java
+++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/v8/AzureArchiveManagerV8.java
@@ -255,29 +255,21 @@ public class AzureArchiveManagerV8 implements SegmentArchiveManager {
}
}
- private void delete(String archiveName, Set<UUID> recoveredEntries) throws IOException {
- getBlobs(archiveName)
- .forEach(cloudBlob -> {
- String name = getName(cloudBlob);
- if (RemoteUtilities.isSegmentName(name) && !recoveredEntries.contains(RemoteUtilities.getSegmentUUID(name))) {
- try {
- cloudBlob.delete();
- } catch (StorageException e) {
- log.error("Can't delete segment {}", cloudBlob.getUri().getPath(), e);
- }
- }
- });
- }
-
/**
- * Method is not deleting segments from the directory given with {@code archiveName}, if they are in the set of recovered segments.
- * Reason for that is because during execution of this method, remote repository can be accessed by another application, and deleting a valid segment can
- * cause consistency issues there.
+ * Moves only discarded segments to the backup archive. Recovered segments
+ * remain in place so that recovery can reuse their data.
*/
@Override
public void backup(@NotNull String archiveName, @NotNull String backupArchiveName, @NotNull Set<UUID> recoveredEntries) throws IOException {
- copyFile(archiveName, backupArchiveName);
- delete(archiveName, recoveredEntries);
+ CloudBlobDirectory backupDirectory = getDirectory(backupArchiveName);
+ for (CloudBlob cloudBlob : getBlobs(archiveName)) {
+ String name = getName(cloudBlob);
+ if (RemoteUtilities.isSegmentName(name)
+ && !recoveredEntries.contains(RemoteUtilities.getSegmentUUID(name))) {
+ writeAccessController.checkWritingAllowed();
+ renameBlob(cloudBlob, backupDirectory);
+ }
+ }
}
protected CloudBlobDirectory getDirectory(String archiveName) throws IOException {
diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/v8/AzureSegmentArchiveWriterV8.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/v8/AzureSegmentArchiveWriterV8.java
index cd3e15ca8e..222e2249a1 100644
--- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/v8/AzureSegmentArchiveWriterV8.java
+++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/v8/AzureSegmentArchiveWriterV8.java
@@ -23,6 +23,8 @@ import static org.apache.jackrabbit.oak.segment.remote.RemoteUtilities.OFF_HEAP;
import java.io.File;
import java.io.IOException;
import java.net.URISyntaxException;
+import java.util.HashMap;
+import java.util.Map;
import java.util.NoSuchElementException;
import java.util.concurrent.TimeUnit;
@@ -74,6 +76,26 @@ public class AzureSegmentArchiveWriterV8 extends AbstractRemoteSegmentArchiveWri
return AzureUtilitiesV8.getName(archiveDirectory);
}
+ @Override
+ protected boolean doTryReuseArchiveEntry(RemoteSegmentArchiveEntry indexEntry) throws IOException {
+ CloudBlockBlob blob = getBlob(getSegmentFileName(indexEntry));
+ Map<String, String> expectedMetadata = AzureBlobMetadata.toSegmentMetadata(indexEntry);
+ try {
+ blob.downloadAttributes();
+ if (!blob.getMetadata().equals(expectedMetadata)) {
+ writeAccessController.checkWritingAllowed();
+ blob.setMetadata(new HashMap<>(expectedMetadata));
+ blob.uploadMetadata(null, writeOptimisedBlobRequestOptions, null);
+ }
+ return true;
+ } catch (StorageException e) {
+ if (e.getHttpStatusCode() == 404) {
+ return false;
+ }
+ throw new IOException(e);
+ }
+ }
+
@Override
protected void doWriteArchiveEntry(RemoteSegmentArchiveEntry indexEntry, byte[] data, int offset, int size) throws IOException {
diff --git a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManagerTest.java b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManagerTest.java
index 8a15229a2d..4da2c50f24 100644
--- a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManagerTest.java
+++ b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManagerTest.java
@@ -157,10 +157,13 @@ public class AzureArchiveManagerTest {
for (int i = 0; i <= 4; i++) {
assertTrue(readBlobContainerClient.getBlobClient("oak/data00000a.tar/000" + i + "." + uuids.get(i)).exists());
+ assertFalse(readBlobContainerClient.getBlobClient("oak/data00000a.tar.bak/000" + i + "." + uuids.get(i)).exists());
}
- for (int i = 5; i <= 9; i++) {
+ assertFalse(readBlobContainerClient.getBlobClient("oak/data00000a.tar.bak/0005." + uuids.get(5)).exists());
+ for (int i = 6; i <= 9; i++) {
assertFalse(String.format("Segment %s.??? should have been deleted.", "oak/data00000a.tar/000" + i), readBlobContainerClient.getBlobClient("oak/data00000a.tar/000" + i + "." + uuids.get(i)).exists());
+ assertTrue(readBlobContainerClient.getBlobClient("oak/data00000a.tar.bak/000" + i + "." + uuids.get(i)).exists());
}
}
@@ -175,6 +178,10 @@ public class AzureArchiveManagerTest {
}
+ ListBlobsOptions segmentOptions = new ListBlobsOptions().setPrefix("oak/data00000a.tar/0000.");
+ String segmentBlob = readBlobContainerClient.listBlobs(segmentOptions, null).iterator().next().getName();
+ String segmentETag = readBlobContainerClient.getBlobClient(segmentBlob).getProperties().getETag();
+
readBlobContainerClient.getBlobClient("oak/data00000a.tar/closed").delete();
readBlobContainerClient.getBlobClient("oak/data00000a.tar/data00000a.tar.brf").delete();
readBlobContainerClient.getBlobClient("oak/data00000a.tar/data00000a.tar.gph").delete();
@@ -183,6 +190,9 @@ public class AzureArchiveManagerTest {
SegmentNodeStore segmentNodeStore = SegmentNodeStoreBuilders.builder(fs).build();
assertEquals("bar", segmentNodeStore.getRoot().getString("foo"));
}
+ assertEquals(segmentETag, readBlobContainerClient.getBlobClient(segmentBlob).getProperties().getETag());
+ assertFalse(readBlobContainerClient.listBlobs(
+ new ListBlobsOptions().setPrefix("oak/data00000a.tar.bak/"), null).iterator().hasNext());
}
@Test
@@ -268,9 +278,9 @@ public class AzureArchiveManagerTest {
listOptions.setPrefix("oak/data00001a.tar.bak");
assertTrue("Backup directory should have been created", readBlobContainerClient.listBlobs(listOptions, null).iterator().hasNext());
- //backup has all segments but 0002 since it was deleted before recovery
+ // backup contains only the discarded tail after the missing segment
listOptions.setPrefix("oak/data00001a.tar.bak/0001.");
- assertTrue(readBlobContainerClient.listBlobs(listOptions, null).iterator().hasNext());
+ assertFalse(readBlobContainerClient.listBlobs(listOptions, null).iterator().hasNext());
listOptions.setPrefix("oak/data00001a.tar.bak/0002.");
assertFalse(readBlobContainerClient.listBlobs(listOptions, null).iterator().hasNext());
listOptions.setPrefix("oak/data00001a.tar.bak/0003.");
diff --git a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/v8/AzureArchiveManagerV8Test.java b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/v8/AzureArchiveManagerV8Test.java
index 9065977487..8a1707b43d 100644
--- a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/v8/AzureArchiveManagerV8Test.java
+++ b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/v8/AzureArchiveManagerV8Test.java
@@ -154,11 +154,14 @@ public class AzureArchiveManagerV8Test {
manager.backup("data00000a.tar", "data00000a.tar.bak", recovered.keySet());
for (int i = 0; i <= 4; i++) {
- assertTrue(container.getBlockBlobReference("oak/data00000a.tar/000"+ i + "." + uuids.get(i)).exists());
+ assertTrue(container.getBlockBlobReference("oak/data00000a.tar/000" + i + "." + uuids.get(i)).exists());
+ assertFalse(container.getBlockBlobReference("oak/data00000a.tar.bak/000" + i + "." + uuids.get(i)).exists());
}
- for (int i = 5; i <= 9; i++) {
- assertFalse(String.format("Segment %s.??? should have been deleted.", "oak/data00000a.tar/000"+ i), container.getBlockBlobReference("oak/data00000a.tar/000"+ i + "." + uuids.get(i)).exists());
+ assertFalse(container.getBlockBlobReference("oak/data00000a.tar.bak/0005." + uuids.get(5)).exists());
+ for (int i = 6; i <= 9; i++) {
+ assertFalse(String.format("Segment %s.??? should have been deleted.", "oak/data00000a.tar/000" + i), container.getBlockBlobReference("oak/data00000a.tar/000" + i + "." + uuids.get(i)).exists());
+ assertTrue(container.getBlockBlobReference("oak/data00000a.tar.bak/000" + i + "." + uuids.get(i)).exists());
}
}
@@ -172,6 +175,10 @@ public class AzureArchiveManagerV8Test {
segmentNodeStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
}
+ CloudBlob segmentBlob = (CloudBlob) container.listBlobs("oak/data00000a.tar/0000.").iterator().next();
+ segmentBlob.downloadAttributes();
+ String segmentETag = segmentBlob.getProperties().getEtag();
+
container.getBlockBlobReference("oak/data00000a.tar/closed").delete();
container.getBlockBlobReference("oak/data00000a.tar/data00000a.tar.brf").delete();
container.getBlockBlobReference("oak/data00000a.tar/data00000a.tar.gph").delete();
@@ -180,6 +187,9 @@ public class AzureArchiveManagerV8Test {
SegmentNodeStore segmentNodeStore = SegmentNodeStoreBuilders.builder(fs).build();
assertEquals("bar", segmentNodeStore.getRoot().getString("foo"));
}
+ segmentBlob.downloadAttributes();
+ assertEquals(segmentETag, segmentBlob.getProperties().getEtag());
+ assertFalse(container.listBlobs("oak/data00000a.tar.bak/").iterator().hasNext());
}
@Test
@@ -257,8 +267,8 @@ public class AzureArchiveManagerV8Test {
assertFalse(container.listBlobs("oak/data00001a.tar/0003.").iterator().hasNext());
assertTrue("Backup directory should have been created", container.listBlobs("oak/data00001a.tar.bak").iterator().hasNext());
- //backup has all segments but 0002 since it was deleted before recovery
- assertTrue(container.listBlobs("oak/data00001a.tar.bak/0001.").iterator().hasNext());
+ // backup contains only the discarded tail after the missing segment
+ assertFalse(container.listBlobs("oak/data00001a.tar.bak/0001.").iterator().hasNext());
assertFalse(container.listBlobs("oak/data00001a.tar.bak/0002.").iterator().hasNext());
assertTrue(container.listBlobs("oak/data00001a.tar.bak/0003.").iterator().hasNext());
diff --git a/oak-segment-remote/src/main/java/org/apache/jackrabbit/oak/segment/remote/AbstractRemoteSegmentArchiveWriter.java b/oak-segment-remote/src/main/java/org/apache/jackrabbit/oak/segment/remote/AbstractRemoteSegmentArchiveWriter.java
index 3e1964bb26..7d79d29007 100644
--- a/oak-segment-remote/src/main/java/org/apache/jackrabbit/oak/segment/remote/AbstractRemoteSegmentArchiveWriter.java
+++ b/oak-segment-remote/src/main/java/org/apache/jackrabbit/oak/segment/remote/AbstractRemoteSegmentArchiveWriter.java
@@ -58,18 +58,41 @@ public abstract class AbstractRemoteSegmentArchiveWriter implements SegmentArchi
@Override
public void writeSegment(long msb, long lsb, @NotNull byte[] data, int offset, int size, int generation,
int fullGeneration, boolean compacted) throws IOException {
+ RemoteSegmentArchiveEntry entry = newEntry(msb, lsb, size, generation, fullGeneration, compacted);
+ writeArchiveEntry(entry, data, offset, size);
+ addEntry(entry);
+ monitor.written(size);
+ }
+
+ @Override
+ public void recoverSegment(long msb, long lsb, @NotNull byte[] data, int offset, int size, int generation,
+ int fullGeneration, boolean compacted) throws IOException {
+ RemoteSegmentArchiveEntry entry = newEntry(msb, lsb, size, generation, fullGeneration, compacted);
+ if (!doTryReuseArchiveEntry(entry)) {
+ writeArchiveEntry(entry, data, offset, size);
+ monitor.written(size);
+ }
+ addEntry(entry);
+ }
+
+ private RemoteSegmentArchiveEntry newEntry(long msb, long lsb, int size, int generation, int fullGeneration,
+ boolean compacted) {
created = true;
+ return new RemoteSegmentArchiveEntry(msb, lsb, entries++, size, generation, fullGeneration, compacted);
+ }
- RemoteSegmentArchiveEntry entry = new RemoteSegmentArchiveEntry(msb, lsb, entries++, size, generation, fullGeneration, compacted);
+ private void addEntry(RemoteSegmentArchiveEntry entry) {
+ index.put(entry.getUuid(), entry);
+ totalLength += entry.getLength();
+ }
+
+ private void writeArchiveEntry(RemoteSegmentArchiveEntry entry, byte[] data, int offset, int size)
+ throws IOException {
if (queue.isPresent()) {
queue.get().addToQueue(entry, data, offset, size);
} else {
doWriteArchiveEntry(entry, data, offset, size);
}
- index.put(entry.getUuid(), entry);
-
- totalLength += size;
- monitor.written(size);
}
@Override
@@ -162,6 +185,16 @@ public abstract class AbstractRemoteSegmentArchiveWriter implements SegmentArchi
return RemoteUtilities.MAX_ENTRY_COUNT;
}
+ /**
+ * Attempts to reuse an existing segment during recovery.
+ *
+ * @param indexEntry the archive index entry to recover
+ * @return {@code true} if the segment already exists and can be reused
+ */
+ protected boolean doTryReuseArchiveEntry(RemoteSegmentArchiveEntry indexEntry) throws IOException {
+ return false;
+ }
+
/**
* Writes a segment to the remote storage.
* @param indexEntry, the archive index entry to write
diff --git a/oak-segment-remote/src/main/java/org/apache/jackrabbit/oak/segment/remote/package-info.java b/oak-segment-remote/src/main/java/org/apache/jackrabbit/oak/segment/remote/package-info.java
index 5c70f9fbe8..b790bad231 100644
--- a/oak-segment-remote/src/main/java/org/apache/jackrabbit/oak/segment/remote/package-info.java
+++ b/oak-segment-remote/src/main/java/org/apache/jackrabbit/oak/segment/remote/package-info.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
@Internal(since = "1.0.0")
-@Version("3.0.0")
+@Version("3.1.0")
package org.apache.jackrabbit.oak.segment.remote;
import org.apache.jackrabbit.oak.commons.annotations.Internal;
diff --git a/oak-segment-remote/src/test/java/org/apache/jackrabbit/oak/segment/remote/AbstractRemoteSegmentArchiveWriterTest.java b/oak-segment-remote/src/test/java/org/apache/jackrabbit/oak/segment/remote/AbstractRemoteSegmentArchiveWriterTest.java
new file mode 100644
index 0000000000..6327120e96
--- /dev/null
+++ b/oak-segment-remote/src/test/java/org/apache/jackrabbit/oak/segment/remote/AbstractRemoteSegmentArchiveWriterTest.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.segment.remote;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.jackrabbit.oak.commons.Buffer;
+import org.apache.jackrabbit.oak.segment.spi.monitor.FileStoreMonitorAdapter;
+import org.apache.jackrabbit.oak.segment.spi.monitor.IOMonitorAdapter;
+import org.junit.Test;
+
+public class AbstractRemoteSegmentArchiveWriterTest {
+
+ private static final byte[] DATA = {1, 2, 3};
+
+ @Test
+ public void recoverSegmentReusesExistingEntry() throws Exception {
+ TestWriter writer = new TestWriter(true);
+
+ writer.recoverSegment(1, 2, DATA, 0, DATA.length, 3, 4, true);
+ writer.close();
+
+ assertEquals(1, writer.recoveredEntries.size());
+ assertEquals(0, writer.writtenEntries.size());
+ assertEquals(0, writer.recoveredEntries.get(0).getPosition());
+ assertEquals(1, writer.getEntryCount());
+ assertEquals(DATA.length, writer.getLength());
+ assertTrue(writer.containsSegment(1, 2));
+ }
+
+ @Test
+ public void recoverSegmentWritesMissingEntry() throws Exception {
+ TestWriter writer = new TestWriter(false);
+
+ writer.recoverSegment(1, 2, DATA, 0, DATA.length, 3, 4, true);
+ writer.close();
+
+ assertEquals(1, writer.recoveredEntries.size());
+ assertEquals(1, writer.writtenEntries.size());
+ assertEquals(0, writer.writtenEntries.get(0).getPosition());
+ }
+
+ @Test
+ public void recoveredEntriesKeepArchiveOrder() throws Exception {
+ TestWriter writer = new TestWriter(true);
+
+ writer.recoverSegment(1, 2, DATA, 0, DATA.length, 0, 0, false);
+ writer.recoverSegment(3, 4, DATA, 0, DATA.length, 0, 0, false);
+ writer.close();
+
+ assertEquals(0, writer.recoveredEntries.get(0).getPosition());
+ assertEquals(1, writer.recoveredEntries.get(1).getPosition());
+ }
+
+ private static final class TestWriter extends AbstractRemoteSegmentArchiveWriter {
+
+ private final boolean reuse;
+ private final List<RemoteSegmentArchiveEntry> recoveredEntries = new ArrayList<>();
+ private final List<RemoteSegmentArchiveEntry> writtenEntries = new ArrayList<>();
+
+ private TestWriter(boolean reuse) {
+ super(new IOMonitorAdapter(), new FileStoreMonitorAdapter());
+ this.reuse = reuse;
+ }
+
+ @Override
+ public String getName() {
+ return "test";
+ }
+
+ @Override
+ protected boolean doTryReuseArchiveEntry(RemoteSegmentArchiveEntry indexEntry) {
+ recoveredEntries.add(indexEntry);
+ return reuse;
+ }
+
+ @Override
+ protected void doWriteArchiveEntry(RemoteSegmentArchiveEntry indexEntry, byte[] data, int offset, int size) {
+ writtenEntries.add(indexEntry);
+ }
+
+ @Override
+ protected Buffer doReadArchiveEntry(RemoteSegmentArchiveEntry indexEntry) {
+ return Buffer.wrap(DATA);
+ }
+
+ @Override
+ protected void doWriteDataFile(byte[] data, String extension) {
+ }
+
+ @Override
+ protected void afterQueueClosed() throws IOException {
+ }
+
+ @Override
+ protected void afterQueueFlushed() throws IOException {
+ }
+ }
+}
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreRegistrar.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreRegistrar.java
index e45592d17d..5978b680a4 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreRegistrar.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreRegistrar.java
@@ -51,6 +51,7 @@ import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException;
import org.apache.jackrabbit.oak.segment.file.MetricsIOMonitor;
import org.apache.jackrabbit.oak.segment.file.MetricsRemoteStoreMonitor;
import org.apache.jackrabbit.oak.segment.file.tar.TarPersistence;
+import org.apache.jackrabbit.oak.segment.file.tar.TarReader;
import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentNodeStorePersistence;
import org.apache.jackrabbit.oak.segment.spi.persistence.persistentcache.CachingPersistence;
import org.apache.jackrabbit.oak.segment.spi.persistence.persistentcache.PersistentCache;
@@ -278,6 +279,11 @@ class SegmentNodeStoreRegistrar {
}
registerCloseable(store);
+ registerCloseable(cfg.getWhiteboard().register(FeatureToggle.class,
+ new FeatureToggle(TarReader.FT_OPTIMIZED_REMOTE_RECOVERY_OAK_12422,
+ TarReader.FT_OPTIMIZED_REMOTE_RECOVERY_OAK_12422_ENABLED),
+ Collections.emptyMap()));
+
// OAK-12214: bug-fix toggle (default on) so L2 eviction policy sees L1 memoised hits
registerCloseable(cfg.getWhiteboard().register(FeatureToggle.class,
new FeatureToggle(SegmentCache.FT_OAK_12214, SegmentCache.FT_OAK_12214_PROPAGATE_L1_HITS_TO_L2_ENABLED),
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java
index f9c7382561..487aa409ef 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java
@@ -37,6 +37,7 @@ import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.UUID;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Consumer;
import java.util.function.Predicate;
@@ -58,6 +59,11 @@ import org.slf4j.LoggerFactory;
public class TarReader implements Closeable {
+ public static final String FT_OPTIMIZED_REMOTE_RECOVERY_OAK_12422 =
+ "FT_OPTIMIZED_REMOTE_RECOVERY_OAK-12422";
+
+ public static final AtomicBoolean FT_OPTIMIZED_REMOTE_RECOVERY_OAK_12422_ENABLED = new AtomicBoolean(true);
+
private static final Logger log = LoggerFactory.getLogger(TarReader.class);
static TarReader open(String file, SegmentArchiveManager archiveManager) throws IOException {
@@ -186,7 +192,7 @@ public class TarReader implements Closeable {
recovery.recoverEntry(entry.getKey(), entry.getValue(), new EntryRecovery() {
@Override
public void recoverEntry(long msb, long lsb, byte[] data, int offset, int size, GCGeneration generation) throws IOException {
- writer.writeEntry(msb, lsb, data, offset, size, generation);
+ writer.recoverEntry(msb, lsb, data, offset, size, generation);
}
@Override
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriter.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriter.java
index d9c57c3e4e..94ec966428 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriter.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriter.java
@@ -137,13 +137,28 @@ class TarWriter implements Closeable {
}
long writeEntry(long msb, long lsb, byte[] data, int offset, int size, GCGeneration generation) throws IOException {
+ return writeEntry(msb, lsb, data, offset, size, generation, false);
+ }
+
+ long recoverEntry(long msb, long lsb, byte[] data, int offset, int size, GCGeneration generation) throws IOException {
+ return writeEntry(msb, lsb, data, offset, size, generation, true);
+ }
+
+ private long writeEntry(long msb, long lsb, byte[] data, int offset, int size, GCGeneration generation,
+ boolean recovery) throws IOException {
requireNonNull(data);
checkFromToIndex(offset, offset + size, data.length);
synchronized (this) {
Validate.checkState(!closed);
- archive.writeSegment(msb, lsb, data, offset, size, generation.getGeneration(), generation.getFullGeneration(), generation.isCompacted());
+ if (recovery && TarReader.FT_OPTIMIZED_REMOTE_RECOVERY_OAK_12422_ENABLED.get()) {
+ archive.recoverSegment(msb, lsb, data, offset, size, generation.getGeneration(),
+ generation.getFullGeneration(), generation.isCompacted());
+ } else {
+ archive.writeSegment(msb, lsb, data, offset, size, generation.getGeneration(),
+ generation.getFullGeneration(), generation.isCompacted());
+ }
segmentCount.inc();
long currentLength = archive.getLength();
int currentEntryCount = archive.getEntryCount();
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/SegmentArchiveManager.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/SegmentArchiveManager.java
index 37ccd0e1ce..b5d80905d0 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/SegmentArchiveManager.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/SegmentArchiveManager.java
@@ -116,15 +116,14 @@ public interface SegmentArchiveManager {
void recoverEntries(@NotNull String archiveName, @NotNull LinkedHashMap<UUID, byte[]> entries) throws IOException;
/**
- * Method that is doing a backup of the archive given with {@code archiveName} into {@code backupArchiveName}.
- * In addition, set of UUIDs of recovered segments is provided which can be inspected during backup.
- * Method is invoked during archive recovery procedure and concrete implementation can decide whether original archive
- * should be deleted or modified.
+ * Backs up an archive during recovery. Implementations may back up the
+ * complete archive or only entries excluded from {@code recoveredEntries},
+ * and may leave recovered entries in the original archive for reuse.
*
- * @param archiveName
- * @param backupArchiveName
- * @param recoveredEntries
- * @throws IOException
+ * @param archiveName archive being recovered
+ * @param backupArchiveName destination for backed-up entries
+ * @param recoveredEntries segment identifiers retained by recovery
+ * @throws IOException if the backup cannot be completed
*/
void backup(@NotNull String archiveName, @NotNull String backupArchiveName, @NotNull Set<UUID> recoveredEntries) throws IOException;
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/SegmentArchiveWriter.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/SegmentArchiveWriter.java
index 7fcff23b93..9ff0e4f437 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/SegmentArchiveWriter.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/SegmentArchiveWriter.java
@@ -32,7 +32,8 @@ import org.jetbrains.annotations.Nullable;
* <ol>
* <li>phase 1:
* <ul>
- * <li>{@link #writeSegment(long, long, byte[], int, int, int, int, boolean)}
+ * <li>{@link #writeSegment(long, long, byte[], int, int, int, int, boolean)} or
+ * {@link #recoverSegment(long, long, byte[], int, int, int, int, boolean)}
* <li>{@link #flush()}
* </ul>
* repeated in an unspecified order
@@ -66,6 +67,26 @@ public interface SegmentArchiveWriter {
@NotNull
void writeSegment(long msb, long lsb, @NotNull byte[] data, int offset, int size, int generation, int fullGeneration, boolean isCompacted) throws IOException;
+ /**
+ * Recover a segment into the archive. Implementations may reuse an already
+ * persisted segment instead of writing its data again. The default
+ * implementation writes the segment normally.
+ *
+ * @param msb the most significant bits of the identifier of the segment
+ * @param lsb the least significant bits of the identifier of the segment
+ * @param data the data.
+ * @param offset the start offset in the data.
+ * @param size the number of bytes to write.
+ * @param generation the segment generation, see {@link SegmentArchiveEntry#getGeneration()}
+ * @param fullGeneration the segment full generation, see {@link SegmentArchiveEntry#getFullGeneration()}
+ * @param isCompacted the segment compaction property, see {@link SegmentArchiveEntry#isCompacted()}
+ * @throws IOException error recovering segment
+ */
+ default void recoverSegment(long msb, long lsb, @NotNull byte[] data, int offset, int size, int generation,
+ int fullGeneration, boolean isCompacted) throws IOException {
+ writeSegment(msb, lsb, data, offset, size, generation, fullGeneration, isCompacted);
+ }
+
/**
* Read the segment.
*
diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriterTest.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriterTest.java
index 11f13e08c1..7edf6489aa 100644
--- a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriterTest.java
+++ b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriterTest.java
@@ -21,6 +21,7 @@ package org.apache.jackrabbit.oak.segment.file.tar;
import static org.apache.jackrabbit.oak.segment.spi.persistence.GCGeneration.newGCGeneration;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotSame;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertThrows;
@@ -38,6 +39,7 @@ import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentArchiveManager;
import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentArchiveWriter;
import org.apache.jackrabbit.oak.stats.NoopStats;
import org.jetbrains.annotations.NotNull;
+import org.junit.After;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -47,6 +49,11 @@ public class TarWriterTest {
@Rule
public TemporaryFolder folder = new TemporaryFolder(new File("target"));
+ @After
+ public void resetRecoveryFeature() {
+ TarReader.FT_OPTIMIZED_REMOTE_RECOVERY_OAK_12422_ENABLED.set(true);
+ }
+
protected TestFileStoreMonitor monitor = new TestFileStoreMonitor();
@Test
@@ -120,6 +127,70 @@ public class TarWriterTest {
}
+ @Test
+ public void recoveryUsesRecoverSegment() throws Exception {
+ boolean[] recovered = {false};
+ IOMonitorAdapter ioMonitor = new IOMonitorAdapter();
+ File segmentstoreDir = folder.newFolder();
+ SegmentArchiveManager archiveManager = new SegmentTarManager(
+ segmentstoreDir, monitor, ioMonitor, false, false) {
+ @Override
+ public @NotNull SegmentArchiveWriter create(String archiveName) {
+ return new SegmentTarWriter(new File(segmentstoreDir, archiveName), monitor, ioMonitor) {
+ @Override
+ public void recoverSegment(long msb, long lsb, byte[] data, int offset, int size,
+ int generation, int fullGeneration, boolean isCompacted) throws IOException {
+ recovered[0] = true;
+ writeSegment(msb, lsb, data, offset, size, generation, fullGeneration, isCompacted);
+ }
+ };
+ }
+ };
+
+ try (TarWriter writer = new TarWriter(archiveManager, "data00000a.tar")) {
+ writer.recoverEntry(0, 0, new byte[42], 0, 42, newGCGeneration(0, 0, false));
+ }
+
+ assertTrue(recovered[0]);
+ }
+
+ @Test
+ public void recoveryCanBeDisabled() throws Exception {
+ boolean[] recovered = {false};
+ boolean[] written = {false};
+ IOMonitorAdapter ioMonitor = new IOMonitorAdapter();
+ File segmentstoreDir = folder.newFolder();
+ SegmentArchiveManager archiveManager = new SegmentTarManager(
+ segmentstoreDir, monitor, ioMonitor, false, false) {
+ @Override
+ public @NotNull SegmentArchiveWriter create(String archiveName) {
+ return new SegmentTarWriter(new File(segmentstoreDir, archiveName), monitor, ioMonitor) {
+ @Override
+ public void recoverSegment(long msb, long lsb, byte[] data, int offset, int size,
+ int generation, int fullGeneration, boolean isCompacted) throws IOException {
+ recovered[0] = true;
+ super.recoverSegment(msb, lsb, data, offset, size, generation, fullGeneration, isCompacted);
+ }
+
+ @Override
+ public void writeSegment(long msb, long lsb, byte[] data, int offset, int size,
+ int generation, int fullGeneration, boolean isCompacted) throws IOException {
+ written[0] = true;
+ super.writeSegment(msb, lsb, data, offset, size, generation, fullGeneration, isCompacted);
+ }
+ };
+ }
+ };
+ TarReader.FT_OPTIMIZED_REMOTE_RECOVERY_OAK_12422_ENABLED.set(false);
+
+ try (TarWriter writer = new TarWriter(archiveManager, "data00000a.tar")) {
+ writer.recoverEntry(0, 0, new byte[42], 0, 42, newGCGeneration(0, 0, false));
+ }
+
+ assertFalse(recovered[0]);
+ assertTrue(written[0]);
+ }
+
@Test
public void testFileStoreMonitor() throws Exception {
try (TarWriter writer = new TarWriter(getSegmentArchiveManager(), 0, NoopStats.INSTANCE)) {