You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by js...@apache.org on 2022/09/01 16:25:45 UTC
[jackrabbit-oak] branch OAK-9914-segment-azure-read-only-without-recovery created (now 5244488295)
This is an automated email from the ASF dual-hosted git repository.
jsedding pushed a change to branch OAK-9914-segment-azure-read-only-without-recovery
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
at 5244488295 OAK-9914 - Starting Oak with Azure persistence in read-only mode while another Oak process is running will initiate repo recovery
This branch includes the following new commits:
new 5244488295 OAK-9914 - Starting Oak with Azure persistence in read-only mode while another Oak process is running will initiate repo recovery
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[jackrabbit-oak] 01/01: OAK-9914 - Starting Oak with Azure persistence in read-only mode while another Oak process is running will initiate repo recovery
Posted by js...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jsedding pushed a commit to branch OAK-9914-segment-azure-read-only-without-recovery
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit 524448829545dbf5896b4cc23d2d50fcdb6a127e
Author: Julian Sedding <js...@apache.org>
AuthorDate: Thu Sep 1 18:25:24 2022 +0200
OAK-9914 - Starting Oak with Azure persistence in read-only mode while another Oak process is running will initiate repo recovery
---
.../oak/segment/azure/AzureArchiveManager.java | 2 +-
.../oak/segment/azure/AzureArchiveManagerTest.java | 38 ++++++++++++++++++++
.../jackrabbit/oak/segment/file/tar/TarReader.java | 42 ++++++++++++++--------
3 files changed, 66 insertions(+), 16 deletions(-)
diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManager.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManager.java
index 7e2d28b9e8..a087d11a83 100644
--- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManager.java
+++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManager.java
@@ -111,7 +111,7 @@ public class AzureArchiveManager implements SegmentArchiveManager {
try {
CloudBlobDirectory archiveDirectory = getDirectory(archiveName);
if (!archiveDirectory.getBlockBlobReference("closed").exists()) {
- throw new IOException("The archive " + archiveName + " hasn't been closed correctly.");
+ return null;
}
return new AzureSegmentArchiveReader(archiveDirectory, ioMonitor);
} catch (StorageException | URISyntaxException e) {
diff --git a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManagerTest.java b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManagerTest.java
index 1a97334a10..295713e9b7 100644
--- a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManagerTest.java
+++ b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManagerTest.java
@@ -21,6 +21,8 @@ import com.microsoft.azure.storage.blob.CloudBlob;
import com.microsoft.azure.storage.blob.CloudBlobContainer;
import com.microsoft.azure.storage.blob.ListBlobItem;
import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.commons.Buffer;
import org.apache.jackrabbit.oak.segment.SegmentId;
import org.apache.jackrabbit.oak.segment.SegmentNodeStore;
@@ -29,6 +31,7 @@ import org.apache.jackrabbit.oak.segment.SegmentNotFoundException;
import org.apache.jackrabbit.oak.segment.file.FileStore;
import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException;
+import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore;
import org.apache.jackrabbit.oak.segment.spi.RepositoryNotReachableException;
import org.apache.jackrabbit.oak.segment.spi.monitor.FileStoreMonitorAdapter;
import org.apache.jackrabbit.oak.segment.spi.monitor.IOMonitorAdapter;
@@ -55,6 +58,10 @@ import java.util.List;
import java.util.UUID;
import static com.google.common.collect.Lists.newArrayList;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.nullValue;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.IsNot.not;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@ -321,4 +328,35 @@ public class AzureArchiveManagerTest {
// SegmentNotFoundException should be thrown here
fileStore.readSegment(new SegmentId(fileStore, u.getMostSignificantBits(), u.getLeastSignificantBits()));
}
+
+ @Test
+ public void testReadOnlyRecovery() throws URISyntaxException, InvalidFileStoreVersionException, IOException, CommitFailedException, StorageException {
+ AzurePersistence rwPersistence = new AzurePersistence(container.getDirectoryReference("oak"));
+ FileStore rwFileStore = FileStoreBuilder.fileStoreBuilder(new File("target")).withCustomPersistence(rwPersistence).build();
+ SegmentNodeStore segmentNodeStore = SegmentNodeStoreBuilders.builder(rwFileStore).build();
+ NodeBuilder builder = segmentNodeStore.getRoot().builder();
+ builder.setProperty("foo", "bar");
+ segmentNodeStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ rwFileStore.flush();
+
+ assertTrue(container.getDirectoryReference("oak/data00000a.tar").listBlobs().iterator().hasNext());
+ assertFalse(container.getDirectoryReference("oak/data00000a.tar.ro.bak").listBlobs().iterator().hasNext());
+
+ // create read-only FS
+ AzurePersistence roPersistence = new AzurePersistence(container.getDirectoryReference("oak"));
+ ReadOnlyFileStore roFileStore = FileStoreBuilder.fileStoreBuilder(new File("target")).withCustomPersistence(roPersistence).buildReadOnly();
+
+ PropertyState fooProperty = SegmentNodeStoreBuilders.builder(roFileStore).build()
+ .getRoot()
+ .getProperty("foo");
+ assertThat(fooProperty, not(nullValue()));
+ assertThat(fooProperty.getValue(Type.STRING), equalTo("bar"));
+
+ roFileStore.close();
+ rwFileStore.close();
+
+ assertTrue(container.getDirectoryReference("oak/data00000a.tar").listBlobs().iterator().hasNext());
+ // after creating a read-only FS, the recovery procedure should not be started since there is another running Oak process
+ assertFalse(container.getDirectoryReference("oak/data00000a.tar.ro.bak").listBlobs().iterator().hasNext());
+ }
}
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java
index 8b7722cb94..db036db950 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java
@@ -112,24 +112,32 @@ public class TarReader implements Closeable {
}
}
+ @FunctionalInterface
+ private interface OpenStrategy {
+ SegmentArchiveReader open(SegmentArchiveManager segmentArchiveManager, String archiveName) throws IOException;
+ }
+
static TarReader openRO(Map<Character, String> files, TarRecovery recovery, SegmentArchiveManager archiveManager) throws IOException {
// for readonly store only try the latest generation of a given
// tar file to prevent any rollback or rewrite
String file = files.get(Collections.max(files.keySet()));
- TarReader reader = openFirstFileWithValidIndex(singletonList(file), archiveManager);
- if (reader != null) {
- return reader;
- }
- log.warn("Could not find a valid tar index in {}, recovering read-only", file);
- // collecting the entries (without touching the original file) and
- // writing them into an artificial tar file '.ro.bak'
- LinkedHashMap<UUID, byte[]> entries = newLinkedHashMap();
- collectFileEntries(file, entries, false, archiveManager);
- file = findAvailGen(file, ".ro.bak", archiveManager);
- generateTarFile(entries, file, recovery, archiveManager);
- reader = openFirstFileWithValidIndex(singletonList(file), archiveManager);
- if (reader != null) {
- return reader;
+
+ OpenStrategy recoverAndOpen = (segmentArchiveManager, archiveName) -> {
+ log.info("Could not find a valid tar index in {}, recovering read-only", archiveName);
+ // collecting the entries (without touching the original file) and
+ // writing them into an artificial tar file '.ro.bak'
+ LinkedHashMap<UUID, byte[]> entries = newLinkedHashMap();
+ collectFileEntries(archiveName, entries, false, segmentArchiveManager);
+ String bakFile = findAvailGen(archiveName, ".ro.bak", segmentArchiveManager);
+ generateTarFile(entries, bakFile, recovery, segmentArchiveManager);
+ return segmentArchiveManager.open(bakFile);
+ };
+
+ for (OpenStrategy openStrategy : new OpenStrategy[]{SegmentArchiveManager::open, SegmentArchiveManager::forceOpen, recoverAndOpen}) {
+ TarReader reader = openFirstFileWithValidIndex(singletonList(file), archiveManager, openStrategy);
+ if (reader != null) {
+ return reader;
+ }
}
throw new IOException("Failed to open tar file " + file);
}
@@ -227,9 +235,13 @@ public class TarReader implements Closeable {
}
private static TarReader openFirstFileWithValidIndex(List<String> archives, SegmentArchiveManager archiveManager) {
+ return openFirstFileWithValidIndex(archives, archiveManager, SegmentArchiveManager::open);
+ }
+
+ private static TarReader openFirstFileWithValidIndex(List<String> archives, SegmentArchiveManager archiveManager, OpenStrategy openStrategy) {
for (String name : archives) {
try {
- SegmentArchiveReader reader = archiveManager.open(name);
+ SegmentArchiveReader reader = openStrategy.open(archiveManager, name);
if (reader != null) {
for (String other : archives) {
if (other != name) {