You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by mi...@apache.org on 2023/04/26 15:50:58 UTC
[jackrabbit-oak] branch issue/OAK-10212 updated: OAK-10212 return null if binary references file is missing for remote store implementation
This is an automated email from the ASF dual-hosted git repository.
miroslav pushed a commit to branch issue/OAK-10212
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/issue/OAK-10212 by this push:
new 17ad0c1e3a OAK-10212 return null if binary references file is missing for remote store implementation
17ad0c1e3a is described below
commit 17ad0c1e3a1276b9033efe67dfcc372407ab8b8c
Author: smiroslav <mi...@apache.org>
AuthorDate: Wed Apr 26 17:50:48 2023 +0200
OAK-10212 return null if binary references file is missing for remote store implementation
---
.../oak/segment/azure/AzureArchiveManagerTest.java | 41 ++++++++++++++++++++++
.../remote/AbstractRemoteSegmentArchiveReader.java | 5 +++
.../oak/segment/remote/package-info.java | 2 +-
.../oak/segment/file/tar/SegmentTarReader.java | 5 +++
.../jackrabbit/oak/segment/file/tar/TarReader.java | 11 +++++-
.../spi/persistence/SegmentArchiveReader.java | 10 ++++++
.../CachingSegmentArchiveReader.java | 5 +++
.../split/UnclosedSegmentArchiveReader.java | 5 +++
8 files changed, 82 insertions(+), 2 deletions(-)
diff --git a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManagerTest.java b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManagerTest.java
index cf379c6318..146b28e86e 100644
--- a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManagerTest.java
+++ b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManagerTest.java
@@ -33,6 +33,7 @@ import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException;
import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore;
import org.apache.jackrabbit.oak.segment.file.tar.TarPersistence;
+import org.apache.jackrabbit.oak.segment.file.tar.TarReader;
import org.apache.jackrabbit.oak.segment.spi.RepositoryNotReachableException;
import org.apache.jackrabbit.oak.segment.spi.monitor.FileStoreMonitorAdapter;
import org.apache.jackrabbit.oak.segment.spi.monitor.IOMonitorAdapter;
@@ -61,6 +62,8 @@ import java.io.IOException;
import java.net.URISyntaxException;
import java.security.InvalidKeyException;
import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.UUID;
@@ -432,6 +435,44 @@ public class AzureArchiveManagerTest {
}
}
+ @Test
+ public void testCollectBlobReferencesDoesNotFailWhenFileIsMissing() throws URISyntaxException, InvalidFileStoreVersionException, IOException, CommitFailedException, StorageException {
+ AzurePersistence rwPersistence = new AzurePersistence(container.getDirectoryReference("oak"));
+ try (FileStore rwFileStore = FileStoreBuilder.fileStoreBuilder(new File("target")).withCustomPersistence(rwPersistence).build()) {
+ SegmentNodeStore segmentNodeStore = SegmentNodeStoreBuilders.builder(rwFileStore).build();
+ NodeBuilder builder = segmentNodeStore.getRoot().builder();
+ builder.setProperty("foo", "bar");
+ segmentNodeStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ rwFileStore.flush();
+
+ // file with binary references has been created
+ assertTrue(container.getDirectoryReference("oak/data00000a.tar").getBlockBlobReference("data00000a.tar.brf").exists());
+
+ // delete binary references file
+ container.getDirectoryReference("oak/data00000a.tar").getBlockBlobReference("data00000a.tar.brf").delete();
+
+ // create read-only FS
+ AzurePersistence roPersistence = new AzurePersistence(container.getDirectoryReference("oak"));
+ try (ReadOnlyFileStore roFileStore = FileStoreBuilder.fileStoreBuilder(new File("target")).withCustomPersistence(roPersistence).buildReadOnly()) {
+
+ PropertyState fooProperty = SegmentNodeStoreBuilders.builder(roFileStore).build()
+ .getRoot()
+ .getProperty("foo");
+
+ assertThat(fooProperty, not(nullValue()));
+ assertThat(fooProperty.getValue(Type.STRING), equalTo("bar"));
+
+ // no exception should be thrown
+ HashSet<String> references = new HashSet<>();
+ roFileStore.collectBlobReferences(reference -> {
+ references.add(reference);
+ });
+
+ assertTrue("No references should have been collected since reference file has been deleted", references.isEmpty());
+ }
+ }
+ }
+
private PersistentCache createPersistenceCache() {
return new AbstractPersistentCache() {
@Override
diff --git a/oak-segment-remote/src/main/java/org/apache/jackrabbit/oak/segment/remote/AbstractRemoteSegmentArchiveReader.java b/oak-segment-remote/src/main/java/org/apache/jackrabbit/oak/segment/remote/AbstractRemoteSegmentArchiveReader.java
index b5b5a0cb67..c2b480f8b4 100644
--- a/oak-segment-remote/src/main/java/org/apache/jackrabbit/oak/segment/remote/AbstractRemoteSegmentArchiveReader.java
+++ b/oak-segment-remote/src/main/java/org/apache/jackrabbit/oak/segment/remote/AbstractRemoteSegmentArchiveReader.java
@@ -135,4 +135,9 @@ public abstract class AbstractRemoteSegmentArchiveReader implements SegmentArchi
* @return the decoded file component of this archive.
*/
protected abstract File archivePathAsFile();
+
+ @Override
+ public boolean isRemote() {
+ return true;
+ }
}
diff --git a/oak-segment-remote/src/main/java/org/apache/jackrabbit/oak/segment/remote/package-info.java b/oak-segment-remote/src/main/java/org/apache/jackrabbit/oak/segment/remote/package-info.java
index fe60cada0c..ce6b89b5db 100644
--- a/oak-segment-remote/src/main/java/org/apache/jackrabbit/oak/segment/remote/package-info.java
+++ b/oak-segment-remote/src/main/java/org/apache/jackrabbit/oak/segment/remote/package-info.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
@Internal(since = "1.0.0")
-@Version("1.0.0")
+@Version("1.1.0")
package org.apache.jackrabbit.oak.segment.remote;
import org.apache.jackrabbit.oak.commons.annotations.Internal;
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/SegmentTarReader.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/SegmentTarReader.java
index 8000a42020..fe800f143f 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/SegmentTarReader.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/SegmentTarReader.java
@@ -197,4 +197,9 @@ public class SegmentTarReader implements SegmentArchiveReader {
return getEntrySize(buffer.getInt(buffer.limit() - 8));
}
+
+ @Override
+ public boolean isRemote() {
+ return false;
+ }
}
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java
index db036db950..4cf9dffcd2 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java
@@ -620,7 +620,16 @@ public class TarReader implements Closeable {
BinaryReferencesIndex getBinaryReferences() {
BinaryReferencesIndex index = null;
try {
- index = BinaryReferencesIndexLoader.parseBinaryReferencesIndex(archive.getBinaryReferences());
+
+ Buffer binaryReferences = archive.getBinaryReferences();
+ if (binaryReferences == null && archive.isRemote()) {
+
+ // This can happen because segment files and binary references files are flushed one after another in
+ // {@link TarWriter#flush}
+ log.info("The remote archive directory {} still does not have file with binary references written.", archive.getName());
+ return null;
+ }
+ index = BinaryReferencesIndexLoader.parseBinaryReferencesIndex(binaryReferences);
} catch (InvalidBinaryReferencesIndexException | IOException e) {
log.warn("Exception while loading binary reference", e);
}
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/SegmentArchiveReader.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/SegmentArchiveReader.java
index cdd1b079ea..2e99a5970e 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/SegmentArchiveReader.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/SegmentArchiveReader.java
@@ -112,4 +112,14 @@ public interface SegmentArchiveReader extends Closeable {
* segment
*/
int getEntrySize(int size);
+
+ /**
+ * This method returns {@code true} if the storage is accessed via a network protocol, not tight to the traditional storage technology,
+ * for example, HTTP. Based on that info, for instance, calling classes can decide to update archive metadata (graph, binary references, index) more frequently,
+ * and not only when the archive is being closed. With that multiple Oak processes can access the storage simultaneously, with one process in read-write mode and
+ * one or more processes in read-only mode.
+ *
+ * @return
+ */
+ boolean isRemote();
}
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/persistentcache/CachingSegmentArchiveReader.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/persistentcache/CachingSegmentArchiveReader.java
index 07e33c9020..9c682d6b3a 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/persistentcache/CachingSegmentArchiveReader.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/persistentcache/CachingSegmentArchiveReader.java
@@ -99,4 +99,9 @@ public class CachingSegmentArchiveReader implements SegmentArchiveReader {
public int getEntrySize(int size) {
return delegate.getEntrySize(size);
}
+
+ @Override
+ public boolean isRemote() {
+ return delegate.isRemote();
+ }
}
\ No newline at end of file
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/split/UnclosedSegmentArchiveReader.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/split/UnclosedSegmentArchiveReader.java
index 2c4e2aa203..37c74024b6 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/split/UnclosedSegmentArchiveReader.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/split/UnclosedSegmentArchiveReader.java
@@ -90,4 +90,9 @@ class UnclosedSegmentArchiveReader implements SegmentArchiveReader {
public int getEntrySize(int size) {
return delegate.getEntrySize(size);
}
+
+ @Override
+ public boolean isRemote() {
+ return delegate.isRemote();
+ }
}