You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by mi...@apache.org on 2022/12/16 16:40:19 UTC
[jackrabbit-oak] 01/01: OAK-10022 for Azure segment store binary references and graph pushed on flush
This is an automated email from the ASF dual-hosted git repository.
miroslav pushed a commit to branch issue/OAK-10022
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit 3cc1f087c65cdbbfc73c9970021409e53c633cc5
Author: smiroslav <mi...@apache.org>
AuthorDate: Fri Dec 16 17:39:27 2022 +0100
OAK-10022 for Azure segment store binary references and graph pushed on flush
---
.../oak/segment/azure/AzureArchiveManagerTest.java | 29 ++++++++++++++++++++++
.../split/SplitPersistenceBlobTest.java | 7 ++++--
.../remote/AbstractRemoteSegmentArchiveWriter.java | 6 +++++
.../oak/segment/remote/package-info.java | 20 +++++++++++++++
.../jackrabbit/oak/segment/file/tar/TarWriter.java | 9 ++++++-
.../spi/persistence/SegmentArchiveWriter.java | 16 ++++++++++++
6 files changed, 84 insertions(+), 3 deletions(-)
diff --git a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManagerTest.java b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManagerTest.java
index decea0bc0f..f8c71f841d 100644
--- a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManagerTest.java
+++ b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManagerTest.java
@@ -400,6 +400,35 @@ public class AzureArchiveManagerTest {
assertFalse(container.getDirectoryReference("oak/data00000a.tar.ro.bak").listBlobs().iterator().hasNext());
}
+ @Test
+ public void testCollectBlobReferencesForReadOnlyFileStore() throws URISyntaxException, InvalidFileStoreVersionException, IOException, CommitFailedException, StorageException {
+ AzurePersistence rwPersistence = new AzurePersistence(container.getDirectoryReference("oak"));
+ FileStore rwFileStore = FileStoreBuilder.fileStoreBuilder(new File("target")).withCustomPersistence(rwPersistence).build();
+ SegmentNodeStore segmentNodeStore = SegmentNodeStoreBuilders.builder(rwFileStore).build();
+ NodeBuilder builder = segmentNodeStore.getRoot().builder();
+ builder.setProperty("foo", "bar");
+ segmentNodeStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ rwFileStore.flush();
+
+ assertTrue(container.getDirectoryReference("oak/data00000a.tar").listBlobs().iterator().hasNext());
+ // create read-only FS
+ AzurePersistence roPersistence = new AzurePersistence(container.getDirectoryReference("oak"));
+ ReadOnlyFileStore roFileStore = FileStoreBuilder.fileStoreBuilder(new File("target")).withCustomPersistence(roPersistence).buildReadOnly();
+
+ PropertyState fooProperty = SegmentNodeStoreBuilders.builder(roFileStore).build()
+ .getRoot()
+ .getProperty("foo");
+
+ assertThat(fooProperty, not(nullValue()));
+ assertThat(fooProperty.getValue(Type.STRING), equalTo("bar"));
+
+ roFileStore.collectBlobReferences(s -> {});
+
+ roFileStore.close();
+
+ rwFileStore.close();
+ }
+
private PersistentCache createPersistenceCache() {
return new AbstractPersistentCache() {
@Override
diff --git a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/spi/persistence/split/SplitPersistenceBlobTest.java b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/spi/persistence/split/SplitPersistenceBlobTest.java
index eee7cb43bf..97de82d683 100644
--- a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/spi/persistence/split/SplitPersistenceBlobTest.java
+++ b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/spi/persistence/split/SplitPersistenceBlobTest.java
@@ -44,6 +44,7 @@ import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.jetbrains.annotations.Nullable;
import org.junit.After;
import org.junit.Before;
import org.junit.ClassRule;
@@ -74,6 +75,8 @@ public class SplitPersistenceBlobTest {
private SegmentNodeStorePersistence splitPersistence;
+ private String sharedPersistenceBlobId;
+
@Before
public void setup() throws IOException, InvalidFileStoreVersionException, CommitFailedException, URISyntaxException, InvalidKeyException, StorageException {
SegmentNodeStorePersistence sharedPersistence =
@@ -103,7 +106,7 @@ public class SplitPersistenceBlobTest {
.build();
base = SegmentNodeStoreBuilders.builder(baseFileStore).build();
- createLoad(base, baseFileStore).getContentIdentity();
+ sharedPersistenceBlobId = createLoad(base, baseFileStore).getContentIdentity();
baseFileStore.flush();
SegmentNodeStorePersistence localPersistence = new TarPersistence(folder.newFolder());
@@ -127,7 +130,7 @@ public class SplitPersistenceBlobTest {
throws IOException, CommitFailedException {
String blobId = createLoad(split, splitFileStore).getContentIdentity();
- assertReferences(2, Sets.newHashSet(baseBlobId, blobId));
+ assertReferences(3, Sets.newHashSet(baseBlobId, sharedPersistenceBlobId, blobId));
}
private static Blob createBlob(NodeStore nodeStore, int size) throws IOException {
diff --git a/oak-segment-remote/src/main/java/org/apache/jackrabbit/oak/segment/remote/AbstractRemoteSegmentArchiveWriter.java b/oak-segment-remote/src/main/java/org/apache/jackrabbit/oak/segment/remote/AbstractRemoteSegmentArchiveWriter.java
index 28bc1f95f5..a3009d4510 100644
--- a/oak-segment-remote/src/main/java/org/apache/jackrabbit/oak/segment/remote/AbstractRemoteSegmentArchiveWriter.java
+++ b/oak-segment-remote/src/main/java/org/apache/jackrabbit/oak/segment/remote/AbstractRemoteSegmentArchiveWriter.java
@@ -150,6 +150,12 @@ public abstract class AbstractRemoteSegmentArchiveWriter implements SegmentArchi
}
}
+ @Override
+ public void flush(Runnable callback) throws IOException {
+ flush();
+ callback.run();
+ }
+
/**
* Writes a segment to the remote storage.
* @param indexEntry, the archive index entry to write
diff --git a/oak-segment-remote/src/main/java/org/apache/jackrabbit/oak/segment/remote/package-info.java b/oak-segment-remote/src/main/java/org/apache/jackrabbit/oak/segment/remote/package-info.java
new file mode 100644
index 0000000000..cc0da5e7a3
--- /dev/null
+++ b/oak-segment-remote/src/main/java/org/apache/jackrabbit/oak/segment/remote/package-info.java
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+@Version("1.0.0")
+package org.apache.jackrabbit.oak.segment.remote;
+
+import org.osgi.annotation.versioning.Version;
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriter.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriter.java
index 6c3b8b7ec1..3f9cb90892 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriter.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriter.java
@@ -183,7 +183,14 @@ class TarWriter implements Closeable {
}
if (doFlush) {
- archive.flush();
+ archive.flush(() -> {
+ try {
+ writeBinaryReferences();
+ writeGraph();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
}
}
}
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/SegmentArchiveWriter.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/SegmentArchiveWriter.java
index 8596e1bdd5..01a42637ed 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/SegmentArchiveWriter.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/SegmentArchiveWriter.java
@@ -19,6 +19,8 @@
package org.apache.jackrabbit.oak.segment.spi.persistence;
import java.io.IOException;
+import java.util.concurrent.Callable;
+import java.util.function.BiConsumer;
import org.apache.jackrabbit.oak.commons.Buffer;
import org.jetbrains.annotations.NotNull;
@@ -140,4 +142,18 @@ public interface SegmentArchiveWriter {
*/
@NotNull
String getName();
+
+ /**
+ * This method allows the caller to perform additional operations after {@link #flush()} has been finished.
+ * Can be used for remote segment store implementations to invoke {@link #writeBinaryReferences(byte[])} and {@link #writeGraph(byte[])}
+ * so that all necessary metadata is available for another Oak process concurrently accessing the storage.
+ *
+ * Default implementation invokes {@link #flush()} without invoking callback method afterwards
+ *
+ * @param onSuccess
+ * @throws IOException
+ */
+ default void flush(Runnable onSuccess) throws IOException {
+ flush();
+ }
}