You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by mi...@apache.org on 2022/12/16 16:40:18 UTC
[jackrabbit-oak] branch issue/OAK-10022 created (now 3cc1f087c6)
This is an automated email from the ASF dual-hosted git repository.
miroslav pushed a change to branch issue/OAK-10022
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
at 3cc1f087c6 OAK-10022 for Azure segment store binary references and graph pushed on flush
This branch includes the following new commits:
new 3cc1f087c6 OAK-10022 for Azure segment store binary references and graph pushed on flush
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[jackrabbit-oak] 01/01: OAK-10022 for Azure segment store binary references and graph pushed on flush
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
miroslav pushed a commit to branch issue/OAK-10022
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit 3cc1f087c65cdbbfc73c9970021409e53c633cc5
Author: smiroslav <mi...@apache.org>
AuthorDate: Fri Dec 16 17:39:27 2022 +0100
OAK-10022 for Azure segment store binary references and graph pushed on flush
---
.../oak/segment/azure/AzureArchiveManagerTest.java | 29 ++++++++++++++++++++++
.../split/SplitPersistenceBlobTest.java | 7 ++++--
.../remote/AbstractRemoteSegmentArchiveWriter.java | 6 +++++
.../oak/segment/remote/package-info.java | 20 +++++++++++++++
.../jackrabbit/oak/segment/file/tar/TarWriter.java | 9 ++++++-
.../spi/persistence/SegmentArchiveWriter.java | 16 ++++++++++++
6 files changed, 84 insertions(+), 3 deletions(-)
diff --git a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManagerTest.java b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManagerTest.java
index decea0bc0f..f8c71f841d 100644
--- a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManagerTest.java
+++ b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManagerTest.java
@@ -400,6 +400,35 @@ public class AzureArchiveManagerTest {
assertFalse(container.getDirectoryReference("oak/data00000a.tar.ro.bak").listBlobs().iterator().hasNext());
}
+ @Test
+ public void testCollectBlobReferencesForReadOnlyFileStore() throws URISyntaxException, InvalidFileStoreVersionException, IOException, CommitFailedException, StorageException {
+ AzurePersistence rwPersistence = new AzurePersistence(container.getDirectoryReference("oak"));
+ FileStore rwFileStore = FileStoreBuilder.fileStoreBuilder(new File("target")).withCustomPersistence(rwPersistence).build();
+ SegmentNodeStore segmentNodeStore = SegmentNodeStoreBuilders.builder(rwFileStore).build();
+ NodeBuilder builder = segmentNodeStore.getRoot().builder();
+ builder.setProperty("foo", "bar");
+ segmentNodeStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ rwFileStore.flush();
+
+ assertTrue(container.getDirectoryReference("oak/data00000a.tar").listBlobs().iterator().hasNext());
+ // create read-only FS
+ AzurePersistence roPersistence = new AzurePersistence(container.getDirectoryReference("oak"));
+ ReadOnlyFileStore roFileStore = FileStoreBuilder.fileStoreBuilder(new File("target")).withCustomPersistence(roPersistence).buildReadOnly();
+
+ PropertyState fooProperty = SegmentNodeStoreBuilders.builder(roFileStore).build()
+ .getRoot()
+ .getProperty("foo");
+
+ assertThat(fooProperty, not(nullValue()));
+ assertThat(fooProperty.getValue(Type.STRING), equalTo("bar"));
+
+ roFileStore.collectBlobReferences(s -> {});
+
+ roFileStore.close();
+
+ rwFileStore.close();
+ }
+
private PersistentCache createPersistenceCache() {
return new AbstractPersistentCache() {
@Override
diff --git a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/spi/persistence/split/SplitPersistenceBlobTest.java b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/spi/persistence/split/SplitPersistenceBlobTest.java
index eee7cb43bf..97de82d683 100644
--- a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/spi/persistence/split/SplitPersistenceBlobTest.java
+++ b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/spi/persistence/split/SplitPersistenceBlobTest.java
@@ -44,6 +44,7 @@ import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.jetbrains.annotations.Nullable;
import org.junit.After;
import org.junit.Before;
import org.junit.ClassRule;
@@ -74,6 +75,8 @@ public class SplitPersistenceBlobTest {
private SegmentNodeStorePersistence splitPersistence;
+ private String sharedPersistenceBlobId;
+
@Before
public void setup() throws IOException, InvalidFileStoreVersionException, CommitFailedException, URISyntaxException, InvalidKeyException, StorageException {
SegmentNodeStorePersistence sharedPersistence =
@@ -103,7 +106,7 @@ public class SplitPersistenceBlobTest {
.build();
base = SegmentNodeStoreBuilders.builder(baseFileStore).build();
- createLoad(base, baseFileStore).getContentIdentity();
+ sharedPersistenceBlobId = createLoad(base, baseFileStore).getContentIdentity();
baseFileStore.flush();
SegmentNodeStorePersistence localPersistence = new TarPersistence(folder.newFolder());
@@ -127,7 +130,7 @@ public class SplitPersistenceBlobTest {
throws IOException, CommitFailedException {
String blobId = createLoad(split, splitFileStore).getContentIdentity();
- assertReferences(2, Sets.newHashSet(baseBlobId, blobId));
+ assertReferences(3, Sets.newHashSet(baseBlobId, sharedPersistenceBlobId, blobId));
}
private static Blob createBlob(NodeStore nodeStore, int size) throws IOException {
diff --git a/oak-segment-remote/src/main/java/org/apache/jackrabbit/oak/segment/remote/AbstractRemoteSegmentArchiveWriter.java b/oak-segment-remote/src/main/java/org/apache/jackrabbit/oak/segment/remote/AbstractRemoteSegmentArchiveWriter.java
index 28bc1f95f5..a3009d4510 100644
--- a/oak-segment-remote/src/main/java/org/apache/jackrabbit/oak/segment/remote/AbstractRemoteSegmentArchiveWriter.java
+++ b/oak-segment-remote/src/main/java/org/apache/jackrabbit/oak/segment/remote/AbstractRemoteSegmentArchiveWriter.java
@@ -150,6 +150,12 @@ public abstract class AbstractRemoteSegmentArchiveWriter implements SegmentArchi
}
}
+ @Override
+ public void flush(Runnable callback) throws IOException {
+ flush();
+ callback.run();
+ }
+
/**
* Writes a segment to the remote storage.
* @param indexEntry, the archive index entry to write
diff --git a/oak-segment-remote/src/main/java/org/apache/jackrabbit/oak/segment/remote/package-info.java b/oak-segment-remote/src/main/java/org/apache/jackrabbit/oak/segment/remote/package-info.java
new file mode 100644
index 0000000000..cc0da5e7a3
--- /dev/null
+++ b/oak-segment-remote/src/main/java/org/apache/jackrabbit/oak/segment/remote/package-info.java
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+@Version("1.0.0")
+package org.apache.jackrabbit.oak.segment.remote;
+
+import org.osgi.annotation.versioning.Version;
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriter.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriter.java
index 6c3b8b7ec1..3f9cb90892 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriter.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriter.java
@@ -183,7 +183,14 @@ class TarWriter implements Closeable {
}
if (doFlush) {
- archive.flush();
+ archive.flush(() -> {
+ try {
+ writeBinaryReferences();
+ writeGraph();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
}
}
}
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/SegmentArchiveWriter.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/SegmentArchiveWriter.java
index 8596e1bdd5..01a42637ed 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/SegmentArchiveWriter.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/SegmentArchiveWriter.java
@@ -19,6 +19,8 @@
package org.apache.jackrabbit.oak.segment.spi.persistence;
import java.io.IOException;
+import java.util.concurrent.Callable;
+import java.util.function.BiConsumer;
import org.apache.jackrabbit.oak.commons.Buffer;
import org.jetbrains.annotations.NotNull;
@@ -140,4 +142,18 @@ public interface SegmentArchiveWriter {
*/
@NotNull
String getName();
+
+ /**
+ * This method allows the caller to perform additional operations after {@link #flush()} has been finished.
+ * Can be used for remote segment store implementations to invoke {@link #writeBinaryReferences(byte[])} and {@link #writeGraph(byte[])}
+ * so that all necessary metadata is available for another Oak process concurrently accessing the storage.
+ *
+ * Default implementation invokes {@link #flush()} without invoking callback method afterwards
+ *
+ * @param onSuccess
+ * @throws IOException
+ */
+ default void flush(Runnable onSuccess) throws IOException {
+ flush();
+ }
}