svn commit: r1877731 [4/4] - in /jackrabbit/oak/trunk: ./ oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/ oak-it/ oak-it/src/test/java/org/apache/jackrabbit/oak/ oak-it/src/test/java/org/apache/jackrabbit/oak/spi/state/ oak-jcr/ oak-parent...

[ad...@apache.org]

Added: jackrabbit/oak/trunk/oak-segment-aws/src/test/java/org/apache/jackrabbit/oak/segment/spi/persistence/split/SplitPersistenceBlobTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-aws/src/test/java/org/apache/jackrabbit/oak/segment/spi/persistence/split/SplitPersistenceBlobTest.java?rev=1877731&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-aws/src/test/java/org/apache/jackrabbit/oak/segment/spi/persistence/split/SplitPersistenceBlobTest.java (added)
+++ jackrabbit/oak/trunk/oak-segment-aws/src/test/java/org/apache/jackrabbit/oak/segment/spi/persistence/split/SplitPersistenceBlobTest.java Thu May 14 11:11:12 2020
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.segment.spi.persistence.split;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.util.Date;
+import java.util.Random;
+import java.util.Set;
+
+import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
+import com.amazonaws.services.dynamodbv2.local.embedded.DynamoDBEmbedded;
+import com.amazonaws.services.s3.AmazonS3;
+import com.google.common.collect.Sets;
+import org.apache.jackrabbit.oak.api.Blob;
+import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.OakFileDataStore;
+import org.apache.jackrabbit.oak.segment.SegmentNodeStore;
+import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
+import org.apache.jackrabbit.oak.segment.aws.S3MockRule;
+import org.apache.jackrabbit.oak.segment.aws.AwsContext;
+import org.apache.jackrabbit.oak.segment.aws.AwsPersistence;
+import org.apache.jackrabbit.oak.segment.file.FileStore;
+import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
+import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException;
+import org.apache.jackrabbit.oak.segment.file.tar.TarPersistence;
+import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentNodeStorePersistence;
+import org.apache.jackrabbit.oak.spi.blob.BlobStore;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import static com.google.common.collect.Sets.newHashSet;
+import static org.junit.Assert.assertEquals;
+
+public class SplitPersistenceBlobTest {
+
+    @ClassRule
+    public static final S3MockRule s3Mock = new S3MockRule();
+
+    @Rule
+    public TemporaryFolder folder = new TemporaryFolder(new File("target"));
+
+    private SegmentNodeStore base;
+
+    private SegmentNodeStore split;
+
+    private FileStore baseFileStore;
+
+    private FileStore splitFileStore;
+
+    private String baseBlobId;
+
+    private SegmentNodeStorePersistence splitPersistence;
+
+    @Before
+    public void setup() throws IOException, InvalidFileStoreVersionException, CommitFailedException {
+        AmazonS3 s3 = s3Mock.createClient();
+        AmazonDynamoDB ddb = DynamoDBEmbedded.create().amazonDynamoDB();
+        long time = new Date().getTime();
+        AwsContext awsContext = AwsContext.create(s3, "bucket-" + time, "oak", ddb, "journaltable-" + time, "locktable-" + time);
+
+        SegmentNodeStorePersistence sharedPersistence = new AwsPersistence(awsContext);
+        
+        File dataStoreDir = new File(folder.getRoot(), "blobstore");
+        BlobStore blobStore = newBlobStore(dataStoreDir);
+
+        baseFileStore = FileStoreBuilder
+                .fileStoreBuilder(folder.newFolder())
+                .withCustomPersistence(sharedPersistence)
+                .withBlobStore(blobStore)
+                .build();
+        base = SegmentNodeStoreBuilders.builder(baseFileStore).build();
+
+        NodeBuilder builder = base.getRoot().builder();
+        builder.child("foo").child("bar").setProperty("version", "v1");
+        base.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+        baseBlobId = createLoad(base, baseFileStore).getContentIdentity();
+        baseFileStore.flush();
+        baseFileStore.close();
+
+        baseFileStore = FileStoreBuilder
+            .fileStoreBuilder(folder.newFolder())
+            .withCustomPersistence(sharedPersistence)
+            .withBlobStore(blobStore)
+            .build();
+        base = SegmentNodeStoreBuilders.builder(baseFileStore).build();
+
+        createLoad(base, baseFileStore).getContentIdentity();
+        baseFileStore.flush();
+
+        SegmentNodeStorePersistence localPersistence = new TarPersistence(folder.newFolder());
+        splitPersistence = new SplitPersistence(sharedPersistence, localPersistence);
+
+        splitFileStore = FileStoreBuilder
+            .fileStoreBuilder(folder.newFolder())
+            .withCustomPersistence(splitPersistence)
+            .withBlobStore(blobStore)
+            .build();
+        split = SegmentNodeStoreBuilders.builder(splitFileStore).build();
+    }
+
+    @After
+    public void tearDown() {
+        baseFileStore.close();
+    }
+
+    @Test
+    public void collectReferences()
+        throws IOException, CommitFailedException {
+        String blobId = createLoad(split, splitFileStore).getContentIdentity();
+
+        assertReferences(2, Sets.newHashSet(baseBlobId, blobId));
+    }
+
+    private static Blob createBlob(NodeStore nodeStore, int size) throws IOException {
+        byte[] data = new byte[size];
+        new Random().nextBytes(data);
+        return nodeStore.createBlob(new ByteArrayInputStream(data));
+    }
+
+    private static BlobStore newBlobStore(File directory) {
+        OakFileDataStore delegate = new OakFileDataStore();
+        delegate.setPath(directory.getAbsolutePath());
+        delegate.init(null);
+        return new DataStoreBlobStore(delegate);
+    }
+
+    private Blob createLoad(SegmentNodeStore store, FileStore fileStore)
+        throws IOException, CommitFailedException {
+        NodeBuilder builder = store.getRoot().builder();
+        Blob blob = createBlob(store, 18000);
+        builder.setProperty("bin", blob);
+        store.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+        fileStore.flush();
+        return blob;
+    }
+
+    private void assertReferences(int count, Set<String> blobIds)
+        throws IOException {
+        Set<String> actualReferences = newHashSet();
+        splitFileStore.collectBlobReferences(actualReferences::add);
+        assertEquals("visible references different", count, actualReferences.size());
+        assertEquals("Binary reference returned should be same", blobIds, actualReferences);
+    }
+}

Added: jackrabbit/oak/trunk/oak-segment-aws/src/test/java/org/apache/jackrabbit/oak/segment/spi/persistence/split/SplitPersistenceTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-aws/src/test/java/org/apache/jackrabbit/oak/segment/spi/persistence/split/SplitPersistenceTest.java?rev=1877731&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-aws/src/test/java/org/apache/jackrabbit/oak/segment/spi/persistence/split/SplitPersistenceTest.java (added)
+++ jackrabbit/oak/trunk/oak-segment-aws/src/test/java/org/apache/jackrabbit/oak/segment/spi/persistence/split/SplitPersistenceTest.java Thu May 14 11:11:12 2020
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.segment.spi.persistence.split;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Date;
+
+import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
+import com.amazonaws.services.dynamodbv2.local.embedded.DynamoDBEmbedded;
+import com.amazonaws.services.s3.AmazonS3;
+
+import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.segment.SegmentNodeStore;
+import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
+import org.apache.jackrabbit.oak.segment.aws.AwsContext;
+import org.apache.jackrabbit.oak.segment.aws.AwsPersistence;
+import org.apache.jackrabbit.oak.segment.aws.S3MockRule;
+import org.apache.jackrabbit.oak.segment.file.FileStore;
+import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
+import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException;
+import org.apache.jackrabbit.oak.segment.file.tar.TarPersistence;
+import org.apache.jackrabbit.oak.segment.file.tar.binaries.BinaryReferencesIndexLoader;
+import org.apache.jackrabbit.oak.segment.file.tar.binaries.InvalidBinaryReferencesIndexException;
+import org.apache.jackrabbit.oak.segment.spi.monitor.FileStoreMonitorAdapter;
+import org.apache.jackrabbit.oak.segment.spi.monitor.IOMonitorAdapter;
+import org.apache.jackrabbit.oak.segment.spi.monitor.RemoteStoreMonitorAdapter;
+import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentArchiveManager;
+import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentArchiveReader;
+import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentNodeStorePersistence;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+public class SplitPersistenceTest {
+
+    @ClassRule
+    public static final S3MockRule s3Mock = new S3MockRule();
+
+    @Rule
+    public TemporaryFolder folder = new TemporaryFolder(new File("target"));
+
+    private SegmentNodeStore base;
+
+    private SegmentNodeStore split;
+
+    private FileStore baseFileStore;
+
+    private FileStore splitFileStore;
+
+    private SegmentNodeStorePersistence splitPersistence;
+
+    @Before
+    public void setup() throws IOException, InvalidFileStoreVersionException, CommitFailedException {
+        AmazonS3 s3 = s3Mock.createClient();
+        AmazonDynamoDB ddb = DynamoDBEmbedded.create().amazonDynamoDB();
+        long time = new Date().getTime();
+        AwsContext awsContext = AwsContext.create(s3, "bucket-" + time, "oak", ddb, "journaltable-" + time, "locktable-" + time);
+
+        SegmentNodeStorePersistence sharedPersistence = new AwsPersistence(awsContext);
+
+        baseFileStore = FileStoreBuilder
+                .fileStoreBuilder(folder.newFolder())
+                .withCustomPersistence(sharedPersistence)
+                .build();
+        base = SegmentNodeStoreBuilders.builder(baseFileStore).build();
+
+        NodeBuilder builder = base.getRoot().builder();
+        builder.child("foo").child("bar").setProperty("version", "v1");
+        base.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+        baseFileStore.flush();
+
+        SegmentNodeStorePersistence localPersistence = new TarPersistence(folder.newFolder());
+        splitPersistence = new SplitPersistence(sharedPersistence, localPersistence);
+
+        splitFileStore = FileStoreBuilder
+                .fileStoreBuilder(folder.newFolder())
+                .withCustomPersistence(splitPersistence)
+                .build();
+        split = SegmentNodeStoreBuilders.builder(splitFileStore).build();
+    }
+
+    @After
+    public void tearDown() {
+        if (splitFileStore != null) {
+            splitFileStore.close();
+        }
+
+        if (baseFileStore != null) {
+            baseFileStore.close();
+        }
+    }
+
+    @Test
+    public void testBaseNodeAvailable() {
+        assertEquals("v1", split.getRoot().getChildNode("foo").getChildNode("bar").getString("version"));
+    }
+
+    @Test
+    public void testChangesAreLocalForBaseRepository() throws CommitFailedException {
+        NodeBuilder builder = base.getRoot().builder();
+        builder.child("foo").child("bar").setProperty("version", "v2");
+        base.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+        assertEquals("v1", split.getRoot().getChildNode("foo").getChildNode("bar").getString("version"));
+    }
+
+    @Test
+    public void testChangesAreLocalForSplitRepository() throws CommitFailedException {
+        NodeBuilder builder = split.getRoot().builder();
+        builder.child("foo").child("bar").setProperty("version", "v2");
+        split.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+        assertEquals("v1", base.getRoot().getChildNode("foo").getChildNode("bar").getString("version"));
+    }
+
+    @Test
+    public void testBinaryReferencesAreNotNull() throws IOException, InvalidBinaryReferencesIndexException {
+        splitFileStore.close();
+        splitFileStore = null;
+
+        SegmentArchiveManager manager = splitPersistence.createArchiveManager(true, false, new IOMonitorAdapter(), new FileStoreMonitorAdapter(), new RemoteStoreMonitorAdapter());
+        for (String archive : manager.listArchives()) {
+            SegmentArchiveReader reader = manager.open(archive);
+            BinaryReferencesIndexLoader.parseBinaryReferencesIndex(reader.getBinaryReferences());
+            reader.close();
+        }
+    }
+}

Modified: jackrabbit/oak/trunk/pom.xml
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/pom.xml?rev=1877731&r1=1877730&r2=1877731&view=diff
==============================================================================
--- jackrabbit/oak/trunk/pom.xml (original)
+++ jackrabbit/oak/trunk/pom.xml Thu May 14 11:11:12 2020
@@ -73,6 +73,7 @@
     <module>oak-examples</module>
     <module>oak-it</module>
     <module>oak-segment-tar</module>
+    <module>oak-segment-aws</module>
     <module>oak-segment-azure</module>
     <module>oak-benchmarks</module>
     <module>oak-search-elastic</module>