You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ad...@apache.org on 2021/03/05 11:10:17 UTC
svn commit: r1887194 - in /jackrabbit/oak/trunk/oak-segment-tar/src:
main/java/org/apache/jackrabbit/oak/backup/impl/
main/java/org/apache/jackrabbit/oak/segment/
main/java/org/apache/jackrabbit/oak/segment/file/
test/java/org/apache/jackrabbit/oak/seg...
Author: adulceanu
Date: Fri Mar 5 11:10:16 2021
New Revision: 1887194
URL: http://svn.apache.org/viewvc?rev=1887194&view=rev
Log:
OAK-6911 - Provide a way to tune inline size while storing binaries
Added:
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/BinariesInlineThresholdIT.java
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreBackupImpl.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreRestoreImpl.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriter.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterBuilder.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterTest.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessorTest.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/test/TemporaryFileStore.java
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreBackupImpl.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreBackupImpl.java?rev=1887194&r1=1887193&r2=1887194&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreBackupImpl.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreBackupImpl.java Fri Mar 5 11:10:16 2021
@@ -83,7 +83,8 @@ public class FileStoreBackupImpl impleme
backup.getSegmentIdProvider(),
backup.getBlobStore(),
new WriterCacheManager.Default(),
- bufferWriter
+ bufferWriter,
+ backup.getBinariesInlineThreshold()
);
ClassicCompactor compactor = new ClassicCompactor(
backup.getReader(),
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreRestoreImpl.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreRestoreImpl.java?rev=1887194&r1=1887193&r2=1887194&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreRestoreImpl.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreRestoreImpl.java Fri Mar 5 11:10:16 2021
@@ -79,7 +79,8 @@ public class FileStoreRestoreImpl implem
store.getSegmentIdProvider(),
store.getBlobStore(),
new WriterCacheManager.Default(),
- bufferWriter
+ bufferWriter,
+ store.getBinariesInlineThreshold()
);
SegmentGCOptions gcOptions = defaultGCOptions().setOffline();
ClassicCompactor compactor = new ClassicCompactor(
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriter.java?rev=1887194&r1=1887193&r2=1887194&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriter.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriter.java Fri Mar 5 11:10:16 2021
@@ -112,19 +112,23 @@ public class DefaultSegmentWriter implem
@NotNull
private final WriteOperationHandler writeOperationHandler;
+
+ private final int binariesInlineThreshold;
/**
* Create a new instance of a {@code SegmentWriter}. Note the thread safety
* properties pointed out in the class comment.
*
- * @param store store to write to
- * @param reader segment reader for the {@code store}
- * @param idProvider segment id provider for the {@code store}
- * @param blobStore the blog store or {@code null} for inlined
- * blobs
- * @param cacheManager cache manager instance for the
- * de-duplication caches used by this writer
- * @param writeOperationHandler handler for write operations.
+ * @param store store to write to
+ * @param reader segment reader for the {@code store}
+ * @param idProvider segment id provider for the {@code store}
+ * @param blobStore the blob store or {@code null} for inlined
+ * blobs
+ * @param cacheManager cache manager instance for the
+ * de-duplication caches used by this writer
+ * @param writeOperationHandler handler for write operations.
+ * @param binariesInlineThreshold threshold in bytes, specifying the limit up to which
+ * blobs will be inlined
*/
public DefaultSegmentWriter(
@NotNull SegmentStore store,
@@ -132,7 +136,8 @@ public class DefaultSegmentWriter implem
@NotNull SegmentIdProvider idProvider,
@Nullable BlobStore blobStore,
@NotNull WriterCacheManager cacheManager,
- @NotNull WriteOperationHandler writeOperationHandler
+ @NotNull WriteOperationHandler writeOperationHandler,
+ int binariesInlineThreshold
) {
this.store = checkNotNull(store);
this.reader = checkNotNull(reader);
@@ -140,6 +145,9 @@ public class DefaultSegmentWriter implem
this.blobStore = blobStore;
this.cacheManager = checkNotNull(cacheManager);
this.writeOperationHandler = checkNotNull(writeOperationHandler);
+ checkArgument(binariesInlineThreshold >= 0);
+ checkArgument(binariesInlineThreshold <= Segment.MEDIUM_LIMIT);
+ this.binariesInlineThreshold = binariesInlineThreshold;
}
@Override
@@ -638,11 +646,13 @@ public class DefaultSegmentWriter implem
}
private RecordId internalWriteStream(@NotNull InputStream stream) throws IOException {
- // Special case for short binaries (up to about 16kB):
+ // Special case for short binaries (up to about binariesInlineThreshold, 16kB by default):
// store them directly as small- or medium-sized value records
- byte[] data = new byte[Segment.MEDIUM_LIMIT];
+
+ byte[] data = new byte[binariesInlineThreshold];
int n = read(stream, data, 0, data.length);
- if (n < Segment.MEDIUM_LIMIT) {
+
+ if (n < binariesInlineThreshold) {
return writeValueRecord(n, data);
}
@@ -651,6 +661,17 @@ public class DefaultSegmentWriter implem
new ByteArrayInputStream(data, 0, n), stream));
return writeBlobId(blobId);
}
+
+ // handle case in which blob store is not configured and
+ // binariesInlineThreshold < Segment.MEDIUM_LIMIT
+ // store the binaries as small or medium sized value records
+
+ data = Arrays.copyOf(data, Segment.MEDIUM_LIMIT);
+ n += read(stream, data, n, Segment.MEDIUM_LIMIT - n);
+
+ if (n < Segment.MEDIUM_LIMIT) {
+ return writeValueRecord(n, data);
+ }
data = Arrays.copyOf(data, Segment.MAX_SEGMENT_SIZE);
n += read(stream, data, n, Segment.MAX_SEGMENT_SIZE - n);
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterBuilder.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterBuilder.java?rev=1887194&r1=1887193&r2=1887194&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterBuilder.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterBuilder.java Fri Mar 5 11:10:16 2021
@@ -151,7 +151,8 @@ public final class DefaultSegmentWriterB
store.getSegmentIdProvider(),
store.getBlobStore(),
cacheManager,
- createWriter(store, pooled)
+ createWriter(store, pooled),
+ store.getBinariesInlineThreshold()
);
}
@@ -187,7 +188,8 @@ public final class DefaultSegmentWriterB
public void flush(@NotNull SegmentStore store) {
throw new UnsupportedOperationException("Cannot write to read-only store");
}
- }
+ },
+ store.getBinariesInlineThreshold()
);
}
@@ -202,7 +204,8 @@ public final class DefaultSegmentWriterB
store.getSegmentIdProvider(),
store.getBlobStore(),
cacheManager,
- createWriter(store, pooled)
+ createWriter(store, pooled),
+ Segment.MEDIUM_LIMIT
);
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java?rev=1887194&r1=1887193&r2=1887194&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java Fri Mar 5 11:10:16 2021
@@ -112,8 +112,8 @@ public class Segment {
* value. And since small values are never stored as medium ones, we can
* extend the size range to cover that many longer values.
*/
- static final int MEDIUM_LIMIT = (1 << (16 - 2)) + SMALL_LIMIT;
-
+ public static final int MEDIUM_LIMIT = (1 << (16 - 2)) + SMALL_LIMIT;
+
/**
* Maximum size of small blob IDs. A small blob ID is stored in a value
* record whose length field contains the pattern "1110" in its most
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java?rev=1887194&r1=1887193&r2=1887194&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java Fri Mar 5 11:10:16 2021
@@ -130,6 +130,8 @@ public abstract class AbstractFileStore
protected final IOMonitor ioMonitor;
protected final RemoteStoreMonitor remoteStoreMonitor;
+
+ protected final int binariesInlineThreshold;
AbstractFileStore(final FileStoreBuilder builder) {
this.directory = builder.getDirectory();
@@ -153,6 +155,7 @@ public abstract class AbstractFileStore
this.ioMonitor = builder.getIOMonitor();
this.remoteStoreMonitor = builder.getRemoteStoreMonitor();
this.segmentBufferMonitor = new SegmentBufferMonitor(builder.getStatsProvider());
+ this.binariesInlineThreshold = builder.getBinariesInlineThreshold();
}
static SegmentNotFoundException asSegmentNotFoundException(Exception e, SegmentId id) {
@@ -189,6 +192,10 @@ public abstract class AbstractFileStore
public SegmentIdProvider getSegmentIdProvider() {
return tracker;
}
+
+ public int getBinariesInlineThreshold() {
+ return binariesInlineThreshold;
+ }
/**
* @return the {@link Revisions} object bound to the current store.
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java?rev=1887194&r1=1887193&r2=1887194&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java Fri Mar 5 11:10:16 2021
@@ -19,6 +19,7 @@
package org.apache.jackrabbit.oak.segment.file;
+import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.collect.Sets.newHashSet;
@@ -37,10 +38,12 @@ import java.io.IOException;
import java.util.Set;
import com.google.common.base.Predicate;
+
import org.apache.jackrabbit.oak.segment.CacheWeights.NodeCacheWeigher;
import org.apache.jackrabbit.oak.segment.CacheWeights.StringCacheWeigher;
import org.apache.jackrabbit.oak.segment.CacheWeights.TemplateCacheWeigher;
import org.apache.jackrabbit.oak.segment.RecordCache;
+import org.apache.jackrabbit.oak.segment.Segment;
import org.apache.jackrabbit.oak.segment.SegmentNotFoundExceptionListener;
import org.apache.jackrabbit.oak.segment.WriterCacheManager;
import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions;
@@ -66,7 +69,7 @@ import org.slf4j.LoggerFactory;
*/
public class FileStoreBuilder {
private static final Logger LOG = LoggerFactory.getLogger(FileStore.class);
-
+
private static final boolean MEMORY_MAPPING_DEFAULT =
"64".equals(System.getProperty("sun.arch.data.model", "32"));
@@ -95,6 +98,8 @@ public class FileStoreBuilder {
private boolean memoryMapping = MEMORY_MAPPING_DEFAULT;
private boolean offHeapAccess = getBoolean("access.off.heap");
+
+ private int binariesInlineThreshold = Segment.MEDIUM_LIMIT;
private SegmentNodeStorePersistence persistence;
@@ -396,6 +401,16 @@ public class FileStoreBuilder {
this.eagerSegmentCaching = eagerSegmentCaching;
return this;
}
+
+ /**
+ * Sets the threshold under which binaries are inlined in data segments.
+ * @param binariesInlineThreshold the threshold
+ * @return this instance
+ */
+ public FileStoreBuilder withBinariesInlineThreshold(int binariesInlineThreshold) {
+ this.binariesInlineThreshold = binariesInlineThreshold;
+ return this;
+ }
public Backend buildProcBackend(AbstractFileStore fileStore) throws IOException {
return new FileStoreProcBackend(fileStore, persistence);
@@ -574,6 +589,10 @@ public class FileStoreBuilder {
boolean getEagerSegmentCaching() {
return eagerSegmentCaching;
}
+
+ int getBinariesInlineThreshold() {
+ return binariesInlineThreshold;
+ }
@Override
public String toString() {
Added: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/BinariesInlineThresholdIT.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/BinariesInlineThresholdIT.java?rev=1887194&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/BinariesInlineThresholdIT.java (added)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/BinariesInlineThresholdIT.java Fri Mar 5 11:10:16 2021
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.segment;
+
+import static org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreBuilder;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.util.Random;
+
+import org.apache.jackrabbit.core.data.FileDataStore;
+import org.apache.jackrabbit.oak.api.Blob;
+import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
+import org.apache.jackrabbit.oak.segment.file.FileStore;
+import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
+import org.apache.jackrabbit.oak.spi.blob.BlobStore;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+public class BinariesInlineThresholdIT {
+ private static final int SMALL_BINARIES_INLINE_THRESHOLD = 1024;
+
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder(new File("target"));
+
+ private BlobStore createBlobStore(File dir) {
+ FileDataStore fds = new FileDataStore();
+ fds.setMinRecordLength(4096);
+ fds.init(dir.getAbsolutePath());
+ return new DataStoreBlobStore(fds);
+ }
+
+ private FileStore createFileStore(File dir, BlobStore blobStore, int binariesInlineThreshold) throws Throwable {
+ FileStoreBuilder builder = fileStoreBuilder(dir)
+ .withMaxFileSize(1)
+ .withMemoryMapping(false)
+ .withNodeDeduplicationCacheSize(1)
+ .withSegmentCacheSize(256)
+ .withStringCacheSize(0)
+ .withTemplateCacheSize(0)
+ .withBinariesInlineThreshold(binariesInlineThreshold);
+
+ if (blobStore != null) {
+ builder.withBlobStore(blobStore);
+ }
+
+ return builder.build();
+ }
+
+ @Test
+ public void testInlineBinaries() throws Throwable {
+ File fileStoreDir = folder.newFolder();
+ File blobStoreDir = folder.newFolder();
+
+ BlobStore blobStore = createBlobStore(blobStoreDir);
+
+ // set binaries inline threshold to default Segment.MEDIUM limit, i.e. 16512
+ // all binaries under this limit should be inlined, if over the limit they
+ // will be stored in the blobstore
+ FileStore fileStore = createFileStore(fileStoreDir, blobStore, Segment.MEDIUM_LIMIT);
+
+ SegmentNodeStore sns = SegmentNodeStoreBuilders.builder(fileStore).build();
+
+ Blob b1 = addTestContent(sns, "a", SMALL_BINARIES_INLINE_THRESHOLD - 1);
+ assertTrue(b1 instanceof SegmentBlob);
+ assertNull(((SegmentBlob) b1).getBlobId());
+ assertFalse(((SegmentBlob) b1).isExternal());
+ assertFalse(b1.isInlined());
+
+ Blob b2 = addTestContent(sns, "b", SMALL_BINARIES_INLINE_THRESHOLD);
+ assertTrue(b2 instanceof SegmentBlob);
+ assertNull(((SegmentBlob) b2).getBlobId());
+ assertFalse(((SegmentBlob) b2).isExternal());
+ assertFalse(b2.isInlined());
+
+ Blob b3 = addTestContent(sns, "c", Segment.MEDIUM_LIMIT - 1);
+ assertTrue(b3 instanceof SegmentBlob);
+ assertNull(((SegmentBlob) b3).getBlobId());
+ assertFalse(((SegmentBlob) b3).isExternal());
+ assertFalse(b3.isInlined());
+
+ Blob b4 = addTestContent(sns, "d", Segment.MEDIUM_LIMIT);
+ assertTrue(b4 instanceof SegmentBlob);
+ assertNotNull(b4.getReference());
+ assertEquals(b4.getContentIdentity(), ((SegmentBlob) b4).getBlobId());
+ assertFalse(b4.isInlined());
+
+ fileStore.close();
+
+ fileStore = createFileStore(fileStoreDir, blobStore, SMALL_BINARIES_INLINE_THRESHOLD);
+ sns = SegmentNodeStoreBuilders.builder(fileStore).build();
+
+ Blob b5 = addTestContent(sns, "e", SMALL_BINARIES_INLINE_THRESHOLD - 1);
+ assertTrue(b5 instanceof SegmentBlob);
+ assertNull(((SegmentBlob) b5).getBlobId());
+ assertFalse(((SegmentBlob) b5).isExternal());
+ assertFalse(b5.isInlined());
+
+ Blob b6 = addTestContent(sns, "f", SMALL_BINARIES_INLINE_THRESHOLD);
+ assertTrue(b6 instanceof SegmentBlob);
+ assertNull(b6.getReference());
+ assertEquals(b6.getContentIdentity(), ((SegmentBlob) b6).getBlobId());
+ assertTrue(b6.isInlined());
+
+ Blob b7 = addTestContent(sns, "g", Segment.MEDIUM_LIMIT - 1);
+ assertTrue(b7 instanceof SegmentBlob);
+ assertNotNull(b7.getReference());
+ assertEquals(b7.getContentIdentity(), ((SegmentBlob) b7).getBlobId());
+ assertFalse(b7.isInlined());
+
+ Blob b8 = addTestContent(sns, "h", Segment.MEDIUM_LIMIT);
+ assertTrue(b8 instanceof SegmentBlob);
+ assertNotNull(b8.getReference());
+ assertEquals(b8.getContentIdentity(), ((SegmentBlob) b8).getBlobId());
+ assertFalse(b8.isInlined());
+
+ fileStore.compactFull();
+
+ // b2 and b3 blobs should have ended now in blob store
+ // as a result of compaction rewriting the repository
+ // using SMALL_BINARIES_INLINE_THRESHOLD
+
+ b2 = getBlob(sns, "b");
+ assertTrue(b2 instanceof SegmentBlob);
+ assertNotNull(((SegmentBlob) b2).getBlobId());
+ assertTrue(((SegmentBlob) b2).isExternal());
+ assertTrue(b2.isInlined());
+
+ b3 = getBlob(sns, "c");
+ assertTrue(b3 instanceof SegmentBlob);
+ assertNotNull(b3.getReference());
+ assertEquals(b3.getContentIdentity(), ((SegmentBlob) b3).getBlobId());
+ assertFalse(b3.isInlined());
+
+ blobStore.close();
+ fileStore.close();
+ }
+
+ private Blob getBlob(SegmentNodeStore sns, String child) {
+ PropertyState ps = sns.getRoot().getChildNode(child).getProperty("testBlob");
+ return ps.getValue(Type.BINARY);
+ }
+
+ private Blob addTestContent(NodeStore store, String child, int size) throws CommitFailedException, IOException {
+ NodeBuilder builder = store.getRoot().builder();
+ builder.child(child).setProperty("ts", System.currentTimeMillis());
+
+ byte[] data = new byte[size];
+ new Random().nextBytes(data);
+ Blob blob = store.createBlob(new ByteArrayInputStream(data));
+
+ builder.child(child).setProperty("testBlob", blob);
+
+ store.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ return blob;
+ }
+}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterTest.java?rev=1887194&r1=1887193&r2=1887194&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterTest.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterTest.java Fri Mar 5 11:10:16 2021
@@ -29,10 +29,17 @@ import static org.junit.Assert.assertFal
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
-import com.google.common.base.Charsets;
-import com.google.common.base.Strings;
-import com.google.common.collect.ImmutableMap;
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import org.apache.commons.io.IOUtils;
import org.apache.jackrabbit.oak.segment.test.TemporaryFileStore;
import org.jetbrains.annotations.NotNull;
import org.junit.Before;
@@ -43,13 +50,9 @@ import org.junit.rules.TemporaryFolder;
import org.mockito.Mockito;
import org.slf4j.LoggerFactory;
-import java.io.File;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
+import com.google.common.base.Charsets;
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableMap;
import ch.qos.logback.classic.Level;
import ch.qos.logback.classic.LoggerContext;
@@ -63,9 +66,11 @@ public class DefaultSegmentWriterTest {
private final byte[] bytes = HELLO_WORLD.getBytes(Charsets.UTF_8);
+ private static final int SMALL_BINARIES_INLINE_THRESHOLD = 4;
+
private TemporaryFolder folder = new TemporaryFolder(new File("target"));
- private TemporaryFileStore store = new TemporaryFileStore(folder, false);
+ private TemporaryFileStore store = new TemporaryFileStore(folder, SMALL_BINARIES_INLINE_THRESHOLD);
@Rule
public RuleChain rules = RuleChain.outerRule(folder).around(store);
@@ -78,6 +83,14 @@ public class DefaultSegmentWriterTest {
}
@Test
+ public void testValueRecord() throws IOException {
+ InputStream stream = new ByteArrayInputStream(bytes);
+ RecordId valueId = writer.writeStream(stream);
+ SegmentBlob blob = new SegmentBlob(null, valueId);
+ assertEquals(HELLO_WORLD, IOUtils.toString(blob.getNewStream(), Charsets.UTF_8));
+ }
+
+ @Test
public void testBlockRecord() throws IOException {
RecordId blockId = writer.writeBlock(bytes, 0, bytes.length);
BlockRecord block = new BlockRecord(blockId, bytes.length);
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java?rev=1887194&r1=1887193&r2=1887194&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java Fri Mar 5 11:10:16 2021
@@ -98,6 +98,8 @@ public class SegmentDataStoreBlobGCIT {
private DataStoreBlobStore blobStore;
private SegmentGCOptions gcOptions = defaultGCOptions();
+
+ private int binariesInlineThreshold = Segment.MEDIUM_LIMIT;
@Rule
public TemporaryFolder folder = new TemporaryFolder(new File("target"));
@@ -152,13 +154,13 @@ public class SegmentDataStoreBlobGCIT {
NodeBuilder a = nodeStore.getRoot().builder();
- /* Create garbage by creating in-lined blobs (size < 16KB) */
+ /* Create garbage by creating in-lined blobs (size < binaries inline threshold) */
int number = 500;
NodeBuilder content = a.child("content");
for (int i = 0; i < number; i++) {
NodeBuilder c = content.child("x" + i);
for (int j = 0; j < 5; j++) {
- c.setProperty("p" + j, nodeStore.createBlob(randomStream(j, 16384)));
+ c.setProperty("p" + j, nodeStore.createBlob(randomStream(j, binariesInlineThreshold - 1)));
}
}
nodeStore.merge(a, EmptyHook.INSTANCE, CommitInfo.EMPTY);
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessorTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessorTest.java?rev=1887194&r1=1887193&r2=1887194&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessorTest.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessorTest.java Fri Mar 5 11:10:16 2021
@@ -52,6 +52,8 @@ public class RemoteBlobProcessorTest {
};
private TemporaryFileStore fileStore = new TemporaryFileStore(folder, blobStore, false);
+
+ private int binariesInlineThreshold = SegmentTestConstants.MEDIUM_LIMIT;
@Rule
public RuleChain rules = RuleChain.outerRule(folder)
@@ -104,7 +106,7 @@ public class RemoteBlobProcessorTest {
SegmentNodeStore store = SegmentNodeStoreBuilders.builder(fileStore.fileStore()).build();
NodeBuilder root = store.getRoot().builder();
- root.setProperty("b", root.createBlob(new NullInputStream(SegmentTestConstants.MEDIUM_LIMIT - 1)));
+ root.setProperty("b", root.createBlob(new NullInputStream(binariesInlineThreshold - 1)));
store.merge(root, EmptyHook.INSTANCE, CommitInfo.EMPTY);
RemoteBlobProcessor processor = new RemoteBlobProcessor(blobStore.blobStore(), blobId -> {
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/test/TemporaryFileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/test/TemporaryFileStore.java?rev=1887194&r1=1887193&r2=1887194&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/test/TemporaryFileStore.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/test/TemporaryFileStore.java Fri Mar 5 11:10:16 2021
@@ -23,6 +23,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser;
+import org.apache.jackrabbit.oak.segment.Segment;
import org.apache.jackrabbit.oak.segment.SegmentNotFoundExceptionListener;
import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions;
import org.apache.jackrabbit.oak.segment.file.FileStore;
@@ -43,6 +44,8 @@ public class TemporaryFileStore extends
private FileStore store;
+ private int binariesInlineThreshold = Segment.MEDIUM_LIMIT;
+
public TemporaryFileStore(TemporaryFolder folder, boolean standby) {
this.folder = folder;
this.standby = standby;
@@ -55,12 +58,20 @@ public class TemporaryFileStore extends
this.standby = standby;
}
+ public TemporaryFileStore(TemporaryFolder folder, int binariesInlineThreshold) {
+ this.folder = folder;
+ this.standby = false;
+ this.blobStore = null;
+ this.binariesInlineThreshold = binariesInlineThreshold;
+ }
+
@Override
protected void before() throws Throwable {
executor = Executors.newSingleThreadScheduledExecutor();
FileStoreBuilder builder = fileStoreBuilder(folder.newFolder())
.withMaxFileSize(1)
.withMemoryMapping(false)
+ .withBinariesInlineThreshold(binariesInlineThreshold)
.withNodeDeduplicationCacheSize(1)
.withSegmentCacheSize(256)
.withStringCacheSize(0)