You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ad...@apache.org on 2019/01/31 16:17:15 UTC
svn commit: r1852612 - in /jackrabbit/oak/trunk/oak-segment-tar/src:
main/java/org/apache/jackrabbit/oak/segment/
main/java/org/apache/jackrabbit/oak/segment/file/
test/java/org/apache/jackrabbit/oak/segment/standby/
test/java/org/apache/jackrabbit/oak...
Author: adulceanu
Date: Thu Jan 31 16:17:14 2019
New Revision: 1852612
URL: http://svn.apache.org/viewvc?rev=1852612&view=rev
Log:
OAK-8006 - SegmentBlob#readLongBlobId might cause SegmentNotFoundException on standby
Added test case
Introduced eagerSegmentCaching option for the FileStore to allow caching segments right after creating them
Added:
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/StandbySegmentBlobTestIT.java (with props)
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreRegistrar.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/test/TemporaryFileStore.java
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java?rev=1852612&r1=1852611&r2=1852612&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java Thu Jan 31 16:17:14 2019
@@ -204,13 +204,7 @@ public class SegmentBlob extends Record
private static String readLongBlobId(Segment segment, int recordNumber) {
RecordId blobId = segment.readRecordId(recordNumber, 1);
- // if the blob id lives in the same segment, avoid reading again
- // the segment, as it will trigger an SNFE on standby, see OAK-8006
- if (blobId.getSegmentId().equals(segment.getSegmentId())) {
- return segment.readString(blobId.getRecordNumber());
- } else {
- return blobId.getSegment().readString(blobId.getRecordNumber());
- }
+ return blobId.getSegment().readString(blobId.getRecordNumber());
}
private List<RecordId> getBulkRecordIds() {
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreRegistrar.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreRegistrar.java?rev=1852612&r1=1852611&r2=1852612&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreRegistrar.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreRegistrar.java Thu Jan 31 16:17:14 2019
@@ -252,6 +252,7 @@ class SegmentNodeStoreRegistrar {
if (cfg.isStandbyInstance()) {
builder.withSnfeListener(IGNORE_SNFE);
+ builder.withEagerSegmentCaching(true);
}
FileStore store;
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java?rev=1852612&r1=1852611&r2=1852612&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java Thu Jan 31 16:17:14 2019
@@ -132,6 +132,8 @@ public class FileStore extends AbstractF
private final GarbageCollectionStrategy garbageCollectionStrategy = newGarbageCollectionStrategy();
+ private final boolean eagerSegmentCaching;
+
FileStore(final FileStoreBuilder builder) throws InvalidFileStoreVersionException, IOException {
super(builder);
@@ -197,6 +199,7 @@ public class FileStore extends AbstractF
);
this.snfeListener = builder.getSnfeListener();
+ this.eagerSegmentCaching = builder.getEagerSegmentCaching();
TimerStats flushTimer = statsProvider.getTimer("oak.segment.flush", METRICS_ONLY);
fileStoreScheduler.scheduleWithFixedDelay(format("TarMK flush [%s]", directory), 5, SECONDS, () -> {
@@ -528,6 +531,11 @@ public class FileStore extends AbstractF
}
segment = new Segment(tracker, segmentReader, id, data);
+
+ if (eagerSegmentCaching) {
+ segmentCache.putSegment(segment);
+ }
+
generation = segment.getGcGeneration();
references = readReferences(segment);
binaryReferences = readBinaryReferences(segment);
@@ -544,7 +552,7 @@ public class FileStore extends AbstractF
);
// Keep this data segment in memory as it's likely to be accessed soon.
- if (segment != null) {
+ if (!eagerSegmentCaching && segment != null) {
segmentCache.putSegment(segment);
}
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java?rev=1852612&r1=1852611&r2=1852612&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java Thu Jan 31 16:17:14 2019
@@ -137,6 +137,8 @@ public class FileStoreBuilder {
private boolean strictVersionCheck;
+ private boolean eagerSegmentCaching;
+
private boolean built;
/**
@@ -357,6 +359,19 @@ public class FileStoreBuilder {
return this;
}
+ /**
+ * Enable eager segment caching. This proves useful when segments need to
+ * be cached as soon as they are created, right before persisting them to disk.
+ * One such scenario is the cold standby, see OAK-8006.
+ *
+ * @param eagerSegmentCaching enables eager segment caching iff {@code true}.
+ * @return this instance
+ */
+ public FileStoreBuilder withEagerSegmentCaching(boolean eagerSegmentCaching) {
+ this.eagerSegmentCaching = eagerSegmentCaching;
+ return this;
+ }
+
public Backend buildProcBackend(AbstractFileStore fileStore) throws IOException {
return new FileStoreProcBackend(fileStore, persistence);
}
@@ -525,6 +540,10 @@ public class FileStoreBuilder {
return strictVersionCheck;
}
+ boolean getEagerSegmentCaching() {
+ return eagerSegmentCaching;
+ }
+
@Override
public String toString() {
return "FileStoreBuilder{" +
Added: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/StandbySegmentBlobTestIT.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/StandbySegmentBlobTestIT.java?rev=1852612&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/StandbySegmentBlobTestIT.java (added)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/StandbySegmentBlobTestIT.java Thu Jan 31 16:17:14 2019
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.segment.standby;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+import org.apache.commons.io.input.NullInputStream;
+import org.apache.jackrabbit.core.data.FileDataStore;
+import org.apache.jackrabbit.oak.api.Blob;
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.commons.junit.TemporaryPort;
+import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
+import org.apache.jackrabbit.oak.segment.SegmentTestConstants;
+import org.apache.jackrabbit.oak.segment.file.FileStore;
+import org.apache.jackrabbit.oak.segment.standby.client.StandbyClientSync;
+import org.apache.jackrabbit.oak.segment.standby.server.StandbyServerSync;
+import org.apache.jackrabbit.oak.segment.test.TemporaryBlobStore;
+import org.apache.jackrabbit.oak.segment.test.TemporaryFileStore;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+
+public class StandbySegmentBlobTestIT extends TestBase {
+
+ // `BLOB_SIZE` has to be chosen in such a way that is both:
+ //
+ // - Less than or equal to `minRecordLength` in the Blob Store. This is
+ // necessary in order to create a sufficiently long blob ID.
+ // - Greater than or equal to `Segment.BLOB_ID_SMALL_LIMIT` in order to save
+ // the blob ID as a large, external blob ID.
+ // - Greater than or equal to `Segment.MEDIUM_LIMIT`, otherwise the content
+ // of the binary will be written as a mere value record instead of a
+ // binary ID referencing an external binary.
+ //
+ // Since `Segment.MEDIUM_LIMIT` is the largest of the constants above, it is
+ // sufficient to set `BLOB_SIZE` to `Segment.MEDIUM_LIMIT`.
+
+ private static final int BLOB_SIZE = 2 * SegmentTestConstants.MEDIUM_LIMIT;
+
+ private TemporaryFolder folder = new TemporaryFolder(new File("target"));
+
+ private TemporaryBlobStore blobStore = new TemporaryBlobStore(folder) {
+
+ @Override
+ protected void configureDataStore(FileDataStore dataStore) {
+ dataStore.setMinRecordLength(BLOB_SIZE);
+ }
+
+ };
+
+ private TemporaryFileStore serverFileStore = new TemporaryFileStore(folder, blobStore, false);
+
+ private TemporaryFileStore clientFileStore = new TemporaryFileStore(folder, blobStore, true);
+
+ @Rule
+ public TemporaryPort serverPort = new TemporaryPort();
+
+ @Rule
+ public RuleChain chain = RuleChain.outerRule(folder)
+ .around(blobStore)
+ .around(serverFileStore)
+ .around(clientFileStore);
+
+ @Test
+ public void testSyncWithLongBlobId() throws Exception {
+ FileStore primary = serverFileStore.fileStore();
+ FileStore secondary = clientFileStore.fileStore();
+
+ NodeStore store = SegmentNodeStoreBuilders.builder(primary).build();
+ try (
+ StandbyServerSync serverSync = new StandbyServerSync(serverPort.getPort(), primary, MB);
+ StandbyClientSync clientSync = new StandbyClientSync(getServerHost(), serverPort.getPort(), secondary, false, getClientTimeout(), false, folder.newFolder())
+ ) {
+ serverSync.start();
+ addTestContent(store, "server");
+ primary.flush();
+
+ clientSync.run();
+
+ assertEquals(primary.getHead(), secondary.getHead());
+
+ PropertyState ps = secondary.getHead().getChildNode("root")
+ .getChildNode("server").getProperty("testBlob");
+ assertNotNull(ps);
+ assertEquals(Type.BINARY.tag(), ps.getType().tag());
+ Blob b = ps.getValue(Type.BINARY);
+ assertEquals(BLOB_SIZE, b.length());
+ }
+ }
+
+ private static void addTestContent(NodeStore store, String child) throws Exception {
+ NodeBuilder builder = store.getRoot().builder();
+ NodeBuilder content = builder.child(child);
+ content.setProperty("ts", System.currentTimeMillis());
+
+ Blob blob = store.createBlob(new NullInputStream(BLOB_SIZE));
+ content.setProperty("testBlob", blob);
+
+ store.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ }
+
+}
Propchange: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/StandbySegmentBlobTestIT.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/test/TemporaryFileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/test/TemporaryFileStore.java?rev=1852612&r1=1852611&r2=1852612&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/test/TemporaryFileStore.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/test/TemporaryFileStore.java Thu Jan 31 16:17:14 2019
@@ -36,7 +36,7 @@ public class TemporaryFileStore extends
private final TemporaryFolder folder;
private final TemporaryBlobStore blobStore;
-
+
private final boolean standby;
private ScheduledExecutorService executor;
@@ -62,7 +62,7 @@ public class TemporaryFileStore extends
.withMaxFileSize(1)
.withMemoryMapping(false)
.withNodeDeduplicationCacheSize(1)
- .withSegmentCacheSize(0)
+ .withSegmentCacheSize(256)
.withStringCacheSize(0)
.withTemplateCacheSize(0)
.withStatisticsProvider(new DefaultStatisticsProvider(executor));
@@ -71,7 +71,8 @@ public class TemporaryFileStore extends
.setRetainedGenerations(1);
builder
.withGCOptions(gcOptions)
- .withSnfeListener(SegmentNotFoundExceptionListener.IGNORE_SNFE);
+ .withSnfeListener(SegmentNotFoundExceptionListener.IGNORE_SNFE)
+ .withEagerSegmentCaching(true);
}
if (blobStore != null) {
builder.withBlobStore(blobStore.blobStore());