You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by md...@apache.org on 2018/12/05 09:34:27 UTC
svn commit: r1848193 - in /jackrabbit/oak/branches/1.6: ./
oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/
oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/
oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment...
Author: mduerig
Date: Wed Dec 5 09:34:27 2018
New Revision: 1848193
URL: http://svn.apache.org/viewvc?rev=1848193&view=rev
Log:
OAK-7909: Backport and validate OAK-7867 to Oak 1.6
Merged r1845405
Modified:
jackrabbit/oak/branches/1.6/ (props changed)
jackrabbit/oak/branches/1.6/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java
jackrabbit/oak/branches/1.6/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriterPool.java
jackrabbit/oak/branches/1.6/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java
jackrabbit/oak/branches/1.6/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriterBuilder.java
jackrabbit/oak/branches/1.6/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/WriteOperationHandler.java
jackrabbit/oak/branches/1.6/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriterPoolTest.java
jackrabbit/oak/branches/1.6/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreIT.java
Propchange: jackrabbit/oak/branches/1.6/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 5 09:34:27 2018
@@ -1,4 +1,4 @@
/jackrabbit/oak/branches/1.0:1665962
/jackrabbit/oak/branches/1.8:1844835
-/jackrabbit/oak/trunk:1781068,1781075,1781248,1781386,1781846,1781907,1782000,1782029,1782196,1782447,1782476,1782770,1782945,1782966,1782973,1782990,1783061,1783066,1783089,1783104-1783105,1783110,1783619,1783720,1783731,1783733,1783738,1783742,1783773,1783855,1783891,1784023,1784034,1784130,1784162,1784251,1784401,1784551,1784574,1784689,1785095,1785108,1785161,1785172,1785283,1785652,1785838,1785916-1785917,1785919,1785946,1786122,1787074,1787145,1787151,1787217,1787425,1788056,1788378,1788387-1788389,1788463,1788476,1788850,1789056,1789534,1789925,1789940,1789987,1790006,1790013,1790069,1790077,1790079,1790382,1790502-1790503,1792049,1792463,1792742,1792746,1793013,1793088,1793618,1793627,1793644,1794393,1794417,1794683,1795138,1795314,1795330,1795475,1795488,1795491,1795502,1795594,1795613,1795618,1796144,1796230,1796239,1796274,1796278,1796988,1797378,1798035,1798832,1798834,1799219,1799389,1799393,1799924,1800244,1800269,1800606,1800613,1800974,1801011,1801013,1801118-1801119
,1801675,1802260,1802262,1802286,1802548,1802905,1802934,1802938,1802973,1803026,1803247-1803249,1803951,1803953-1803955,1804437,1805851-1805852,1806668,1807308,1807688,1808022,1808053,1808125,1808128,1808142,1808240,1808246,1808731,1809024,1809026,1809131,1809163,1809178-1809179,1809253,1809255-1809256,1809289,1809745,1811071-1811072,1811155,1811380,1811655,1811952,1811963,1811986,1813192,1813538,1814189,1814332,1814397,1814475,1815201,1815438,1815926,1817326,1817919,1817987-1817988,1817990,1818038,1818042,1818056,1818124,1818137,1818554,1818576,1818645,1819048,1819050,1821237,1821325,1821358,1821495,1821516,1821847,1822207,1822850,1823172,1823655,1824896,1825471,1825654,1826237,1826338,1826532,1826640,1826932,1826957,1827472,1827486,1827977,1828502,1829527,1829569,1829587,1829665,1829854,1829864,1829987,1829998,1830019,1830160,1830239,1830748,1831190,1831374,1832258,1832379,1832535,1833308,1834112,1834648-1834649,1834681,1835060,1836082,1837475,1837998,1838637,1839746,1840024,1840
226,1842677,1843175,1843222,1843231,1844549,1844642,1844728,1845135,1845730-1845731,1845863,1845865,1846057,1846617
+/jackrabbit/oak/trunk:1781068,1781075,1781248,1781386,1781846,1781907,1782000,1782029,1782196,1782447,1782476,1782770,1782945,1782966,1782973,1782990,1783061,1783066,1783089,1783104-1783105,1783110,1783619,1783720,1783731,1783733,1783738,1783742,1783773,1783855,1783891,1784023,1784034,1784130,1784162,1784251,1784401,1784551,1784574,1784689,1785095,1785108,1785161,1785172,1785283,1785652,1785838,1785916-1785917,1785919,1785946,1786122,1787074,1787145,1787151,1787217,1787425,1788056,1788378,1788387-1788389,1788463,1788476,1788850,1789056,1789534,1789925,1789940,1789987,1790006,1790013,1790069,1790077,1790079,1790382,1790502-1790503,1792049,1792463,1792742,1792746,1793013,1793088,1793618,1793627,1793644,1794393,1794417,1794683,1795138,1795314,1795330,1795475,1795488,1795491,1795502,1795594,1795613,1795618,1796144,1796230,1796239,1796274,1796278,1796988,1797378,1798035,1798832,1798834,1799219,1799389,1799393,1799924,1800244,1800269,1800606,1800613,1800974,1801011,1801013,1801118-1801119
,1801675,1802260,1802262,1802286,1802548,1802905,1802934,1802938,1802973,1803026,1803247-1803249,1803951,1803953-1803955,1804437,1805851-1805852,1806668,1807308,1807688,1808022,1808053,1808125,1808128,1808142,1808240,1808246,1808731,1809024,1809026,1809131,1809163,1809178-1809179,1809253,1809255-1809256,1809289,1809745,1811071-1811072,1811155,1811380,1811655,1811952,1811963,1811986,1813192,1813538,1814189,1814332,1814397,1814475,1815201,1815438,1815926,1817326,1817919,1817987-1817988,1817990,1818038,1818042,1818056,1818124,1818137,1818554,1818576,1818645,1819048,1819050,1821237,1821325,1821358,1821495,1821516,1821847,1822207,1822850,1823172,1823655,1824896,1825471,1825654,1826237,1826338,1826532,1826640,1826932,1826957,1827472,1827486,1827977,1828502,1829527,1829569,1829587,1829665,1829854,1829864,1829987,1829998,1830019,1830160,1830239,1830748,1831190,1831374,1832258,1832379,1832535,1833308,1834112,1834648-1834649,1834681,1835060,1836082,1837475,1837998,1838637,1839746,1840024,1840
226,1842677,1843175,1843222,1843231,1844549,1844642,1844728,1845135,1845405,1845730-1845731,1845863,1845865,1846057,1846617
/jackrabbit/trunk:1345480
Modified: jackrabbit/oak/branches/1.6/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java?rev=1848193&r1=1848192&r2=1848193&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.6/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java (original)
+++ jackrabbit/oak/branches/1.6/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java Wed Dec 5 09:34:27 2018
@@ -173,11 +173,15 @@ public class SegmentBufferWriter impleme
@Nonnull
@Override
- public RecordId execute(@Nonnull WriteOperation writeOperation) throws IOException {
+ public RecordId execute(int generation,
+ @Nonnull WriteOperation writeOperation)
+ throws IOException {
+ checkState(generation == this.generation);
return writeOperation.execute(this);
}
- int getGeneration() {
+ @Override
+ public int getGeneration() {
return generation;
}
Modified: jackrabbit/oak/branches/1.6/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriterPool.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriterPool.java?rev=1848193&r1=1848192&r2=1848193&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.6/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriterPool.java (original)
+++ jackrabbit/oak/branches/1.6/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriterPool.java Wed Dec 5 09:34:27 2018
@@ -27,6 +27,7 @@ import static com.google.common.collect.
import static java.lang.Thread.currentThread;
import java.io.IOException;
+import java.util.AbstractMap.SimpleImmutableEntry;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -39,7 +40,7 @@ import com.google.common.util.concurrent
/**
* This {@link WriteOperationHandler} uses a pool of {@link SegmentBufferWriter}s,
- * which it passes to its {@link #execute(WriteOperation) execute} method.
+ * which it passes to its {@link #execute(int, WriteOperation) execute} method.
* <p>
* Instances of this class are thread safe. See also the class comment of
* {@link SegmentWriter}.
@@ -103,14 +104,22 @@ public class SegmentBufferWriterPool imp
this.gcGeneration = checkNotNull(gcGeneration);
}
+ @Override
+ public int getGeneration() {
+ return gcGeneration.get();
+ }
+
@Nonnull
@Override
- public RecordId execute(@Nonnull WriteOperation writeOperation) throws IOException {
- SegmentBufferWriter writer = borrowWriter(currentThread());
+ public RecordId execute(int generation,
+ @Nonnull WriteOperation writeOperation)
+ throws IOException {
+ SimpleImmutableEntry<?,?> key = new SimpleImmutableEntry<>(currentThread(), generation);
+ SegmentBufferWriter writer = borrowWriter(key);
try {
return writeOperation.execute(writer);
} finally {
- returnWriter(currentThread(), writer);
+ returnWriter(key, writer);
}
}
@@ -185,7 +194,7 @@ public class SegmentBufferWriterPool imp
monitor.enterWhen(guard);
return true;
} catch (InterruptedException ignore) {
- Thread.currentThread().interrupt();
+ currentThread().interrupt();
return false;
}
}
@@ -203,15 +212,6 @@ public class SegmentBufferWriterPool imp
writer = new SegmentBufferWriter(
store,
tracker.getSegmentCounter(),
- reader,
- getWriterId(wid),
- gcGeneration.get()
- );
- } else if (writer.getGeneration() != gcGeneration.get()) {
- disposedOldGen.add(writer);
- writer = new SegmentBufferWriter(
- store,
- tracker.getSegmentCounter(),
reader,
getWriterId(wid),
gcGeneration.get()
Modified: jackrabbit/oak/branches/1.6/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java?rev=1848193&r1=1848192&r2=1848193&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.6/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java (original)
+++ jackrabbit/oak/branches/1.6/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java Wed Dec 5 09:34:27 2018
@@ -67,6 +67,7 @@ import org.apache.jackrabbit.oak.api.Pro
import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.api.jmx.CacheStatsMBean;
import org.apache.jackrabbit.oak.plugins.memory.ModifiedNodeState;
+import org.apache.jackrabbit.oak.segment.RecordWriters.RecordWriter;
import org.apache.jackrabbit.oak.segment.WriteOperationHandler.WriteOperation;
import org.apache.jackrabbit.oak.segment.file.GCNodeWriteMonitor;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
@@ -176,16 +177,10 @@ public class SegmentWriter {
* @throws IOException
*/
@Nonnull
- public MapRecord writeMap(@Nullable final MapRecord base,
- @Nonnull final Map<String, RecordId> changes)
+ public MapRecord writeMap(@Nullable final MapRecord base, @Nonnull final Map<String, RecordId> changes)
throws IOException {
- RecordId mapId = writeOperationHandler.execute(new SegmentWriteOperation() {
- @Nonnull
- @Override
- public RecordId execute(@Nonnull SegmentBufferWriter writer) throws IOException {
- return with(writer).writeMap(base, changes);
- }
- });
+ RecordId mapId = new SegmentWriteOperation(writeOperationHandler.getGeneration())
+ .writeMap(base, changes);
return new MapRecord(reader, mapId);
}
@@ -197,13 +192,8 @@ public class SegmentWriter {
*/
@Nonnull
public RecordId writeList(@Nonnull final List<RecordId> list) throws IOException {
- return writeOperationHandler.execute(new SegmentWriteOperation() {
- @Nonnull
- @Override
- public RecordId execute(@Nonnull SegmentBufferWriter writer) throws IOException {
- return with(writer).writeList(list);
- }
- });
+ return new SegmentWriteOperation(writeOperationHandler.getGeneration())
+ .writeList(list);
}
/**
@@ -214,13 +204,8 @@ public class SegmentWriter {
*/
@Nonnull
public RecordId writeString(@Nonnull final String string) throws IOException {
- return writeOperationHandler.execute(new SegmentWriteOperation() {
- @Nonnull
- @Override
- public RecordId execute(@Nonnull SegmentBufferWriter writer) throws IOException {
- return with(writer).writeString(string);
- }
- });
+ return new SegmentWriteOperation(writeOperationHandler.getGeneration())
+ .writeString(string);
}
/**
@@ -231,13 +216,8 @@ public class SegmentWriter {
*/
@Nonnull
public SegmentBlob writeBlob(@Nonnull final Blob blob) throws IOException {
- RecordId blobId = writeOperationHandler.execute(new SegmentWriteOperation() {
- @Nonnull
- @Override
- public RecordId execute(@Nonnull SegmentBufferWriter writer) throws IOException {
- return with(writer).writeBlob(blob);
- }
- });
+ RecordId blobId = new SegmentWriteOperation(writeOperationHandler.getGeneration())
+ .writeBlob(blob);
return new SegmentBlob(blobStore, blobId);
}
@@ -251,14 +231,9 @@ public class SegmentWriter {
*/
@Nonnull
public RecordId writeBlock(@Nonnull final byte[] bytes, final int offset, final int length)
- throws IOException {
- return writeOperationHandler.execute(new SegmentWriteOperation() {
- @Nonnull
- @Override
- public RecordId execute(@Nonnull SegmentBufferWriter writer) throws IOException {
- return with(writer).writeBlock(bytes, offset, length);
- }
- });
+ throws IOException {
+ return new SegmentWriteOperation(writeOperationHandler.getGeneration())
+ .writeBlock(bytes, offset, length);
}
/**
@@ -271,13 +246,8 @@ public class SegmentWriter {
*/
@Nonnull
public SegmentBlob writeStream(@Nonnull final InputStream stream) throws IOException {
- RecordId blobId = writeOperationHandler.execute(new SegmentWriteOperation() {
- @Nonnull
- @Override
- public RecordId execute(@Nonnull SegmentBufferWriter writer) throws IOException {
- return with(writer).writeStream(stream);
- }
- });
+ RecordId blobId = new SegmentWriteOperation(writeOperationHandler.getGeneration())
+ .writeStream(stream);
return new SegmentBlob(blobStore, blobId);
}
@@ -288,15 +258,9 @@ public class SegmentWriter {
* @throws IOException
*/
@Nonnull
- public SegmentPropertyState writeProperty(@Nonnull final PropertyState state)
- throws IOException {
- RecordId id = writeOperationHandler.execute(new SegmentWriteOperation() {
- @Nonnull
- @Override
- public RecordId execute(@Nonnull SegmentBufferWriter writer) throws IOException {
- return with(writer).writeProperty(state);
- }
- });
+ public SegmentPropertyState writeProperty(@Nonnull final PropertyState state) throws IOException {
+ RecordId id = new SegmentWriteOperation(writeOperationHandler.getGeneration())
+ .writeProperty(state);
return new SegmentPropertyState(reader, id, state.getName(), state.getType());
}
@@ -315,13 +279,8 @@ public class SegmentWriter {
*/
@Nonnull
public SegmentNodeState writeNode(@Nonnull final NodeState state) throws IOException {
- RecordId nodeId = writeOperationHandler.execute(new SegmentWriteOperation() {
- @Nonnull
- @Override
- public RecordId execute(@Nonnull SegmentBufferWriter writer) throws IOException {
- return with(writer).writeNode(state);
- }
- });
+ RecordId nodeId = new SegmentWriteOperation(writeOperationHandler.getGeneration())
+ .writeNode(state);
return new SegmentNodeState(reader, this, nodeId);
}
@@ -343,13 +302,8 @@ public class SegmentWriter {
public SegmentNodeState writeNode(@Nonnull final NodeState state, @Nonnull Supplier<Boolean> cancel)
throws IOException {
try {
- RecordId nodeId = writeOperationHandler.execute(new SegmentWriteOperation(cancel) {
- @Nonnull
- @Override
- public RecordId execute(@Nonnull SegmentBufferWriter writer) throws IOException {
- return with(writer).writeNode(state);
- }
- });
+ RecordId nodeId = new SegmentWriteOperation(writeOperationHandler.getGeneration(), cancel)
+ .writeNode(state);
return new SegmentNodeState(reader, this, nodeId);
} catch (SegmentWriteOperation.CancelledWriteException ignore) {
return null;
@@ -362,7 +316,7 @@ public class SegmentWriter {
* as a separate argument (a poor mans monad). As such it is entirely
* <em>not thread safe</em>.
*/
- private abstract class SegmentWriteOperation implements WriteOperation {
+ private class SegmentWriteOperation {
private class NodeWriteStats {
/*
* Total number of nodes in the subtree rooted at the node passed
@@ -427,32 +381,31 @@ public class SegmentWriter {
@CheckForNull
private NodeWriteStats nodeWriteStats;
- private SegmentBufferWriter writer;
- private RecordCache<String> stringCache;
- private RecordCache<Template> templateCache;
- private NodeCache nodeCache;
+ private final int generation;
+ private final RecordCache<String> stringCache;
+ private final RecordCache<Template> templateCache;
+ private final NodeCache nodeCache;
- protected SegmentWriteOperation(@Nonnull Supplier<Boolean> cancel) {
+ SegmentWriteOperation(int generation, @Nonnull Supplier<Boolean> cancel) {
this.cancel = cancel;
+ this.generation = generation;
+ this.stringCache = cacheManager.getStringCache(generation);
+ this.templateCache = cacheManager.getTemplateCache(generation);
+ this.nodeCache = cacheManager.getNodeCache(generation);
}
- protected SegmentWriteOperation() {
- this(Suppliers.ofInstance(false));
+ SegmentWriteOperation(int generation) {
+ this(generation, Suppliers.ofInstance(false));
}
- @Nonnull
- @Override
- public abstract RecordId execute(@Nonnull SegmentBufferWriter writer) throws IOException;
-
- @Nonnull
- SegmentWriteOperation with(@Nonnull SegmentBufferWriter writer) {
- checkState(this.writer == null);
- this.writer = writer;
- int generation = writer.getGeneration();
- this.stringCache = cacheManager.getStringCache(generation);
- this.templateCache = cacheManager.getTemplateCache(generation);
- this.nodeCache = cacheManager.getNodeCache(generation);
- return this;
+ private WriteOperation newWriteOperation(final RecordWriter recordWriter) {
+ return new WriteOperation() {
+ @Nonnull
+ @Override
+ public RecordId execute(@Nonnull SegmentBufferWriter writer) throws IOException {
+ return recordWriter.write(writer);
+ }
+ };
}
private RecordId writeMap(@Nullable MapRecord base,
@@ -478,8 +431,10 @@ public class SegmentWriter {
if (value.equals(entry.getValue())) {
return base.getRecordId();
} else {
- return RecordWriters.newMapBranchWriter(entry.getHash(), asList(entry.getKey(),
- value, base.getRecordId())).write(writer);
+ return writeOperationHandler.execute(generation, newWriteOperation(
+ RecordWriters.newMapBranchWriter(
+ entry.getHash(),
+ asList(entry.getKey(), value, base.getRecordId()))));
}
}
}
@@ -513,7 +468,8 @@ public class SegmentWriter {
checkElementIndex(size, MapRecord.MAX_SIZE);
checkPositionIndex(level, MapRecord.MAX_NUMBER_OF_LEVELS);
checkArgument(size != 0 || level == MapRecord.MAX_NUMBER_OF_LEVELS);
- return RecordWriters.newMapLeafWriter(level, entries).write(writer);
+ return writeOperationHandler.execute(generation, newWriteOperation(
+ RecordWriters.newMapLeafWriter(level, entries)));
}
private RecordId writeMapBranch(int level, int size, MapRecord... buckets) throws IOException {
@@ -525,7 +481,8 @@ public class SegmentWriter {
bucketIds.add(buckets[i].getRecordId());
}
}
- return RecordWriters.newMapBranchWriter(level, size, bitmap, bucketIds).write(writer);
+ return writeOperationHandler.execute(generation, newWriteOperation(
+ RecordWriters.newMapBranchWriter(level, size, bitmap, bucketIds)));
}
private RecordId writeMapBucket(MapRecord base, Collection<MapEntry> entries, int level)
@@ -535,7 +492,8 @@ public class SegmentWriter {
if (base != null) {
return base.getRecordId();
} else if (level == 0) {
- return RecordWriters.newMapLeafWriter().write(writer);
+ return writeOperationHandler.execute(generation, newWriteOperation(
+ RecordWriters.newMapLeafWriter()));
} else {
return null;
}
@@ -644,7 +602,8 @@ public class SegmentWriter {
private RecordId writeListBucket(List<RecordId> bucket) throws IOException {
checkArgument(bucket.size() > 1);
- return RecordWriters.newListBucketWriter(bucket).write(writer);
+ return writeOperationHandler.execute(generation, newWriteOperation(
+ RecordWriters.newListBucketWriter(bucket)));
}
private List<List<MapEntry>> splitToBuckets(Collection<MapEntry> entries, int level) {
@@ -667,12 +626,14 @@ public class SegmentWriter {
private RecordId writeValueRecord(long length, RecordId blocks) throws IOException {
long len = (length - Segment.MEDIUM_LIMIT) | (0x3L << 62);
- return RecordWriters.newValueWriter(blocks, len).write(writer);
+ return writeOperationHandler.execute(generation, newWriteOperation(
+ RecordWriters.newValueWriter(blocks, len)));
}
private RecordId writeValueRecord(int length, byte... data) throws IOException {
checkArgument(length < Segment.MEDIUM_LIMIT);
- return RecordWriters.newValueWriter(length, data).write(writer);
+ return writeOperationHandler.execute(generation, newWriteOperation(
+ RecordWriters.newValueWriter(length, data)));
}
/**
@@ -769,23 +730,22 @@ public class SegmentWriter {
private RecordId writeBlobId(String blobId) throws IOException {
byte[] data = blobId.getBytes(UTF_8);
- RecordId recordId;
-
if (data.length < Segment.BLOB_ID_SMALL_LIMIT) {
- recordId = RecordWriters.newBlobIdWriter(data).write(writer);
+ return writeOperationHandler.execute(generation, newWriteOperation(
+ RecordWriters.newBlobIdWriter(data)));
} else {
RecordId refId = writeString(blobId);
- recordId = RecordWriters.newBlobIdWriter(refId).write(writer);
+ return writeOperationHandler.execute(generation, newWriteOperation(
+ RecordWriters.newBlobIdWriter(refId)));
}
-
- return recordId;
}
private RecordId writeBlock(@Nonnull byte[] bytes, int offset, int length)
throws IOException {
checkNotNull(bytes);
checkPositionIndexes(offset, offset + length, bytes.length);
- return RecordWriters.newBlockWriter(bytes, offset, length).write(writer);
+ return writeOperationHandler.execute(generation, newWriteOperation(
+ RecordWriters.newBlockWriter(bytes, offset, length)));
}
private RecordId writeStream(@Nonnull InputStream stream) throws IOException {
@@ -884,9 +844,12 @@ public class SegmentWriter {
if (!type.isArray()) {
return valueIds.iterator().next();
} else if (count == 0) {
- return RecordWriters.newListWriter().write(writer);
+ return writeOperationHandler.execute(generation, newWriteOperation(
+ RecordWriters.newListWriter()));
} else {
- return RecordWriters.newListWriter(count, writeList(valueIds)).write(writer);
+ RecordId lid = writeList(valueIds);
+ return writeOperationHandler.execute(generation, newWriteOperation(
+ RecordWriters.newListWriter(count, lid)));
}
}
@@ -957,9 +920,10 @@ public class SegmentWriter {
checkState(propertyNames.length < (1 << 18));
head |= propertyNames.length;
- RecordId tid = RecordWriters.newTemplateWriter(ids, propertyNames,
- propertyTypes, head, primaryId, mixinIds, childNameId,
- propNamesId).write(writer);
+ RecordId tid = writeOperationHandler.execute(generation, newWriteOperation(
+ RecordWriters.newTemplateWriter(
+ ids, propertyNames, propertyTypes, head, primaryId, mixinIds,
+ childNameId, propNamesId)));
templateCache.put(template, tid);
return tid;
}
@@ -1114,7 +1078,8 @@ public class SegmentWriter {
byte[] id = ((SegmentNodeState) state).getStableIdBytes();
stableId = writeBlock(id, 0, id.length);
}
- return newNodeStateWriter(stableId, ids).write(writer);
+ return writeOperationHandler.execute(generation, newWriteOperation(
+ newNodeStateWriter(stableId, ids)));
}
/**
@@ -1188,7 +1153,7 @@ public class SegmentWriter {
private boolean isOldGeneration(RecordId id) {
try {
int thatGen = id.getSegmentId().getGcGeneration();
- int thisGen = writer.getGeneration();
+ int thisGen = generation;
return thatGen < thisGen;
} catch (SegmentNotFoundException snfe) {
// This SNFE means a defer compacted node state is too far
Modified: jackrabbit/oak/branches/1.6/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriterBuilder.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriterBuilder.java?rev=1848193&r1=1848192&r2=1848193&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.6/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriterBuilder.java (original)
+++ jackrabbit/oak/branches/1.6/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriterBuilder.java Wed Dec 5 09:34:27 2018
@@ -21,6 +21,8 @@ package org.apache.jackrabbit.oak.segmen
import static com.google.common.base.Preconditions.checkNotNull;
+import java.io.IOException;
+
import javax.annotation.Nonnull;
import com.google.common.base.Supplier;
@@ -165,9 +167,16 @@ public final class SegmentWriterBuilder
store.getBlobStore(),
cacheManager,
new WriteOperationHandler() {
+ @Override
+ public int getGeneration() {
+ throw new UnsupportedOperationException("Cannot write to read-only store");
+ }
+
@Nonnull
@Override
- public RecordId execute(@Nonnull WriteOperation writeOperation) {
+ public RecordId execute(int generation,
+ @Nonnull WriteOperation writeOperation)
+ throws IOException {
throw new UnsupportedOperationException("Cannot write to read-only store");
}
Modified: jackrabbit/oak/branches/1.6/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/WriteOperationHandler.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/WriteOperationHandler.java?rev=1848193&r1=1848192&r2=1848193&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.6/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/WriteOperationHandler.java (original)
+++ jackrabbit/oak/branches/1.6/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/WriteOperationHandler.java Wed Dec 5 09:34:27 2018
@@ -30,6 +30,11 @@ import javax.annotation.Nonnull;
interface WriteOperationHandler {
/**
+ * @return the current generation of the store.
+ */
+ int getGeneration();
+
+ /**
* A {@code WriteOperation} encapsulates an operation on a {@link SegmentWriter}.
* Executing it performs the actual act of persisting changes to a
* {@link SegmentBufferWriter}.
@@ -49,12 +54,14 @@ interface WriteOperationHandler {
/**
* Execute the passed {@code writeOperation} by passing it a {@link SegmentBufferWriter}.
+ * @param generation the generation the changes should persisted with.
* @param writeOperation {@link WriteOperation} to execute
* @return {@code RecordId} that resulted from persisting the changes.
* @throws IOException
*/
@Nonnull
- RecordId execute(@Nonnull WriteOperation writeOperation) throws IOException;
+ RecordId execute(int generation, @Nonnull WriteOperation writeOperation)
+ throws IOException;
/**
* Flush any pending changes on any {@link SegmentBufferWriter} managed by this instance.
Modified: jackrabbit/oak/branches/1.6/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriterPoolTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriterPoolTest.java?rev=1848193&r1=1848192&r2=1848193&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.6/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriterPoolTest.java (original)
+++ jackrabbit/oak/branches/1.6/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriterPoolTest.java Wed Dec 5 09:34:27 2018
@@ -36,10 +36,10 @@ import java.util.concurrent.ExecutorServ
import java.util.concurrent.Future;
import java.util.concurrent.TimeoutException;
-import javax.annotation.CheckForNull;
import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
-import com.google.common.base.Suppliers;
+import com.google.common.base.Supplier;
import org.apache.jackrabbit.oak.segment.WriteOperationHandler.WriteOperation;
import org.apache.jackrabbit.oak.segment.memory.MemoryStore;
import org.junit.After;
@@ -50,12 +50,19 @@ public class SegmentBufferWriterPoolTest
private final RecordId rootId = store.getRevisions().getHead();
+ private int generation = 0;
+
private final SegmentBufferWriterPool pool = new SegmentBufferWriterPool(
store,
store.getTracker(),
store.getReader(),
"",
- Suppliers.ofInstance(0)
+ new Supplier<Integer>() {
+ @Override
+ public Integer get() {
+ return generation;
+ }
+ }
);
private final ExecutorService[] executors = new ExecutorService[] {
@@ -70,18 +77,19 @@ public class SegmentBufferWriterPoolTest
}
}
- private Future<RecordId> execute(final WriteOperation op, int executor) {
+ private Future<RecordId> execute(final int generation, final WriteOperation op, int executor) {
return executors[executor].submit(new Callable<RecordId>() {
@Override
public RecordId call() throws Exception {
- return pool.execute(op);
+ return pool.execute(generation, op);
}
});
}
private WriteOperation createOp(final String key, final ConcurrentMap<String, SegmentBufferWriter> map) {
return new WriteOperation() {
- @Nonnull @Override
+ @Nonnull
+ @Override
public RecordId execute(@Nonnull SegmentBufferWriter writer) {
map.put(key, writer);
return rootId;
@@ -91,10 +99,11 @@ public class SegmentBufferWriterPoolTest
@Test
public void testThreadAffinity() throws IOException, ExecutionException, InterruptedException {
+ int gen = pool.getGeneration();
ConcurrentMap<String, SegmentBufferWriter> map1 = newConcurrentMap();
- Future<RecordId> res1 = execute(createOp("a", map1), 0);
- Future<RecordId> res2 = execute(createOp("b", map1), 1);
- Future<RecordId> res3 = execute(createOp("c", map1), 2);
+ Future<RecordId> res1 = execute(gen, createOp("a", map1), 0);
+ Future<RecordId> res2 = execute(gen, createOp("b", map1), 1);
+ Future<RecordId> res3 = execute(gen, createOp("c", map1), 2);
// Give the tasks some time to complete
sleepUninterruptibly(10, MILLISECONDS);
@@ -105,9 +114,9 @@ public class SegmentBufferWriterPoolTest
assertEquals(3, map1.size());
ConcurrentMap<String, SegmentBufferWriter> map2 = newConcurrentMap();
- Future<RecordId> res4 = execute(createOp("a", map2), 0);
- Future<RecordId> res5 = execute(createOp("b", map2), 1);
- Future<RecordId> res6 = execute(createOp("c", map2), 2);
+ Future<RecordId> res4 = execute(gen, createOp("a", map2), 0);
+ Future<RecordId> res5 = execute(gen, createOp("b", map2), 1);
+ Future<RecordId> res6 = execute(gen, createOp("c", map2), 2);
// Give the tasks some time to complete
sleepUninterruptibly(10, MILLISECONDS);
@@ -121,10 +130,11 @@ public class SegmentBufferWriterPoolTest
@Test
public void testFlush() throws ExecutionException, InterruptedException, IOException {
+ int gen = pool.getGeneration();
ConcurrentMap<String, SegmentBufferWriter> map1 = newConcurrentMap();
- Future<RecordId> res1 = execute(createOp("a", map1), 0);
- Future<RecordId> res2 = execute(createOp("b", map1), 1);
- Future<RecordId> res3 = execute(createOp("c", map1), 2);
+ Future<RecordId> res1 = execute(gen, createOp("a", map1), 0);
+ Future<RecordId> res2 = execute(gen, createOp("b", map1), 1);
+ Future<RecordId> res3 = execute(gen, createOp("c", map1), 2);
// Give the tasks some time to complete
sleepUninterruptibly(10, MILLISECONDS);
@@ -137,9 +147,9 @@ public class SegmentBufferWriterPoolTest
pool.flush();
ConcurrentMap<String, SegmentBufferWriter> map2 = newConcurrentMap();
- Future<RecordId> res4 = execute(createOp("a", map2), 0);
- Future<RecordId> res5 = execute(createOp("b", map2), 1);
- Future<RecordId> res6 = execute(createOp("c", map2), 2);
+ Future<RecordId> res4 = execute(gen, createOp("a", map2), 0);
+ Future<RecordId> res5 = execute(gen, createOp("b", map2), 1);
+ Future<RecordId> res6 = execute(gen, createOp("c", map2), 2);
// Give the tasks some time to complete
sleepUninterruptibly(10, MILLISECONDS);
@@ -152,10 +162,61 @@ public class SegmentBufferWriterPoolTest
}
@Test
+ public void testCompaction() throws ExecutionException, InterruptedException, IOException {
+ int gen = pool.getGeneration();
+ ConcurrentMap<String, SegmentBufferWriter> map1 = newConcurrentMap();
+ Future<RecordId> res1 = execute(gen, createOp("a", map1), 0);
+ Future<RecordId> res2 = execute(gen, createOp("b", map1), 1);
+ Future<RecordId> res3 = execute(gen, createOp("c", map1), 2);
+
+ // Give the tasks some time to complete
+ sleepUninterruptibly(10, MILLISECONDS);
+
+ assertEquals(rootId, res1.get());
+ assertEquals(rootId, res2.get());
+ assertEquals(rootId, res3.get());
+ assertEquals(3, map1.size());
+
+ // Simulate compaction by increasing the global gc generation
+ generation++;
+
+ // Write using previous generation
+ ConcurrentMap<String, SegmentBufferWriter> map2 = newConcurrentMap();
+ Future<RecordId> res4 = execute(gen, createOp("a", map2), 0);
+ Future<RecordId> res5 = execute(gen, createOp("b", map2), 1);
+ Future<RecordId> res6 = execute(gen, createOp("c", map2), 2);
+
+ // Give the tasks some time to complete
+ sleepUninterruptibly(10, MILLISECONDS);
+
+ assertEquals(rootId, res4.get());
+ assertEquals(rootId, res5.get());
+ assertEquals(rootId, res6.get());
+ assertEquals(3, map2.size());
+ assertEquals(map1, map2);
+
+ // Write using current generation
+ ConcurrentMap<String, SegmentBufferWriter> map3 = newConcurrentMap();
+ Future<RecordId> res7 = execute(gen + 1, createOp("a", map3), 0);
+ Future<RecordId> res8 = execute(gen + 1, createOp("b", map3), 1);
+ Future<RecordId> res9 = execute(gen + 1, createOp("c", map3), 2);
+
+ // Give the tasks some time to complete
+ sleepUninterruptibly(10, MILLISECONDS);
+
+ assertEquals(rootId, res7.get());
+ assertEquals(rootId, res8.get());
+ assertEquals(rootId, res9.get());
+ assertEquals(3, map3.size());
+ assertTrue(intersection(newHashSet(map1.values()), newHashSet(map3.values())).isEmpty());
+ }
+
+ @Test
public void testFlushBlocks() throws ExecutionException, InterruptedException {
- Future<RecordId> res = execute(new WriteOperation() {
- @Nonnull
- @CheckForNull @Override
+ int generation = pool.getGeneration();
+ Future<RecordId> res = execute(generation, new WriteOperation() {
+ @Nullable
+ @Override
public RecordId execute(@Nonnull SegmentBufferWriter writer) {
try {
// This should deadlock as flush waits for this write
Modified: jackrabbit/oak/branches/1.6/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreIT.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreIT.java?rev=1848193&r1=1848192&r2=1848193&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.6/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreIT.java (original)
+++ jackrabbit/oak/branches/1.6/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreIT.java Wed Dec 5 09:34:27 2018
@@ -21,20 +21,19 @@ package org.apache.jackrabbit.oak.segmen
import static com.google.common.collect.Lists.newArrayList;
import static com.google.common.collect.Maps.newLinkedHashMap;
import static com.google.common.collect.Sets.newTreeSet;
-import static org.apache.jackrabbit.oak.segment.SegmentWriterBuilder.segmentWriterBuilder;
import static java.util.concurrent.Executors.newSingleThreadExecutor;
import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.apache.jackrabbit.oak.segment.SegmentWriterBuilder.segmentWriterBuilder;
import static org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreBuilder;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
-import static org.junit.Assert.fail;
-
import java.io.InputStream;
import java.io.RandomAccessFile;
import java.util.Collections;
@@ -56,16 +55,15 @@ import com.google.common.base.Strings;
import com.google.common.util.concurrent.Monitor;
import com.google.common.util.concurrent.Monitor.Guard;
import org.apache.jackrabbit.oak.api.Blob;
-import org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState;
import org.apache.jackrabbit.oak.plugins.memory.AbstractBlob;
+import org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState;
+import org.apache.jackrabbit.oak.segment.SegmentWriter;
import org.apache.jackrabbit.oak.segment.RecordId;
import org.apache.jackrabbit.oak.segment.Segment;
import org.apache.jackrabbit.oak.segment.SegmentNodeBuilder;
import org.apache.jackrabbit.oak.segment.SegmentNodeState;
-import org.apache.jackrabbit.oak.segment.SegmentWriter;
import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
import org.apache.jackrabbit.oak.spi.state.NodeState;
-import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -319,7 +317,6 @@ public class FileStoreIT {
}
}
- @Ignore("OAK-7867")
@Test
public void blockingBlob() throws Exception {