You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by fr...@apache.org on 2018/09/05 08:57:56 UTC
svn commit: r1840106 - in /jackrabbit/oak/branches/1.6: ./
oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java
oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriterTest.java
Author: frm
Date: Wed Sep 5 08:57:56 2018
New Revision: 1840106
URL: http://svn.apache.org/viewvc?rev=1840106&view=rev
Log:
OAK-7721 - Check for too big records when allocating space
Backport r1840024 from trunk.
Modified:
jackrabbit/oak/branches/1.6/ (props changed)
jackrabbit/oak/branches/1.6/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java
jackrabbit/oak/branches/1.6/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriterTest.java
Propchange: jackrabbit/oak/branches/1.6/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Sep 5 08:57:56 2018
@@ -1,3 +1,3 @@
/jackrabbit/oak/branches/1.0:1665962
-/jackrabbit/oak/trunk:1781068,1781075,1781248,1781386,1781846,1781907,1782000,1782029,1782196,1782447,1782476,1782770,1782945,1782966,1782973,1782990,1783061,1783066,1783089,1783104-1783105,1783110,1783619,1783720,1783731,1783733,1783738,1783742,1783773,1783855,1783891,1784023,1784034,1784130,1784162,1784251,1784401,1784551,1784574,1784689,1785095,1785108,1785161,1785172,1785283,1785652,1785838,1785916-1785917,1785919,1785946,1786122,1787074,1787145,1787151,1787217,1787425,1788056,1788378,1788387-1788389,1788463,1788476,1788850,1789056,1789534,1789925,1789940,1789987,1790006,1790013,1790069,1790077,1790079,1790382,1790502-1790503,1792049,1792463,1792742,1792746,1793013,1793088,1793618,1793627,1793644,1794393,1794417,1794683,1795138,1795314,1795330,1795475,1795488,1795491,1795502,1795594,1795613,1795618,1796144,1796230,1796239,1796274,1796278,1796988,1797378,1798035,1798832,1798834,1799219,1799389,1799393,1799924,1800244,1800269,1800606,1800613,1800974,1801011,1801013,1801118-1801119
,1801675,1802260,1802262,1802286,1802548,1802905,1802934,1802938,1802973,1803026,1803247-1803249,1803951,1803953-1803955,1804437,1805851-1805852,1806668,1807308,1807688,1808022,1808125,1808128,1808142,1808240,1808246,1809024,1809026,1809131,1809163,1809178-1809179,1809253,1809255-1809256,1809289,1809745,1811071-1811072,1811155,1811380,1811655,1811952,1811963,1811986,1813192,1813538,1814189,1814332,1814397,1815201,1815438,1815926,1817326,1817919,1817987-1817988,1817990,1818038,1818042,1818056,1818124,1818137,1818554,1818576,1818645,1819048,1819050,1821237,1821325,1821358,1821495,1821516,1821847,1822207,1822850,1823172,1823655,1824896,1825471,1825654,1826237,1826338,1826532,1826640,1826932,1826957,1827472,1827486,1827977,1828502,1829527,1829569,1829587,1829665,1829854,1829864,1829987,1829998,1830019,1830160,1830239,1830748,1831190,1831374,1832379,1832535,1833308,1834648-1834649,1834681,1835060,1837475,1837998,1838637,1839746
+/jackrabbit/oak/trunk:1781068,1781075,1781248,1781386,1781846,1781907,1782000,1782029,1782196,1782447,1782476,1782770,1782945,1782966,1782973,1782990,1783061,1783066,1783089,1783104-1783105,1783110,1783619,1783720,1783731,1783733,1783738,1783742,1783773,1783855,1783891,1784023,1784034,1784130,1784162,1784251,1784401,1784551,1784574,1784689,1785095,1785108,1785161,1785172,1785283,1785652,1785838,1785916-1785917,1785919,1785946,1786122,1787074,1787145,1787151,1787217,1787425,1788056,1788378,1788387-1788389,1788463,1788476,1788850,1789056,1789534,1789925,1789940,1789987,1790006,1790013,1790069,1790077,1790079,1790382,1790502-1790503,1792049,1792463,1792742,1792746,1793013,1793088,1793618,1793627,1793644,1794393,1794417,1794683,1795138,1795314,1795330,1795475,1795488,1795491,1795502,1795594,1795613,1795618,1796144,1796230,1796239,1796274,1796278,1796988,1797378,1798035,1798832,1798834,1799219,1799389,1799393,1799924,1800244,1800269,1800606,1800613,1800974,1801011,1801013,1801118-1801119
,1801675,1802260,1802262,1802286,1802548,1802905,1802934,1802938,1802973,1803026,1803247-1803249,1803951,1803953-1803955,1804437,1805851-1805852,1806668,1807308,1807688,1808022,1808125,1808128,1808142,1808240,1808246,1809024,1809026,1809131,1809163,1809178-1809179,1809253,1809255-1809256,1809289,1809745,1811071-1811072,1811155,1811380,1811655,1811952,1811963,1811986,1813192,1813538,1814189,1814332,1814397,1815201,1815438,1815926,1817326,1817919,1817987-1817988,1817990,1818038,1818042,1818056,1818124,1818137,1818554,1818576,1818645,1819048,1819050,1821237,1821325,1821358,1821495,1821516,1821847,1822207,1822850,1823172,1823655,1824896,1825471,1825654,1826237,1826338,1826532,1826640,1826932,1826957,1827472,1827486,1827977,1828502,1829527,1829569,1829587,1829665,1829854,1829864,1829987,1829998,1830019,1830160,1830239,1830748,1831190,1831374,1832379,1832535,1833308,1834648-1834649,1834681,1835060,1837475,1837998,1838637,1839746,1840024
/jackrabbit/trunk:1345480
Modified: jackrabbit/oak/branches/1.6/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java?rev=1840106&r1=1840105&r2=1840106&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.6/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java (original)
+++ jackrabbit/oak/branches/1.6/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java Wed Sep 5 08:57:56 2018
@@ -470,15 +470,36 @@ public class SegmentBufferWriter impleme
segmentSize = align(headerSize + recordSize + length, 16);
}
+ // If the resulting segment buffer would be too big we need to allocate
+ // additional space. Allocating additional space is a recursive
+ // operation guarded by the `dirty` flag. The recursion can iterate at
+ // most two times. The base case happens when the `dirty` flag is
+ // `false`: the current buffer is empty, the record is too big to fit in
+ // an empty segment, and we fail with an `IllegalArgumentException`. The
+ // recursive step happens when the `dirty` flag is `true`:
+ // the current buffer is non-empty, we flush it, allocate a new buffer
+ // for an empty segment, and invoke `prepare()` once more.
+
if (segmentSize > buffer.length) {
- flush();
+ if (dirty) {
+ LOG.debug("Flushing full segment {} (headerSize={}, recordSize={}, length={}, segmentSize={})",
+ segment.getSegmentId(), headerSize, recordSize, length, segmentSize);
+ flush();
+ return prepare(type, size, ids);
+ }
+ throw new IllegalArgumentException(String.format(
+ "Record too big: type=%s, size=%s, recordIds=%s, total=%s",
+ type,
+ size,
+ ids.size(),
+ recordSize
+ ));
}
statistics.recordCount++;
length += recordSize;
position = buffer.length - length;
- checkState(position >= 0);
int recordNumber = recordNumbers.addRecord(type, position);
return new RecordId(segment.getSegmentId(), recordNumber);
Modified: jackrabbit/oak/branches/1.6/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriterTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriterTest.java?rev=1840106&r1=1840105&r2=1840106&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.6/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriterTest.java (original)
+++ jackrabbit/oak/branches/1.6/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriterTest.java Wed Sep 5 08:57:56 2018
@@ -24,6 +24,7 @@ import static org.junit.Assert.assertEqu
import static org.junit.Assert.assertNotEquals;
import java.io.File;
+import java.util.ArrayList;
import java.util.List;
import org.apache.jackrabbit.oak.segment.file.FileStore;
@@ -95,4 +96,43 @@ public class SegmentBufferWriterTest {
assertNotEquals(before, after);
}
+ @Test
+ public void tooBigRecord() throws Exception {
+
+ // See OAK-7721 to understand why this test exists.
+
+ try (FileStore store = openFileStore()) {
+
+ // Please don't change anything from the following statement yet.
+ // Read the next comment to understand why.
+
+ SegmentBufferWriter writer = new SegmentBufferWriter(
+ store,
+ store.getTracker().getSegmentCounter(),
+ store.getReader(),
+ "t",
+ store.getRevisions().getHead().getSegment().getGcGeneration()
+ );
+
+ // The size of the record is chosen with the precise intention to
+ // fool `writer` into having enough space to write the record. In
+ // particular, at the end of `prepare()`, `writer` will have
+ // `this.length = 262144`, which is `MAX_SEGMENT_SIZE`, and
+ // `this.position = 0`. This result is particularly sensitive to the
+ // initial content of the segment, which in turn is influenced by
+ // the segment info. Try to change the writer ID in the constructor
+ // of `SegmentBufferWriter` to a longer string, and you will have
+ // `prepare()` throw ISEs because the writer ID is embedded in the
+ // segment info.
+
+ IllegalArgumentException error = null;
+ try {
+ writer.prepare(RecordType.BLOCK, 262101, new ArrayList<RecordId>());
+ } catch (IllegalArgumentException e) {
+ error = e;
+ }
+ assertEquals("Record too big: type=BLOCK, size=262101, recordIds=0, total=262104", error != null ? error.getMessage() : null);
+ }
+ }
+
}