You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ju...@apache.org on 2013/02/20 14:14:40 UTC
svn commit: r1448151 - in /jackrabbit/oak/trunk/oak-core/src:
main/java/org/apache/jackrabbit/oak/plugins/segment/
test/java/org/apache/jackrabbit/oak/plugins/segment/
Author: jukka
Date: Wed Feb 20 13:14:39 2013
New Revision: 1448151
URL: http://svn.apache.org/r1448151
Log:
OAK-629: SegmentMK: Reverse writing of segments
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/ListRecord.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/MapRecord.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/MemoryStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/MongoStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/NodeTemplate.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/RecordId.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentReader.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/RecordTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentSizeTest.java
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/ListRecord.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/ListRecord.java?rev=1448151&r1=1448150&r2=1448151&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/ListRecord.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/ListRecord.java Wed Feb 20 13:14:39 2013
@@ -51,7 +51,8 @@ class ListRecord extends Record {
} else {
int bucketIndex = index / bucketSize;
int bucketOffset = index % bucketSize;
- RecordId bucketId = reader.readRecordId(getRecordId(), bucketIndex * 4);
+ RecordId bucketId = reader.readRecordId(
+ getRecordId(), bucketIndex * Segment.RECORD_ID_BYTES);
ListRecord bucket =
new ListRecord(bucketId, bucketSize);
return bucket.getEntry(reader, bucketOffset);
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/MapRecord.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/MapRecord.java?rev=1448151&r1=1448150&r2=1448151&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/MapRecord.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/MapRecord.java Wed Feb 20 13:14:39 2013
@@ -46,6 +46,20 @@ class MapRecord extends Record {
return getEntry(reader, key, 0);
}
+ private int getHash(SegmentReader reader, int index) {
+ return reader.readInt(getRecordId(), 4 + index * 4);
+ }
+
+ private String getKey(SegmentReader reader, int size, int index) {
+ int offset = 4 + size * 4 + index * Segment.RECORD_ID_BYTES;
+ return reader.readString(reader.readRecordId(getRecordId(), offset));
+ }
+
+ private RecordId getValue(SegmentReader reader, int size, int index) {
+ int offset = 4 + size * 4 + (size + index) * Segment.RECORD_ID_BYTES;
+ return reader.readRecordId(getRecordId(), offset);
+ }
+
private RecordId getEntry(SegmentReader reader, String key, int level) {
int size = 1 << LEVEL_BITS;
int mask = size - 1;
@@ -56,16 +70,15 @@ class MapRecord extends Record {
if (bucketSize == 0) {
return null;
} else if (bucketSize <= size || shift >= 32) {
- int offset = 0;
- while (offset < bucketSize && reader.readInt(getRecordId(), 4 + offset * 4) < code) {
- offset++;
+ int index = 0;
+ while (index < bucketSize && getHash(reader, index) < code) {
+ index++;
}
- while (offset < bucketSize && reader.readInt(getRecordId(), 4 + offset * 4) == code) {
- RecordId keyId = reader.readRecordId(getRecordId(), 4 + (bucketSize + offset) * 4);
- if (key.equals(reader.readString(keyId))) {
- return reader.readRecordId(getRecordId(), 4 + (2 * bucketSize + offset) * 4);
+ while (index < bucketSize && getHash(reader, index) == code) {
+ if (key.equals(getKey(reader, bucketSize, index))) {
+ return getValue(reader, bucketSize, index);
}
- offset++;
+ index++;
}
return null;
} else {
@@ -74,7 +87,7 @@ class MapRecord extends Record {
long bucketBit = 1L << bucketIndex;
if ((bucketMap & bucketBit) != 0) {
bucketIndex = Long.bitCount(bucketMap & (bucketBit - 1));
- RecordId bucketId = reader.readRecordId(getRecordId(), 12 + bucketIndex * 4);
+ RecordId bucketId = reader.readRecordId(getRecordId(), 12 + bucketIndex * Segment.RECORD_ID_BYTES);
return new MapRecord(bucketId).getEntry(reader, key, level + 1);
} else {
return null;
@@ -106,19 +119,20 @@ class MapRecord extends Record {
}
@Override
public Entry next() {
- final int offset = index++;
+ final int i = index++;
return new Entry() {
@Override
public String getKey() {
- RecordId id = reader.readRecordId(getRecordId(), 4 + (bucketSize + offset) * 4);
- return reader.readString(id);
+ return MapRecord.this.getKey(
+ reader, bucketSize, i);
}
@Override
public RecordId getValue() {
- return reader.readRecordId(getRecordId(), 4 + (2 * bucketSize + offset) * 4);
+ return MapRecord.this.getValue(
+ reader, bucketSize, i);
}
@Override
- public RecordId setValue(RecordId arg0) {
+ public RecordId setValue(RecordId value) {
throw new UnsupportedOperationException();
}
};
@@ -136,7 +150,8 @@ class MapRecord extends Record {
List<Iterable<Entry>> iterables =
Lists.newArrayListWithCapacity(bucketCount);
for (int i = 0; i < bucketCount; i++) {
- RecordId bucketId = reader.readRecordId(getRecordId(), 12 + i * 4);
+ RecordId bucketId = reader.readRecordId(
+ getRecordId(), 12 + i * Segment.RECORD_ID_BYTES);
iterables.add(new MapRecord(bucketId).getEntries(reader, level + 1));
}
return Iterables.concat(iterables);
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/MemoryStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/MemoryStore.java?rev=1448151&r1=1448150&r2=1448151&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/MemoryStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/MemoryStore.java Wed Feb 20 13:14:39 2013
@@ -25,8 +25,6 @@ import org.apache.jackrabbit.oak.plugins
public class MemoryStore implements SegmentStore {
- private static final int MAX_SEGMENT_SIZE = 1 << 20; // 1MB
-
private final Map<UUID, Segment> segments =
Collections.synchronizedMap(new HashMap<UUID, Segment>());
@@ -54,11 +52,6 @@ public class MemoryStore implements Segm
}
@Override
- public int getMaxSegmentSize() {
- return MAX_SEGMENT_SIZE;
- }
-
- @Override
public Segment readSegment(UUID segmentId) {
Segment segment = segments.get(segmentId);
if (segment != null) {
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/MongoStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/MongoStore.java?rev=1448151&r1=1448150&r2=1448151&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/MongoStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/MongoStore.java Wed Feb 20 13:14:39 2013
@@ -19,13 +19,9 @@ package org.apache.jackrabbit.oak.plugin
import java.util.List;
import java.util.UUID;
import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeState;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.mongodb.BasicDBObject;
@@ -36,8 +32,6 @@ import com.mongodb.Mongo;
public class MongoStore implements SegmentStore {
- private static final int MAX_SEGMENT_SIZE = 1 << 23; // 8MB
-
private final DBCollection segments;
private final DBCollection journals;
@@ -85,11 +79,6 @@ public class MongoStore implements Segme
}
@Override
- public int getMaxSegmentSize() {
- return MAX_SEGMENT_SIZE;
- }
-
- @Override
public Segment readSegment(final UUID segmentId) {
return cache.getSegment(segmentId, new Callable<Segment>() {
@Override
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/NodeTemplate.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/NodeTemplate.java?rev=1448151&r1=1448150&r2=1448151&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/NodeTemplate.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/NodeTemplate.java Wed Feb 20 13:14:39 2013
@@ -204,13 +204,13 @@ class NodeTemplate {
checkNotNull(recordId);
checkElementIndex(index, properties.length);
- int offset = 8;
- if (hasNoChildNodes()) {
- offset = 4;
+ int offset = 1;
+ if (!hasNoChildNodes()) {
+ offset++;
}
return new SegmentPropertyState(
- properties[index], reader,
- reader.readRecordId(recordId, offset + index * 4));
+ properties[index], reader, reader.readRecordId(
+ recordId, (offset + index) * Segment.RECORD_ID_BYTES));
}
public Iterable<PropertyState> getProperties(
@@ -223,12 +223,13 @@ class NodeTemplate {
if (mixinTypes != null) {
list.add(mixinTypes);
}
- int offset = 8;
- if (hasNoChildNodes()) {
- offset = 4;
+ int offset = 1;
+ if (!hasNoChildNodes()) {
+ offset++;
}
for (int i = 0; i < properties.length; i++) {
- RecordId propertyId = reader.readRecordId(recordId, offset + i * 4);
+ RecordId propertyId = reader.readRecordId(
+ recordId, (offset + i) * Segment.RECORD_ID_BYTES);
list.add(new SegmentPropertyState(
properties[i], reader, propertyId));
}
@@ -239,7 +240,8 @@ class NodeTemplate {
if (hasNoChildNodes()) {
return 0;
} else if (hasManyChildNodes()) {
- RecordId childNodesId = reader.readRecordId(recordId, 4);
+ RecordId childNodesId =
+ reader.readRecordId(recordId, Segment.RECORD_ID_BYTES);
return new MapRecord(childNodesId).size(reader);
} else {
return 1;
@@ -251,7 +253,8 @@ class NodeTemplate {
if (hasNoChildNodes()) {
return false;
} else if (hasManyChildNodes()) {
- RecordId childNodesId = reader.readRecordId(recordId, 4);
+ RecordId childNodesId =
+ reader.readRecordId(recordId, Segment.RECORD_ID_BYTES);
return new MapRecord(childNodesId).getEntry(reader, name) != null;
} else {
return name.equals(childName);
@@ -263,7 +266,8 @@ class NodeTemplate {
if (hasNoChildNodes()) {
return null;
} else if (hasManyChildNodes()) {
- RecordId childNodesId = reader.readRecordId(recordId, 4);
+ RecordId childNodesId =
+ reader.readRecordId(recordId, Segment.RECORD_ID_BYTES);
RecordId childNodeId =
new MapRecord(childNodesId).getEntry(reader, name);
if (childNodeId != null) {
@@ -273,7 +277,7 @@ class NodeTemplate {
}
} else if (name.equals(childName)) {
RecordId childNodeId =
- reader.readRecordId(recordId, 4);
+ reader.readRecordId(recordId, Segment.RECORD_ID_BYTES);
return new SegmentNodeState(reader, childNodeId);
} else {
return null;
@@ -285,7 +289,8 @@ class NodeTemplate {
if (hasNoChildNodes()) {
return Collections.emptyList();
} else if (hasManyChildNodes()) {
- RecordId childNodesId = reader.readRecordId(recordId, 4);
+ RecordId childNodesId =
+ reader.readRecordId(recordId, Segment.RECORD_ID_BYTES);
return Iterables.transform(
new MapRecord(childNodesId).getEntries(reader),
new Function<MapRecord.Entry, String>() {
@@ -304,7 +309,8 @@ class NodeTemplate {
if (hasNoChildNodes()) {
return Collections.emptyList();
} else if (hasManyChildNodes()) {
- RecordId childNodesId = reader.readRecordId(recordId, 4);
+ RecordId childNodesId =
+ reader.readRecordId(recordId, Segment.RECORD_ID_BYTES);
return Iterables.transform(
new MapRecord(childNodesId).getEntries(reader),
new Function<MapRecord.Entry, ChildNodeEntry>() {
@@ -316,7 +322,8 @@ class NodeTemplate {
}
});
} else {
- RecordId childNodeId = reader.readRecordId(recordId, 4);
+ RecordId childNodeId =
+ reader.readRecordId(recordId, Segment.RECORD_ID_BYTES);
return Collections.singletonList(new MemoryChildNodeEntry(
childName, new SegmentNodeState(reader, childNodeId)));
}
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/RecordId.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/RecordId.java?rev=1448151&r1=1448150&r2=1448151&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/RecordId.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/RecordId.java Wed Feb 20 13:14:39 2013
@@ -16,9 +16,10 @@
*/
package org.apache.jackrabbit.oak.plugins.segment;
-import java.util.UUID;
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
-import com.google.common.base.Preconditions;
+import java.util.UUID;
public final class RecordId implements Comparable<RecordId> {
@@ -40,7 +41,9 @@ public final class RecordId implements C
private final int offset;
public RecordId(UUID segmentId, int offset) {
- this.segmentId = Preconditions.checkNotNull(segmentId);
+ checkArgument(offset < Segment.MAX_SEGMENT_SIZE);
+ checkArgument((offset & (Segment.RECORD_ALIGN_BYTES - 1)) == 0);
+ this.segmentId = checkNotNull(segmentId);
this.offset = offset;
}
@@ -56,6 +59,7 @@ public final class RecordId implements C
@Override
public int compareTo(RecordId that) {
+ checkNotNull(that);
int diff = segmentId.compareTo(that.segmentId);
if (diff == 0) {
diff = offset - that.offset;
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java?rev=1448151&r1=1448150&r2=1448151&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java Wed Feb 20 13:14:39 2013
@@ -27,6 +27,52 @@ import com.google.common.cache.Weigher;
class Segment {
+ /**
+ * Number of bytes used for storing a record identifier. One byte
+ * is used for identifying the segment and two for the record offset
+ * within that segment.
+ */
+ static final int RECORD_ID_BYTES = 1 + 2;
+
+ /**
+ * The limit on segment references within one segment. Since record
+ * identifiers use one byte to indicate the referenced segment, a single
+ * segment can hold references to up to 256 segments.
+ */
+ static final int SEGMENT_REFERENCE_LIMIT = 1 << 8; // 256
+
+ /**
+ * The number of bytes (or bits of address space) to use for the
+ * alignment boundary of segment records.
+ */
+ static final int RECORD_ALIGN_BITS = 2;
+ static final int RECORD_ALIGN_BYTES = 1 << RECORD_ALIGN_BITS; // 4
+
+ /**
+ * Maximum segment size. Record identifiers are stored as three-byte
+ * sequences with the first byte indicating the segment and the next
+ * two the offset within that segment. Since all records are aligned
+ * at four-byte boundaries, the two bytes can address up to 256kB of
+ * record data.
+ */
+ static final int MAX_SEGMENT_SIZE = 1 << (16 + RECORD_ALIGN_BITS); // 256kB
+
+ /**
+ * The size limit for small values. The variable length of small values
+ * is encoded as a single byte with the high bit as zero, which gives us
+ * seven bits for encoding the length of the value.
+ */
+ static final int SMALL_LIMIT = 1 << 7;
+
+ /**
+ * The size limit for medium values. The variable length of medium values
+ * is encoded as two bytes with the highest bits of the first byte set to
+ * one and zero, which gives us 14 bits for encoding the length of the
+ * value. And since small values are never stored as medium ones, we can
+ * extend the size range to cover that many longer values.
+ */
+ static final int MEDIUM_LIMIT = 1 << (16-2) + SMALL_LIMIT;
+
static final Weigher<UUID, Segment> WEIGHER =
new Weigher<UUID, Segment>() {
@Override
@@ -35,10 +81,6 @@ class Segment {
}
};
- static final int SMALL_LIMIT = 1 << 7;
-
- static final int MEDIUM_LIMIT = 1 << 14 + SMALL_LIMIT;
-
private final UUID uuid;
private final byte[] data;
@@ -68,8 +110,9 @@ class Segment {
}
public byte readByte(int position) {
- checkElementIndex(position, data.length);
- return data[position];
+ int pos = position - (MAX_SEGMENT_SIZE - data.length);
+ checkElementIndex(pos, data.length);
+ return data[pos];
}
/**
@@ -82,29 +125,33 @@ class Segment {
* @param length number of bytes to read
*/
public void readBytes(int position, byte[] buffer, int offset, int length) {
- checkPositionIndexes(position, position + length, data.length);
+ int pos = position - (MAX_SEGMENT_SIZE - data.length);
+ checkPositionIndexes(pos, pos + length, data.length);
checkNotNull(buffer);
checkPositionIndexes(offset, offset + length, buffer.length);
- System.arraycopy(data, position, buffer, offset, length);
+ System.arraycopy(data, pos, buffer, offset, length);
}
- RecordId readRecordId(int offset) {
+ RecordId readRecordId(int position) {
+ int pos = position - (MAX_SEGMENT_SIZE - data.length);
+ checkPositionIndexes(pos, pos + RECORD_ID_BYTES, data.length);
return new RecordId(
- uuids[data[offset] & 0xff],
- (data[offset + 1] & 0xff) << 16
- | (data[offset + 2] & 0xff) << 8
- | (data[offset + 3] & 0xff));
+ uuids[data[pos] & 0xff],
+ (data[pos + 1] & 0xff) << (8 + Segment.RECORD_ALIGN_BITS)
+ | (data[pos + 2] & 0xff) << Segment.RECORD_ALIGN_BITS);
}
- public int readInt(int offset) {
- checkPositionIndexes(offset, offset + 4, data.length);
- return ByteBuffer.wrap(data).getInt(offset);
+ public int readInt(int position) {
+ int pos = position - (MAX_SEGMENT_SIZE - data.length);
+ checkPositionIndexes(pos, pos + 4, data.length);
+ return ByteBuffer.wrap(data).getInt(pos);
}
- public long readLong(int offset) {
- checkPositionIndexes(offset, offset + 8, data.length);
- return ByteBuffer.wrap(data).getLong(offset);
+ public long readLong(int position) {
+ int pos = position - (Segment.MAX_SEGMENT_SIZE - data.length);
+ checkPositionIndexes(pos, pos + 8, data.length);
+ return ByteBuffer.wrap(data).getLong(pos);
}
}
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentReader.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentReader.java?rev=1448151&r1=1448150&r2=1448151&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentReader.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentReader.java Wed Feb 20 13:14:39 2013
@@ -96,7 +96,7 @@ public class SegmentReader {
RecordId primaryId = segment.readRecordId(offset);
primaryType = PropertyStates.createProperty(
"jcr:primaryType", readString(primaryId), Type.NAME);
- offset += 4;
+ offset += Segment.RECORD_ID_BYTES;
}
PropertyState mixinTypes = null;
@@ -105,7 +105,7 @@ public class SegmentReader {
for (int i = 0; i < mixins.length; i++) {
RecordId mixinId = segment.readRecordId(offset);
mixins[i] = readString(mixinId);
- offset += 4;
+ offset += Segment.RECORD_ID_BYTES;
}
mixinTypes = PropertyStates.createProperty(
"jcr:mixinTypes", Arrays.asList(mixins), Type.NAMES);
@@ -117,18 +117,18 @@ public class SegmentReader {
} else if (!zeroChildNodes) {
RecordId childNameId = segment.readRecordId(offset);
childName = readString(childNameId);
- offset += 4;
+ offset += Segment.RECORD_ID_BYTES;
}
PropertyTemplate[] properties =
new PropertyTemplate[propertyCount];
for (int i = 0; i < properties.length; i++) {
RecordId propertyNameId = segment.readRecordId(offset);
- byte type = segment.readByte(offset + 4);
+ offset += Segment.RECORD_ID_BYTES;
+ byte type = segment.readByte(offset++);
properties[i] = new PropertyTemplate(
readString(propertyNameId),
Type.fromTag(Math.abs(type), type < 0));
- offset += 5;
}
return new NodeTemplate(
@@ -168,7 +168,7 @@ public class SegmentReader {
} else if ((length & 0x40) == 0) {
return ((length & 0x3f) << 8
| segment.readByte(offset) & 0xff)
- + 0x80;
+ + Segment.SMALL_LIMIT;
} else {
return (((long) length & 0x3f) << 56
| ((long) (segment.readByte(offset++) & 0xff)) << 48
@@ -178,7 +178,7 @@ public class SegmentReader {
| ((long) (segment.readByte(offset++) & 0xff)) << 16
| ((long) (segment.readByte(offset++) & 0xff)) << 8
| ((long) (segment.readByte(offset) & 0xff)))
- + 0x4080;
+ + Segment.MEDIUM_LIMIT;
}
}
@@ -186,8 +186,8 @@ public class SegmentReader {
Segment segment = store.readSegment(recordId.getSegmentId());
int offset = recordId.getOffset();
long length = readLength(segment, offset);
- if (length < 0x4080) {
- if (length < 0x80) {
+ if (length < Segment.MEDIUM_LIMIT) {
+ if (length < Segment.SMALL_LIMIT) {
offset += 1;
} else {
offset += 2;
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentStore.java?rev=1448151&r1=1448150&r2=1448151&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentStore.java Wed Feb 20 13:14:39 2013
@@ -24,8 +24,6 @@ public interface SegmentStore {
boolean setJournalHead(RecordId head, RecordId base);
- int getMaxSegmentSize();
-
Segment readSegment(UUID segmentId);
void createSegment(Segment segment);
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java?rev=1448151&r1=1448150&r2=1448151&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java Wed Feb 20 13:14:39 2013
@@ -16,6 +16,7 @@
*/
package org.apache.jackrabbit.oak.plugins.segment;
+import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkPositionIndexes;
import static com.google.common.base.Preconditions.checkState;
@@ -23,7 +24,6 @@ import static com.google.common.base.Pre
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -49,44 +49,59 @@ import com.google.common.io.ByteStreams;
public class SegmentWriter {
- private static final int INITIAL_BUFFER_SIZE = 1 << 12; // 4kB
-
static final int BLOCK_SIZE = 1 << 12; // 4kB
private final SegmentStore store;
- private final int blocksPerSegment;
-
- private final int blockSegmentSize;
-
private final Map<String, RecordId> strings = Maps.newHashMap();
private final Map<NodeTemplate, RecordId> templates = Maps.newHashMap();
private UUID uuid = UUID.randomUUID();
- private List<UUID> uuids = new ArrayList<UUID>(255);
+ private final List<UUID> uuids =
+ Lists.newArrayListWithCapacity(Segment.SEGMENT_REFERENCE_LIMIT);
+
+ /**
+ * The segment write buffer, filled from the end to the beginning
+ * (see OAK-629). Note that we currently allocate the entire buffer
+ * right from the beginning. It might turn out that a better approach
+ * would be to start with a smaller buffer that grows automatically,
+ * or to use a pool of pre-allocated buffers.
+ */
+ private final byte[] buffer = new byte[Segment.MAX_SEGMENT_SIZE];
+
+ /**
+ * The number of bytes already written (or allocated). Counted from
+ * the <em>end</em> of the buffer.
+ */
+ private int length = 0;
- private ByteBuffer buffer = ByteBuffer.allocate(INITIAL_BUFFER_SIZE);
+ /**
+ * Current write position within the buffer. Grows up when raw data
+ * is written, but shifted downwards by the prepare methods.
+ */
+ private int position;
public SegmentWriter(SegmentStore store) {
this.store = store;
- this.blocksPerSegment = store.getMaxSegmentSize() / BLOCK_SIZE;
- this.blockSegmentSize = blocksPerSegment * BLOCK_SIZE;
}
public synchronized void flush() {
- if (buffer.position() > 0) {
- byte[] data = new byte[buffer.position()];
- buffer.flip();
- buffer.get(data);
+ if (length > 0) {
+ byte[] data = buffer;
+ if (length < buffer.length) {
+ data = new byte[length];
+ int start = buffer.length - length;
+ System.arraycopy(buffer, start, data, 0, data.length);
+ }
store.createSegment(new Segment(
- uuid, data, uuids.toArray(new UUID[0])));
+ uuid, data, uuids.toArray(new UUID[uuids.size()])));
uuid = UUID.randomUUID();
+ length = 0;
uuids.clear();
- buffer.clear();
}
}
@@ -95,42 +110,61 @@ public class SegmentWriter {
}
private synchronized RecordId prepare(int size, Collection<RecordId> ids) {
+ checkArgument(size >= 0);
+
Set<UUID> segmentIds = new HashSet<UUID>();
- for (RecordId id : ids) {
+ for (RecordId id : checkNotNull(ids)) {
UUID segmentId = id.getSegmentId();
if (!uuids.contains(segmentId)) {
segmentIds.add(segmentId);
}
}
- int fullSize = size + 4 * ids.size();
- if (buffer.position() + fullSize > store.getMaxSegmentSize()
+ int fullSize = size + ids.size() * Segment.RECORD_ID_BYTES;
+ checkArgument(fullSize > 0);
+
+ int alignment = Segment.RECORD_ALIGN_BYTES - 1;
+ int alignedSize = (fullSize + alignment) & ~alignment;
+ if (length + alignedSize > buffer.length
|| uuids.size() + segmentIds.size() > 0x100) {
flush();
}
- if (fullSize > buffer.remaining()) {
- int n = Math.min(buffer.capacity() * 2, store.getMaxSegmentSize());
- while (n < buffer.position() + fullSize) {
- n = Math.min(n * 2, store.getMaxSegmentSize());
- }
- ByteBuffer newBuffer = ByteBuffer.allocate(n);
- buffer.flip();
- newBuffer.put(buffer);
- buffer = newBuffer;
- }
- return new RecordId(uuid, buffer.position());
+
+ length += alignedSize;
+ position = buffer.length - length;
+ return new RecordId(uuid, position);
}
private synchronized void writeRecordId(RecordId id) {
+ checkNotNull(id);
+
UUID segmentId = id.getSegmentId();
int index = uuids.indexOf(segmentId);
if (index == -1) {
index = uuids.size();
uuids.add(segmentId);
}
- checkState(index < (1 << 8));
- checkState(id.getOffset() < (1 << 24));
- buffer.putInt(index << 24 | id.getOffset());
+ int offset = id.getOffset();
+
+ checkState(index < Segment.SEGMENT_REFERENCE_LIMIT);
+ checkState(0 <= offset && offset < buffer.length);
+ checkState((offset & (Segment.RECORD_ALIGN_BYTES - 1)) == 0);
+
+ buffer[position++] = (byte) index;
+ buffer[position++] = (byte) (offset >> (8 + Segment.RECORD_ALIGN_BITS));
+ buffer[position++] = (byte) (offset >> Segment.RECORD_ALIGN_BITS);
+ }
+
+ private synchronized void writeInt(int value) {
+ buffer[position++] = (byte) (value >> 24);
+ buffer[position++] = (byte) (value >> 16);
+ buffer[position++] = (byte) (value >> 8);
+ buffer[position++] = (byte) value;
+ }
+
+ private synchronized void writeLong(long value) {
+ writeInt((int) (value >> 32));
+ writeInt((int) value);
}
private synchronized RecordId writeListBucket(List<RecordId> bucket) {
@@ -196,9 +230,9 @@ public class SegmentWriter {
}
RecordId bucketId = prepare(4 + entries.size() * 4, ids);
- buffer.putInt(entries.size());
+ writeInt(entries.size());
for (MapEntry entry : entries) {
- buffer.putInt(entry.hashCode);
+ writeInt(entry.hashCode);
}
for (MapEntry entry : entries) {
writeRecordId(entry.key);
@@ -227,8 +261,8 @@ public class SegmentWriter {
}
RecordId bucketId = prepare(12, bucketIds);
- buffer.putInt(entries.size());
- buffer.putLong(bucketMap);
+ writeInt(entries.size());
+ writeLong(bucketMap);
for (RecordId id : bucketIds) {
writeRecordId(id);
}
@@ -239,7 +273,7 @@ public class SegmentWriter {
private synchronized RecordId writeValueRecord(
long length, RecordId blocks) {
RecordId valueId = prepare(8, Collections.singleton(blocks));
- buffer.putLong((length - 0x4080) | (0x3L << 62));
+ writeLong((length - Segment.MEDIUM_LIMIT) | (0x3L << 62));
writeRecordId(blocks);
return valueId;
}
@@ -258,7 +292,8 @@ public class SegmentWriter {
checkPositionIndexes(offset, offset + length, bytes.length);
RecordId blockId = prepare(length);
- buffer.put(bytes, offset, length);
+ System.arraycopy(bytes, offset, buffer, position, length);
+ position += length;
return blockId;
}
@@ -270,10 +305,7 @@ public class SegmentWriter {
*/
public RecordId writeList(List<RecordId> list) {
checkNotNull(list);
-
- if (list.isEmpty()) {
- return prepare(0); // special case
- }
+ checkArgument(list.size() > 0);
List<RecordId> thisLevel = list;
while (thisLevel.size() > 1) {
@@ -337,23 +369,27 @@ public class SegmentWriter {
return id;
}
- private RecordId internalWriteStream(InputStream stream)
+ private synchronized RecordId internalWriteStream(InputStream stream)
throws IOException {
// First read the head of the stream. This covers most small
// values and the frequently accessed head of larger ones.
// The head gets inlined in the current segment.
- byte[] head = new byte[0x4080];
+ byte[] head = new byte[Segment.MEDIUM_LIMIT];
int headLength = ByteStreams.read(stream, head, 0, head.length);
- if (headLength < 0x80) {
+ if (headLength < Segment.SMALL_LIMIT) {
RecordId id = prepare(1 + headLength);
- buffer.put((byte) headLength);
- buffer.put(head, 0, headLength);
+ buffer[position++] = (byte) headLength;
+ System.arraycopy(head, 0, buffer, position, headLength);
+ position += headLength;
return id;
- } else if (headLength < 0x4080) {
+ } else if (headLength < Segment.MEDIUM_LIMIT) {
RecordId id = prepare(2 + headLength);
- buffer.putShort((short) ((headLength - 0x80) | 0x8000));
- buffer.put(head, 0, headLength);
+ int len = (headLength - Segment.SMALL_LIMIT) | 0x8000;
+ buffer[position++] = (byte) (len >> 8);
+ buffer[position++] = (byte) len;
+ System.arraycopy(head, 0, buffer, position, headLength);
+ position += headLength;
return id;
} else {
// If the stream filled the full head buffer, it's likely
@@ -363,14 +399,18 @@ public class SegmentWriter {
long length = 0;
List<RecordId> blockIds = new ArrayList<RecordId>();
- byte[] bulk = new byte[blockSegmentSize];
+ byte[] bulk = new byte[Segment.MAX_SEGMENT_SIZE];
System.arraycopy(head, 0, bulk, 0, headLength);
int bulkLength = headLength + ByteStreams.read(
stream, bulk, headLength, bulk.length - headLength);
while (bulkLength > 0) {
UUID segmentId = UUID.randomUUID();
- store.createSegment(segmentId, bulk, 0, bulkLength);
- for (int pos = 0; pos < bulkLength; pos += BLOCK_SIZE) {
+ int align = Segment.RECORD_ALIGN_BYTES - 1;
+ int bulkAlignLength = (bulkLength + align) & ~align;
+ store.createSegment(segmentId, bulk, 0, bulkAlignLength);
+ for (int pos = Segment.MAX_SEGMENT_SIZE - bulkAlignLength;
+ pos < Segment.MAX_SEGMENT_SIZE;
+ pos += BLOCK_SIZE) {
blockIds.add(new RecordId(segmentId, pos));
}
length += bulkLength;
@@ -381,7 +421,7 @@ public class SegmentWriter {
}
}
- private RecordId writeProperty(PropertyState state) {
+ private synchronized RecordId writeProperty(PropertyState state) {
Type<?> type = state.getType();
int count = state.count();
@@ -398,15 +438,19 @@ public class SegmentWriter {
valueIds.add(writeString(state.getValue(Type.STRING, i)));
}
}
- RecordId valueId = writeList(valueIds);
- if (type.isArray()) {
- RecordId propertyId = prepare(4, Collections.singleton(valueId));
- buffer.putInt(count);
- writeRecordId(valueId);
+ if (!type.isArray()) {
+ return valueIds.iterator().next();
+ } else if (count == 0) {
+ RecordId propertyId = prepare(4);
+ writeInt(0);
return propertyId;
} else {
- return valueId;
+ RecordId listId = writeList(valueIds);
+ RecordId propertyId = prepare(4, Collections.singleton(listId));
+ writeInt(count);
+ writeRecordId(listId);
+ return propertyId;
}
}
@@ -463,7 +507,7 @@ public class SegmentWriter {
head |= propertyNames.length;
id = prepare(4 + propertyTypes.length, ids);
- buffer.putInt(head);
+ writeInt(head);
if (primaryId != null) {
writeRecordId(primaryId);
}
@@ -477,7 +521,7 @@ public class SegmentWriter {
}
for (int i = 0; i < propertyNames.length; i++) {
writeRecordId(propertyNames[i]);
- buffer.put(propertyTypes[i]);
+ buffer[position++] = propertyTypes[i];
}
templates.put(template, id);
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/RecordTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/RecordTest.java?rev=1448151&r1=1448150&r2=1448151&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/RecordTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/RecordTest.java Wed Feb 20 13:14:39 2013
@@ -26,7 +26,6 @@ import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
-import java.util.Calendar;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
@@ -81,7 +80,6 @@ public class RecordTest {
public void testListRecord() {
RecordId blockId = writer.writeBlock(bytes, 0, bytes.length);
- ListRecord zero = writeList(0, blockId);
ListRecord one = writeList(1, blockId);
ListRecord level1 = writeList(LEVEL_SIZE, blockId);
ListRecord level1p = writeList(LEVEL_SIZE + 1, blockId);
@@ -89,7 +87,6 @@ public class RecordTest {
ListRecord level2p = writeList(LEVEL_SIZE * LEVEL_SIZE + 1, blockId);
writer.flush();
- assertEquals(0, zero.size());
assertEquals(1, one.size());
assertEquals(blockId, one.getEntry(reader, 0));
assertEquals(LEVEL_SIZE, level1.size());
@@ -121,10 +118,10 @@ public class RecordTest {
checkRandomStreamRecord(0x4080);
checkRandomStreamRecord(SegmentWriter.BLOCK_SIZE);
checkRandomStreamRecord(SegmentWriter.BLOCK_SIZE + 1);
- checkRandomStreamRecord(store.getMaxSegmentSize());
- checkRandomStreamRecord(store.getMaxSegmentSize() + 1);
- checkRandomStreamRecord(store.getMaxSegmentSize() * 2);
- checkRandomStreamRecord(store.getMaxSegmentSize() * 2 + 1);
+ checkRandomStreamRecord(Segment.MAX_SEGMENT_SIZE);
+ checkRandomStreamRecord(Segment.MAX_SEGMENT_SIZE + 1);
+ checkRandomStreamRecord(Segment.MAX_SEGMENT_SIZE * 2);
+ checkRandomStreamRecord(Segment.MAX_SEGMENT_SIZE * 2 + 1);
}
private void checkRandomStreamRecord(int size) throws IOException {
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentSizeTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentSizeTest.java?rev=1448151&r1=1448150&r2=1448151&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentSizeTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentSizeTest.java Wed Feb 20 13:14:39 2013
@@ -36,8 +36,6 @@ import com.google.common.collect.Immutab
*/
public class SegmentSizeTest {
- private static final int BYTES_PER_REFERENCE = 4;
-
@Test
public void testNodeSize() {
NodeBuilder builder = MemoryNodeState.EMPTY_NODE.builder();
@@ -46,13 +44,13 @@ public class SegmentSizeTest {
builder = MemoryNodeState.EMPTY_NODE.builder();
builder.setProperty("foo", "bar");
- assertEquals(25, getSize(builder));
+ assertEquals(24, getSize(builder));
assertEquals(8, getAmortizedSize(builder));
builder = MemoryNodeState.EMPTY_NODE.builder();
builder.setProperty("foo", "bar");
builder.setProperty("baz", 123);
- assertEquals(42, getSize(builder));
+ assertEquals(40, getSize(builder));
assertEquals(12, getAmortizedSize(builder));
builder = MemoryNodeState.EMPTY_NODE.builder();
@@ -63,8 +61,8 @@ public class SegmentSizeTest {
builder = MemoryNodeState.EMPTY_NODE.builder();
builder.child("foo");
builder.child("bar");
- assertEquals(60, getSize(builder));
- assertEquals(44, getAmortizedSize(builder));
+ assertEquals(56, getSize(builder));
+ assertEquals(40, getAmortizedSize(builder));
}
@Test
@@ -77,12 +75,12 @@ public class SegmentSizeTest {
int base = getSize(builder);
builder.setProperty(PropertyStates.createProperty(
- "test", Collections.nCopies(10, string), Type.STRINGS));
- assertEquals(base + 10 * BYTES_PER_REFERENCE, getSize(builder));
+ "test", Collections.nCopies(12, string), Type.STRINGS));
+ assertEquals(base + 12 * Segment.RECORD_ID_BYTES, getSize(builder));
builder.setProperty(PropertyStates.createProperty(
"test", Collections.nCopies(100, string), Type.STRINGS));
- assertEquals(base + 100 * BYTES_PER_REFERENCE, getSize(builder));
+ assertEquals(base + 100 * Segment.RECORD_ID_BYTES, getSize(builder));
}
@Test
@@ -95,12 +93,12 @@ public class SegmentSizeTest {
int base = getSize(builder);
builder.setProperty(PropertyStates.createProperty(
- "test", Collections.nCopies(10, now), Type.DATES));
- assertEquals(base + 10 * BYTES_PER_REFERENCE, getSize(builder));
+ "test", Collections.nCopies(12, now), Type.DATES));
+ assertEquals(base + 12 * Segment.RECORD_ID_BYTES, getSize(builder));
builder.setProperty(PropertyStates.createProperty(
"test", Collections.nCopies(100, now), Type.DATES));
- assertEquals(base + 100 * BYTES_PER_REFERENCE, getSize(builder));
+ assertEquals(base + 100 * Segment.RECORD_ID_BYTES, getSize(builder));
}
@Test
@@ -115,7 +113,7 @@ public class SegmentSizeTest {
deny.setProperty("rep:principalName", "everyone");
deny.setProperty(PropertyStates.createProperty(
"rep:privileges", ImmutableList.of("jcr:read"), Type.NAMES));
- assertEquals(134, getSize(builder));
+ assertEquals(144, getSize(builder));
assertEquals(28, getAmortizedSize(builder));
NodeBuilder allow = builder.child("allow");
@@ -123,8 +121,8 @@ public class SegmentSizeTest {
allow.setProperty("rep:principalName", "administrators");
allow.setProperty(PropertyStates.createProperty(
"rep:privileges", ImmutableList.of("jcr:all"), Type.NAMES));
- assertEquals(259, getSize(builder));
- assertEquals(80, getAmortizedSize(builder));
+ assertEquals(264, getSize(builder));
+ assertEquals(72, getAmortizedSize(builder));
NodeBuilder deny0 = builder.child("deny0");
deny0.setProperty("jcr:primaryType", "rep:DenyACE", Type.NAME);
@@ -132,16 +130,16 @@ public class SegmentSizeTest {
deny0.setProperty("rep:glob", "*/activities/*");
builder.setProperty(PropertyStates.createProperty(
"rep:privileges", ImmutableList.of("jcr:read"), Type.NAMES));
- assertEquals(348, getSize(builder));
- assertEquals(116, getAmortizedSize(builder));
+ assertEquals(356, getSize(builder));
+ assertEquals(108, getAmortizedSize(builder));
NodeBuilder allow0 = builder.child("allow0");
allow0.setProperty("jcr:primaryType", "rep:GrantACE");
allow0.setProperty("rep:principalName", "user-administrators");
allow0.setProperty(PropertyStates.createProperty(
"rep:privileges", ImmutableList.of("jcr:all"), Type.NAMES));
- assertEquals(411, getSize(builder));
- assertEquals(152, getAmortizedSize(builder));
+ assertEquals(412, getSize(builder));
+ assertEquals(136, getAmortizedSize(builder));
}
private int getSize(NodeBuilder builder) {