You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ju...@apache.org on 2013/04/22 09:51:28 UTC
svn commit: r1470404 - in /jackrabbit/oak/trunk/oak-core/src:
main/java/org/apache/jackrabbit/oak/plugins/segment/
test/java/org/apache/jackrabbit/oak/plugins/segment/
Author: jukka
Date: Mon Apr 22 07:51:28 2013
New Revision: 1470404
URL: http://svn.apache.org/r1470404
Log:
OAK-593: Segment-based MK
Use ByteBuffers instead of raw byte[] arrays to allow more flexibility in memory management
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/MemoryStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/MongoStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentSizeTest.java
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/MemoryStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/MemoryStore.java?rev=1470404&r1=1470403&r2=1470404&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/MemoryStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/MemoryStore.java Mon Apr 22 07:51:28 2013
@@ -19,6 +19,7 @@ package org.apache.jackrabbit.oak.plugin
import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
+import java.nio.ByteBuffer;
import java.util.Collections;
import java.util.Map;
import java.util.UUID;
@@ -76,7 +77,9 @@ public class MemoryStore implements Segm
UUID segmentId, byte[] data, int offset, int length) {
byte[] segment = new byte[length];
System.arraycopy(data, offset, segment, 0, length);
- createSegment(new Segment(this, segmentId, segment, Collections.<UUID>emptySet()));
+ createSegment(new Segment(
+ this, segmentId, ByteBuffer.wrap(segment),
+ Collections.<UUID>emptySet()));
}
@Override
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/MongoStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/MongoStore.java?rev=1470404&r1=1470403&r2=1470404&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/MongoStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/MongoStore.java Mon Apr 22 07:51:28 2013
@@ -22,6 +22,7 @@ import static com.mongodb.ReadPreference
import static com.mongodb.ReadPreference.primary;
import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
+import java.nio.ByteBuffer;
import java.util.List;
import java.util.Map;
import java.util.UUID;
@@ -89,10 +90,9 @@ public class MongoStore implements Segme
@Override
public void createSegment(Segment segment) {
cache.addSegment(segment);
- insertSegment(
- segment.getSegmentId(),
- segment.getData(),
- segment.getUUIDs());
+ byte[] bytes = new byte[segment.size()];
+ segment.getData().duplicate().get(bytes);
+ insertSegment(segment.getSegmentId(), bytes, segment.getUUIDs());
}
@Override
@@ -125,7 +125,7 @@ public class MongoStore implements Segme
for (Object object : list) {
uuids.add(UUID.fromString(object.toString()));
}
- return new Segment(this, segmentId, data, uuids);
+ return new Segment(this, segmentId, ByteBuffer.wrap(data), uuids);
}
private void insertSegment(UUID segmentId, byte[] data, UUID[] uuids) {
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java?rev=1470404&r1=1470403&r2=1470404&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java Mon Apr 22 07:51:28 2013
@@ -20,6 +20,7 @@ import static com.google.common.base.Pre
import static com.google.common.base.Preconditions.checkPositionIndexes;
import static org.apache.jackrabbit.oak.plugins.segment.SegmentWriter.BLOCK_SIZE;
+import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.Collection;
import java.util.Map;
@@ -94,7 +95,7 @@ class Segment {
private final UUID uuid;
- private final byte[] data;
+ private final ByteBuffer data;
private final UUID[] uuids;
@@ -103,7 +104,7 @@ class Segment {
private final OffsetCache<Template> templates;
Segment(SegmentStore store,
- UUID uuid, byte[] data, Collection<UUID> uuids) {
+ UUID uuid, ByteBuffer data, Collection<UUID> uuids) {
this.store = checkNotNull(store);
this.uuid = checkNotNull(uuid);
this.data = checkNotNull(data);
@@ -127,7 +128,7 @@ class Segment {
Map<String, RecordId> strings, Map<Template, RecordId> templates) {
this.store = store;
this.uuid = uuid;
- this.data = data;
+ this.data = ByteBuffer.wrap(data);
this.uuids = uuids.toArray(new UUID[uuids.size()]);
this.strings = new OffsetCache<String>(strings) {
@Override
@@ -153,8 +154,8 @@ class Segment {
* @return position within the data array
*/
private int pos(int offset, int length) {
- int pos = offset - (MAX_SEGMENT_SIZE - data.length);
- checkPositionIndexes(pos, pos + length, data.length);
+ int pos = offset - (MAX_SEGMENT_SIZE - size());
+ checkPositionIndexes(pos, pos + length, size());
return pos;
}
@@ -162,7 +163,7 @@ class Segment {
return uuid;
}
- public byte[] getData() {
+ public ByteBuffer getData() {
return data;
}
@@ -171,11 +172,11 @@ class Segment {
}
public int size() {
- return data.length;
+ return data.limit();
}
byte readByte(int offset) {
- return data[pos(offset, 1)];
+ return data.get(pos(offset, 1));
}
/**
@@ -190,7 +191,9 @@ class Segment {
void readBytes(int position, byte[] buffer, int offset, int length) {
checkNotNull(buffer);
checkPositionIndexes(offset, offset + length, buffer.length);
- System.arraycopy(data, pos(position, length), buffer, offset, length);
+ ByteBuffer d = data.duplicate();
+ d.position(pos(position, length));
+ d.get(buffer, offset, length);
}
RecordId readRecordId(int offset) {
@@ -200,17 +203,17 @@ class Segment {
private RecordId internalReadRecordId(int pos) {
return new RecordId(
- uuids[data[pos] & 0xff],
- (data[pos + 1] & 0xff) << (8 + Segment.RECORD_ALIGN_BITS)
- | (data[pos + 2] & 0xff) << Segment.RECORD_ALIGN_BITS);
+ uuids[data.get(pos) & 0xff],
+ (data.get(pos + 1) & 0xff) << (8 + Segment.RECORD_ALIGN_BITS)
+ | (data.get(pos + 2) & 0xff) << Segment.RECORD_ALIGN_BITS);
}
int readInt(int offset) {
int pos = pos(offset, 4);
- return (data[pos] & 0xff) << 24
- | (data[pos + 1] & 0xff) << 16
- | (data[pos + 2] & 0xff) << 8
- | (data[pos + 3] & 0xff);
+ return (data.get(pos) & 0xff) << 24
+ | (data.get(pos + 1) & 0xff) << 16
+ | (data.get(pos + 2) & 0xff) << 8
+ | (data.get(pos + 3) & 0xff);
}
String readString(int offset) {
@@ -230,9 +233,17 @@ class Segment {
int pos = pos(offset, 1);
long length = internalReadLength(pos);
if (length < SMALL_LIMIT) {
- return new String(data, pos + 1, (int) length, Charsets.UTF_8);
+ byte[] bytes = new byte[(int) length];
+ ByteBuffer buffer = data.duplicate();
+ buffer.position(pos + 1);
+ buffer.get(bytes);
+ return new String(bytes, Charsets.UTF_8);
} else if (length < MEDIUM_LIMIT) {
- return new String(data, pos + 2, (int) length, Charsets.UTF_8);
+ byte[] bytes = new byte[(int) length];
+ ByteBuffer buffer = data.duplicate();
+ buffer.position(pos + 2);
+ buffer.get(bytes);
+ return new String(bytes, Charsets.UTF_8);
} else if (length < Integer.MAX_VALUE) {
int size = (int) ((length + BLOCK_SIZE - 1) / BLOCK_SIZE);
ListRecord list =
@@ -321,22 +332,22 @@ class Segment {
}
private long internalReadLength(int pos) {
- int length = data[pos++] & 0xff;
+ int length = data.get(pos++) & 0xff;
if ((length & 0x80) == 0) {
return length;
} else if ((length & 0x40) == 0) {
return ((length & 0x3f) << 8
- | data[pos++] & 0xff)
+ | data.get(pos++) & 0xff)
+ SMALL_LIMIT;
} else {
return (((long) length & 0x3f) << 56
- | ((long) (data[pos++] & 0xff)) << 48
- | ((long) (data[pos++] & 0xff)) << 40
- | ((long) (data[pos++] & 0xff)) << 32
- | ((long) (data[pos++] & 0xff)) << 24
- | ((long) (data[pos++] & 0xff)) << 16
- | ((long) (data[pos++] & 0xff)) << 8
- | ((long) (data[pos++] & 0xff)))
+ | ((long) (data.get(pos++) & 0xff)) << 48
+ | ((long) (data.get(pos++) & 0xff)) << 40
+ | ((long) (data.get(pos++) & 0xff)) << 32
+ | ((long) (data.get(pos++) & 0xff)) << 24
+ | ((long) (data.get(pos++) & 0xff)) << 16
+ | ((long) (data.get(pos++) & 0xff)) << 8
+ | ((long) (data.get(pos++) & 0xff)))
+ MEDIUM_LIMIT;
}
}
@@ -347,11 +358,13 @@ class Segment {
long length = internalReadLength(pos);
if (length < Segment.MEDIUM_LIMIT) {
byte[] inline = new byte[(int) length];
+ ByteBuffer buffer = data.duplicate();
if (length < Segment.SMALL_LIMIT) {
- System.arraycopy(data, pos + 1, inline, 0, inline.length);
+ buffer.position(pos + 1);
} else {
- System.arraycopy(data, pos + 2, inline, 0, inline.length);
+ buffer.position(pos + 2);
}
+ buffer.get(inline);
return new SegmentStream(id, inline);
} else {
int size = (int) ((length + BLOCK_SIZE - 1) / BLOCK_SIZE);
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentSizeTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentSizeTest.java?rev=1470404&r1=1470403&r2=1470404&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentSizeTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentSizeTest.java Mon Apr 22 07:51:28 2013
@@ -26,7 +26,6 @@ import org.apache.jackrabbit.oak.api.Typ
import org.apache.jackrabbit.oak.plugins.memory.PropertyStates;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeState;
-import org.apache.jackrabbit.util.ISO8601;
import org.junit.Test;
import com.google.common.collect.ImmutableList;
@@ -155,14 +154,14 @@ public class SegmentSizeTest {
SegmentNodeState state = writer.writeNode(builder.getNodeState());
writer.flush();
Segment segment = store.readSegment(state.getRecordId().getSegmentId());
- assertEquals(26788, segment.getData().length);
+ assertEquals(26788, segment.size());
builder = state.builder();
builder.child("child1000");
state = writer.writeNode(builder.getNodeState());
writer.flush();
segment = store.readSegment(state.getRecordId().getSegmentId());
- assertEquals(256, segment.getData().length);
+ assertEquals(256, segment.size());
}
private int getSize(NodeBuilder builder) {
@@ -171,7 +170,7 @@ public class SegmentSizeTest {
RecordId id = writer.writeNode(builder.getNodeState()).getRecordId();
writer.flush();
Segment segment = store.readSegment(id.getSegmentId());
- return segment.getData().length;
+ return segment.size();
}
private int getAmortizedSize(NodeBuilder builder) {
@@ -180,12 +179,12 @@ public class SegmentSizeTest {
NodeState state = builder.getNodeState();
RecordId id = writer.writeNode(state).getRecordId();
writer.flush();
- int base = store.readSegment(id.getSegmentId()).getData().length;
+ int base = store.readSegment(id.getSegmentId()).size();
writer.writeNode(state);
id = writer.writeNode(state).getRecordId();
writer.flush();
Segment segment = store.readSegment(id.getSegmentId());
- return segment.getData().length - base;
+ return segment.size() - base;
}
}