You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ju...@apache.org on 2013/10/25 18:27:18 UTC
svn commit: r1535774 - in /jackrabbit/oak/trunk/oak-core/src:
main/java/org/apache/jackrabbit/oak/plugins/segment/
main/java/org/apache/jackrabbit/oak/plugins/segment/file/
main/java/org/apache/jackrabbit/oak/plugins/segment/memory/
main/java/org/apach...
Author: jukka
Date: Fri Oct 25 16:27:18 2013
New Revision: 1535774
URL: http://svn.apache.org/r1535774
Log:
OAK-593: Segment-based MK
Inline the list of identifiers of referenced segments into the
segment byte sequence. This avoids the need to keep the identifiers
in memory when they're not being accessed.
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Record.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/memory/MemoryStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/mongo/MongoStore.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentSizeTest.java
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Record.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Record.java?rev=1535774&r1=1535773&r2=1535774&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Record.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Record.java Fri Oct 25 16:27:18 2013
@@ -20,8 +20,6 @@ import static com.google.common.base.Obj
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
-import java.nio.ByteBuffer;
-import java.util.Collections;
import java.util.UUID;
import javax.annotation.Nonnull;
@@ -79,13 +77,6 @@ class Record {
this.offset = offset;
}
- // TODO: remove this ugly hack
- protected Record(SegmentStore store, RecordId id) {
- this(new Segment(
- store, UUID.randomUUID(),
- ByteBuffer.allocate(0), Collections.<UUID>emptyList()), id);
- }
-
/**
* Returns the segment that contains this record.
*
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java?rev=1535774&r1=1535773&r2=1535774&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java Fri Oct 25 16:27:18 2013
@@ -24,7 +24,7 @@ import static org.apache.jackrabbit.oak.
import java.nio.ByteBuffer;
import java.util.Arrays;
-import java.util.List;
+import java.util.Collection;
import java.util.UUID;
import java.util.concurrent.Callable;
@@ -97,14 +97,29 @@ public class Segment {
private final ByteBuffer data;
- private final List<UUID> uuids;
+ public Segment(SegmentStore store, UUID uuid, ByteBuffer data) {
+ this.store = checkNotNull(store);
+ this.uuid = checkNotNull(uuid);
+ this.data = checkNotNull(data);
+ }
public Segment(
- SegmentStore store, UUID uuid, ByteBuffer data, List<UUID> uuids) {
+ SegmentStore store, UUID uuid, Collection<UUID> refids,
+ byte[] buffer, int offset, int length) {
this.store = checkNotNull(store);
this.uuid = checkNotNull(uuid);
- this.data = checkNotNull(data);
- this.uuids = checkNotNull(uuids);
+
+ checkNotNull(refids);
+ checkNotNull(buffer);
+ checkPositionIndexes(offset, offset + length, buffer.length);
+
+ this.data = ByteBuffer.allocate(refids.size() * 16 + length);
+ for (UUID refid : refids) {
+ data.putLong(refid.getMostSignificantBits());
+ data.putLong(refid.getLeastSignificantBits());
+ }
+ data.put(buffer, offset, length);
+ data.rewind();
}
/**
@@ -155,7 +170,6 @@ public class Segment {
return getSegment(checkNotNull(id).getSegmentId());
}
-
/**
* Reads the given number of bytes starting from the given position
* in this segment.
@@ -179,10 +193,14 @@ public class Segment {
}
private RecordId internalReadRecordId(int pos) {
- return new RecordId(
- uuids.get(data.get(pos) & 0xff),
- (data.get(pos + 1) & 0xff) << (8 + Segment.RECORD_ALIGN_BITS)
- | (data.get(pos + 2) & 0xff) << Segment.RECORD_ALIGN_BITS);
+ int refpos = data.position() + (data.get(pos) & 0xff) * 16;
+ UUID refid = new UUID(data.getLong(refpos), data.getLong(refpos + 8));
+
+ int offset =
+ (((data.get(pos + 1) & 0xff) << 8) | (data.get(pos + 2) & 0xff))
+ << RECORD_ALIGN_BITS;
+
+ return new RecordId(refid, offset);
}
int readInt(int offset) {
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentStore.java?rev=1535774&r1=1535773&r2=1535774&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentStore.java Fri Oct 25 16:27:18 2013
@@ -16,7 +16,7 @@
*/
package org.apache.jackrabbit.oak.plugins.segment;
-import java.util.List;
+import java.util.Collection;
import java.util.UUID;
import java.util.concurrent.Callable;
@@ -32,14 +32,14 @@ public interface SegmentStore {
* Writes the given segment to the segment store.
*
* @param segmentId segment identifier
+ * @param referencedSegmentIds identifiers of all the referenced segments
* @param bytes byte buffer that contains the raw contents of the segment
* @param offset start offset within the byte buffer
* @param length length of the segment
- * @param referencedSegmentIds identifiers of all the referenced segments
*/
void writeSegment(
- UUID segmentId, byte[] bytes, int offset, int length,
- List<UUID> referencedSegmentIds);
+ UUID segmentId, Collection<UUID> referencedSegmentIds,
+ byte[] bytes, int offset, int length);
void deleteSegment(UUID segmentId);
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java?rev=1535774&r1=1535773&r2=1535774&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java Fri Oct 25 16:27:18 2013
@@ -23,7 +23,6 @@ import static com.google.common.base.Pre
import static com.google.common.base.Preconditions.checkPositionIndex;
import static com.google.common.base.Preconditions.checkPositionIndexes;
import static com.google.common.base.Preconditions.checkState;
-import static com.google.common.collect.Lists.newArrayList;
import static com.google.common.collect.Maps.newHashMap;
import static java.util.Collections.emptyMap;
import static org.apache.jackrabbit.oak.api.Type.NAME;
@@ -34,7 +33,6 @@ import static org.apache.jackrabbit.oak.
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -118,18 +116,16 @@ public class SegmentWriter {
public SegmentWriter(SegmentStore store) {
this.store = store;
this.dummySegment = new Segment(
- store, UUID.randomUUID(),
- ByteBuffer.allocate(0), Collections.<UUID>emptyList());
+ store, UUID.randomUUID(), Collections.<UUID>emptyList(),
+ new byte[0], 0, 0);
}
public synchronized Segment getCurrentSegment(UUID id) {
if (equal(id, uuid)) {
if (currentSegment == null) {
- ByteBuffer b = ByteBuffer.allocate(length);
- System.arraycopy(
- buffer, buffer.length - length, b.array(), 0, length);
currentSegment = new Segment(
- store, uuid, b, newArrayList(uuids.keySet()));
+ store, uuid, uuids.keySet(),
+ buffer, buffer.length - length, length);
}
return currentSegment;
} else {
@@ -144,8 +140,8 @@ public class SegmentWriter {
public synchronized void flush() {
if (length > 0) {
store.writeSegment(
- uuid, buffer, buffer.length - length, length,
- newArrayList(uuids.keySet()));
+ uuid, uuids.keySet(),
+ buffer, buffer.length - length, length);
uuid = UUID.randomUUID();
uuids.clear();
@@ -556,8 +552,8 @@ public class SegmentWriter {
int align = Segment.RECORD_ALIGN_BYTES - 1;
int bulkAlignLength = (bulkLength + align) & ~align;
store.writeSegment(
- segmentId, bulk, 0, bulkAlignLength,
- Collections.<UUID>emptyList());
+ segmentId, Collections.<UUID>emptyList(),
+ bulk, 0, bulkAlignLength);
for (int pos = Segment.MAX_SEGMENT_SIZE - bulkAlignLength;
pos < Segment.MAX_SEGMENT_SIZE;
pos += BLOCK_SIZE) {
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java?rev=1535774&r1=1535773&r2=1535774&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java Fri Oct 25 16:27:18 2013
@@ -19,7 +19,6 @@ package org.apache.jackrabbit.oak.plugin
import static com.google.common.base.Charsets.UTF_8;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
-import static com.google.common.collect.Lists.newArrayListWithCapacity;
import static com.google.common.collect.Lists.newLinkedList;
import static com.google.common.collect.Maps.newHashMap;
import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
@@ -27,8 +26,8 @@ import static org.apache.jackrabbit.oak.
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.Collection;
import java.util.LinkedList;
-import java.util.List;
import java.util.Map;
import java.util.UUID;
@@ -136,21 +135,8 @@ public class FileStore extends AbstractS
if (buffer != null) {
checkState(SEGMENT_MAGIC == buffer.getLong());
int length = buffer.getInt();
- int count = buffer.getInt();
-
- checkState(id.equals(new UUID(
- buffer.getLong(), buffer.getLong())));
-
- List<UUID> referencedIds = newArrayListWithCapacity(count);
- for (int i = 0; i < count; i++) {
- referencedIds.add(new UUID(
- buffer.getLong(), buffer.getLong()));
- }
-
buffer.limit(buffer.position() + length);
- return new Segment(
- FileStore.this, id,
- buffer.slice(), referencedIds);
+ return new Segment(FileStore.this, id, buffer.slice());
}
}
@@ -159,22 +145,19 @@ public class FileStore extends AbstractS
@Override
public synchronized void writeSegment(
- UUID segmentId, byte[] data, int offset, int length,
- List<UUID> referencedSegmentIds) {
- int size = 8 + 4 + 4 + 16 + 16 * referencedSegmentIds.size() + length;
+ UUID segmentId, Collection<UUID> referencedSegmentIds,
+ byte[] data, int offset, int length) {
+ int size = 8 + 4 + 16 * referencedSegmentIds.size() + length;
ByteBuffer buffer = ByteBuffer.allocate(size);
buffer.putLong(SEGMENT_MAGIC);
- buffer.putInt(length);
- buffer.putInt(referencedSegmentIds.size());
- buffer.putLong(segmentId.getMostSignificantBits());
- buffer.putLong(segmentId.getLeastSignificantBits());
+ buffer.putInt(16 * referencedSegmentIds.size() + length);
+
for (UUID referencedSegmentId : referencedSegmentIds) {
buffer.putLong(referencedSegmentId.getMostSignificantBits());
buffer.putLong(referencedSegmentId.getLeastSignificantBits());
}
- int pos = buffer.position();
buffer.put(data, offset, length);
try {
@@ -182,8 +165,6 @@ public class FileStore extends AbstractS
} catch (IOException e) {
throw new RuntimeException(e);
}
-
- buffer.position(pos);
}
private void writeEntry(UUID segmentId, byte[] buffer)
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/memory/MemoryStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/memory/MemoryStore.java?rev=1535774&r1=1535773&r2=1535774&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/memory/MemoryStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/memory/MemoryStore.java Fri Oct 25 16:27:18 2013
@@ -18,8 +18,7 @@ package org.apache.jackrabbit.oak.plugin
import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
-import java.nio.ByteBuffer;
-import java.util.List;
+import java.util.Collection;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ConcurrentMap;
@@ -30,7 +29,6 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeState;
-import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
public class MemoryStore extends AbstractStore {
@@ -77,13 +75,10 @@ public class MemoryStore extends Abstrac
@Override
public void writeSegment(
- UUID segmentId, byte[] data, int offset, int length,
- List<UUID> referencedSegmentIds) {
- byte[] buffer = new byte[length];
- System.arraycopy(data, offset, buffer, 0, length);
+ UUID segmentId, Collection<UUID> referencedSegmentIds,
+ byte[] data, int offset, int length) {
Segment segment = new Segment(
- this, segmentId, ByteBuffer.wrap(buffer),
- Lists.newArrayList(referencedSegmentIds));
+ this, segmentId, referencedSegmentIds, data, offset, length);
if (segments.putIfAbsent(segment.getSegmentId(), segment) != null) {
throw new IllegalStateException(
"Segment override: " + segment.getSegmentId());
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/mongo/MongoStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/mongo/MongoStore.java?rev=1535774&r1=1535773&r2=1535774&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/mongo/MongoStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/mongo/MongoStore.java Fri Oct 25 16:27:18 2013
@@ -22,7 +22,6 @@ import static com.mongodb.ReadPreference
import static com.mongodb.ReadPreference.primary;
import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
-import java.nio.ByteBuffer;
import java.util.Collection;
import java.util.List;
import java.util.Map;
@@ -84,8 +83,8 @@ public class MongoStore extends Abstract
@Override
public void writeSegment(
- UUID segmentId, byte[] data, int offset, int length,
- List<UUID> referencedSegmentIds) {
+ UUID segmentId, Collection<UUID> referencedSegmentIds,
+ byte[] data, int offset, int length) {
byte[] d = new byte[length];
System.arraycopy(data, offset, d, 0, length);
insertSegment(segmentId, d, referencedSegmentIds);
@@ -111,7 +110,7 @@ public class MongoStore extends Abstract
for (Object object : list) {
uuids.add(UUID.fromString(object.toString()));
}
- return new Segment(this, segmentId, ByteBuffer.wrap(data), uuids);
+ return new Segment(this, segmentId, uuids, data, 0, data.length);
}
private void insertSegment(
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentSizeTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentSizeTest.java?rev=1535774&r1=1535773&r2=1535774&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentSizeTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentSizeTest.java Fri Oct 25 16:27:18 2013
@@ -40,29 +40,29 @@ public class SegmentSizeTest {
@Test
public void testNodeSize() {
NodeBuilder builder = EMPTY_NODE.builder();
- assertEquals(4, getSize(builder));
+ assertEquals(20, getSize(builder));
assertEquals(4, getAmortizedSize(builder));
builder = EMPTY_NODE.builder();
builder.setProperty("foo", "bar");
- assertEquals(24, getSize(builder));
+ assertEquals(40, getSize(builder));
assertEquals(8, getAmortizedSize(builder));
builder = EMPTY_NODE.builder();
builder.setProperty("foo", "bar");
builder.setProperty("baz", 123);
- assertEquals(40, getSize(builder));
+ assertEquals(56, getSize(builder));
assertEquals(12, getAmortizedSize(builder));
builder = EMPTY_NODE.builder();
builder.child("foo");
- assertEquals(24, getSize(builder));
+ assertEquals(56, getSize(builder));
assertEquals(12, getAmortizedSize(builder));
builder = EMPTY_NODE.builder();
builder.child("foo");
builder.child("bar");
- assertEquals(52, getSize(builder));
+ assertEquals(84, getSize(builder));
assertEquals(40, getAmortizedSize(builder));
}
@@ -106,7 +106,7 @@ public class SegmentSizeTest {
public void testAccessControlNodes() {
NodeBuilder builder = EMPTY_NODE.builder();
builder.setProperty("jcr:primaryType", "rep:ACL", Type.NAME);
- assertEquals(20, getSize(builder));
+ assertEquals(36, getSize(builder));
assertEquals(4, getAmortizedSize(builder));
NodeBuilder deny = builder.child("deny");
@@ -114,7 +114,7 @@ public class SegmentSizeTest {
deny.setProperty("rep:principalName", "everyone");
deny.setProperty(PropertyStates.createProperty(
"rep:privileges", ImmutableList.of("jcr:read"), Type.NAMES));
- assertEquals(144, getSize(builder));
+ assertEquals(160, getSize(builder));
assertEquals(28, getAmortizedSize(builder));
NodeBuilder allow = builder.child("allow");
@@ -122,7 +122,7 @@ public class SegmentSizeTest {
allow.setProperty("rep:principalName", "administrators");
allow.setProperty(PropertyStates.createProperty(
"rep:privileges", ImmutableList.of("jcr:all"), Type.NAMES));
- assertEquals(264, getSize(builder));
+ assertEquals(280, getSize(builder));
assertEquals(72, getAmortizedSize(builder));
NodeBuilder deny0 = builder.child("deny0");
@@ -131,7 +131,7 @@ public class SegmentSizeTest {
deny0.setProperty("rep:glob", "*/activities/*");
builder.setProperty(PropertyStates.createProperty(
"rep:privileges", ImmutableList.of("jcr:read"), Type.NAMES));
- assertEquals(356, getSize(builder));
+ assertEquals(372, getSize(builder));
assertEquals(108, getAmortizedSize(builder));
NodeBuilder allow0 = builder.child("allow0");
@@ -139,7 +139,7 @@ public class SegmentSizeTest {
allow0.setProperty("rep:principalName", "user-administrators");
allow0.setProperty(PropertyStates.createProperty(
"rep:privileges", ImmutableList.of("jcr:all"), Type.NAMES));
- assertEquals(412, getSize(builder));
+ assertEquals(428, getSize(builder));
assertEquals(136, getAmortizedSize(builder));
}
@@ -155,7 +155,7 @@ public class SegmentSizeTest {
SegmentNodeState state = writer.writeNode(builder.getNodeState());
Segment segment = store.readSegment(state.getRecordId().getSegmentId());
- assertEquals(26728, Segment.WEIGHER.weigh(null, segment));
+ assertEquals(26760, Segment.WEIGHER.weigh(null, segment));
writer.flush(); // force flushing of the previous segment
@@ -163,7 +163,7 @@ public class SegmentSizeTest {
builder.child("child1000");
state = writer.writeNode(builder.getNodeState());
segment = store.readSegment(state.getRecordId().getSegmentId());
- assertEquals(96, Segment.WEIGHER.weigh(null, segment));
+ assertEquals(144, Segment.WEIGHER.weigh(null, segment));
}
private int getSize(NodeBuilder builder) {