You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by fr...@apache.org on 2016/08/23 15:38:15 UTC
svn commit: r1757390 - in /jackrabbit/oak/trunk/oak-segment-tar/src:
main/java/org/apache/jackrabbit/oak/segment/
main/java/org/apache/jackrabbit/oak/segment/file/
main/java/org/apache/jackrabbit/oak/segment/tool/
test/java/org/apache/jackrabbit/oak/se...
Author: frm
Date: Tue Aug 23 15:38:14 2016
New Revision: 1757390
URL: http://svn.apache.org/viewvc?rev=1757390&view=rev
Log:
OAK-4631 - Simplify the format of segments and serialized records
Added:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/BinaryUtils.java (with props)
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentReferencesTest.java (with props)
Removed:
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/RecordUsageAnalyserTest.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentSizeTest.java
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarWriter.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/DebugStore.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentIdFactoryTest.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentParserTest.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/TarFileTest.java
Added: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/BinaryUtils.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/BinaryUtils.java?rev=1757390&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/BinaryUtils.java (added)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/BinaryUtils.java Tue Aug 23 15:38:14 2016
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jackrabbit.oak.segment;
+
+class BinaryUtils {
+
+ private BinaryUtils() {
+ // Prevent instantiation
+ }
+
+ static int writeByte(byte[] buffer, int position, byte value) {
+ buffer[position++] = value;
+ return position;
+ }
+
+ static int writeShort(byte[] buffer, int position, short value) {
+ position = writeByte(buffer, position, (byte) (value >> 8));
+ position = writeByte(buffer, position, (byte) (value));
+ return position;
+ }
+
+ static int writeInt(byte[] buffer, int position, int value) {
+ position = writeShort(buffer, position, (short) (value >> 16));
+ position = writeShort(buffer, position, (short) (value));
+ return position;
+ }
+
+ static int writeLong(byte[] buffer, int position, long value) {
+ position = writeInt(buffer, position, (int) (value >> 32));
+ position = writeInt(buffer, position, (int) (value));
+ return position;
+ }
+
+}
Propchange: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/BinaryUtils.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java?rev=1757390&r1=1757389&r2=1757390&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java Tue Aug 23 15:38:14 2016
@@ -22,7 +22,7 @@ import static com.google.common.base.Pre
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkPositionIndexes;
import static com.google.common.base.Preconditions.checkState;
-import static com.google.common.collect.Lists.newArrayListWithCapacity;
+import static com.google.common.collect.Maps.newHashMap;
import static org.apache.jackrabbit.oak.commons.IOUtils.closeQuietly;
import static org.apache.jackrabbit.oak.segment.SegmentId.isDataSegmentId;
import static org.apache.jackrabbit.oak.segment.SegmentVersion.LATEST_VERSION;
@@ -34,7 +34,7 @@ import java.io.PrintWriter;
import java.io.StringWriter;
import java.nio.ByteBuffer;
import java.util.Arrays;
-import java.util.List;
+import java.util.Map;
import java.util.UUID;
import javax.annotation.CheckForNull;
@@ -57,12 +57,14 @@ import org.apache.jackrabbit.oak.plugins
*/
public class Segment {
+ static final int HEADER_SIZE = 18;
+
/**
* Number of bytes used for storing a record identifier. One byte
* is used for identifying the segment and two for the record offset
* within that segment.
*/
- static final int RECORD_ID_BYTES = 1 + 2;
+ static final int RECORD_ID_BYTES = 8 + 8 + 2;
/**
* The limit on segment references within one segment. Since record
@@ -111,12 +113,12 @@ public class Segment {
*/
public static final int BLOB_ID_SMALL_LIMIT = 1 << 12;
- public static final int REF_COUNT_OFFSET = 5;
-
static final int ROOT_COUNT_OFFSET = 6;
public static final int GC_GENERATION_OFFSET = 10;
+ public static final int REFERENCED_SEGMENT_ID_COUNT_OFFSET = 14;
+
@Nonnull
private final SegmentStore store;
@@ -135,12 +137,7 @@ public class Segment {
@Nonnull
private final SegmentVersion version;
- /**
- * Referenced segment identifiers. Entries are initialized lazily in
- * {@link #getRefId(int)}. Set to {@code null} for bulk segments.
- */
- @CheckForNull
- private final SegmentId[] refids;
+ private final Map<Integer, RecordId> recordIdCache = newHashMap();
/**
* Unpacks a 4 byte aligned segment offset.
@@ -193,11 +190,8 @@ public class Segment {
+ toHex(data.array());
}
});
- this.refids = new SegmentId[getRefCount()];
- this.refids[0] = id;
this.version = SegmentVersion.fromByte(segmentVersion);
} else {
- this.refids = null;
this.version = LATEST_VERSION;
}
}
@@ -223,8 +217,6 @@ public class Segment {
this.id = store.newDataSegmentId();
this.info = checkNotNull(info);
this.data = ByteBuffer.wrap(checkNotNull(buffer));
- this.refids = new SegmentId[SEGMENT_REFERENCE_LIMIT + 1];
- this.refids[0] = id;
this.version = SegmentVersion.fromByte(buffer[3]);
id.loaded(this);
}
@@ -253,14 +245,28 @@ public class Segment {
return id;
}
- int getRefCount() {
- return (data.get(REF_COUNT_OFFSET) & 0xff) + 1;
- }
-
public int getRootCount() {
return data.getShort(ROOT_COUNT_OFFSET) & 0xffff;
}
+ public int getReferencedSegmentIdCount() {
+ return data.getInt(REFERENCED_SEGMENT_ID_COUNT_OFFSET);
+ }
+
+ public UUID getReferencedSegmentId(int index) {
+ checkArgument(index < getReferencedSegmentIdCount());
+
+ int position = data.position();
+
+ position += HEADER_SIZE;
+ position += index * 16;
+
+ long msb = data.getLong(position);
+ long lsb = data.getLong(position + 8);
+
+ return new UUID(msb, lsb);
+ }
+
/**
* Determine the gc generation a segment from its data. Note that bulk segments don't have
* generations (i.e. stay at 0).
@@ -285,16 +291,28 @@ public class Segment {
}
public RecordType getRootType(int index) {
- int refCount = getRefCount();
checkArgument(index < getRootCount());
- return RecordType.values()[data.get(data.position() + refCount * 16 + index * 3) & 0xff];
+
+ int position = data.position();
+
+ position += HEADER_SIZE;
+ position += getReferencedSegmentIdCount() * 16;
+ position += index * 3;
+
+ return RecordType.values()[data.get(position) & 0xff];
}
public int getRootOffset(int index) {
- int refCount = getRefCount();
checkArgument(index < getRootCount());
- return (data.getShort(data.position() + refCount * 16 + index * 3 + 1) & 0xffff)
- << RECORD_ALIGN_BITS;
+
+ int position = data.position();
+
+ position += HEADER_SIZE;
+ position += getReferencedSegmentIdCount() * 16;
+ position += index * 3;
+ position += 1;
+
+ return (data.getShort(position) & 0xffff) << RECORD_ALIGN_BITS;
}
private volatile String info;
@@ -316,48 +334,12 @@ public class Segment {
*/
@CheckForNull
public String getSegmentInfo() {
- if (info == null && getRefCount() != 0) {
+ if (info == null && id.isDataSegmentId()) {
info = readString(getRootOffset(0));
}
return info;
}
- SegmentId getRefId(int index) {
- if (refids == null || index >= refids.length) {
- String type = "data";
- if (!id.isDataSegmentId()) {
- type = "bulk";
- }
- long delta = System.currentTimeMillis() - id.getCreationTime();
- throw new IllegalStateException("RefId '" + index
- + "' doesn't exist in " + type + " segment " + id
- + ". Creation date delta is " + delta + " ms.");
- }
- SegmentId refid = refids[index];
- if (refid == null) {
- synchronized (this) {
- refid = refids[index];
- if (refid == null) {
- int refpos = data.position() + index * 16;
- long msb = data.getLong(refpos);
- long lsb = data.getLong(refpos + 8);
- refid = store.newSegmentId(msb, lsb);
- refids[index] = refid;
- }
- }
- }
- return refid;
- }
-
- public List<SegmentId> getReferencedIds() {
- int refcount = getRefCount();
- List<SegmentId> ids = newArrayListWithCapacity(refcount);
- for (int refid = 0; refid < refcount; refid++) {
- ids.add(getRefId(refid));
- }
- return ids;
- }
-
public int size() {
return data.remaining();
}
@@ -401,9 +383,28 @@ public class Segment {
}
private RecordId internalReadRecordId(int pos) {
- SegmentId refid = getRefId(data.get(pos) & 0xff);
- int offset = ((data.get(pos + 1) & 0xff) << 8) | (data.get(pos + 2) & 0xff);
- return new RecordId(refid, offset << RECORD_ALIGN_BITS);
+ RecordId recordId = recordIdCache.get(pos);
+
+ if (recordId != null) {
+ return recordId;
+ }
+
+ synchronized (recordIdCache) {
+ recordId = recordIdCache.get(pos);
+
+ if (recordId != null) {
+ return recordId;
+ }
+
+ long msb = data.getLong(pos);
+ long lsb = data.getLong(pos + 8);
+ int offset = (data.getShort(pos + 16) & 0xffff) << RECORD_ALIGN_BITS;
+ recordId = new RecordId(store.newSegmentId(msb, lsb), offset);
+
+ recordIdCache.put(pos, recordId);
+
+ return recordId;
+ }
}
@Nonnull
@@ -538,17 +539,13 @@ public class Segment {
}
if (id.isDataSegmentId()) {
writer.println("--------------------------------------------------------------------------");
- int refcount = getRefCount();
- for (int refid = 0; refid < refcount; refid++) {
- writer.format("reference %02x: %s%n", refid, getRefId(refid));
+
+ for (int i = 0; i < getReferencedSegmentIdCount(); i++) {
+ writer.format("reference %02x: %s%n", i, getReferencedSegmentId(i));
}
- int rootcount = data.getShort(ROOT_COUNT_OFFSET) & 0xffff;
- int pos = data.position() + refcount * 16;
- for (int rootid = 0; rootid < rootcount; rootid++) {
- writer.format(
- "root %d: %s at %04x%n", rootid,
- RecordType.values()[data.get(pos + rootid * 3) & 0xff],
- data.getShort(pos + rootid * 3 + 1) & 0xffff);
+
+ for (int i = 0; i < getRootCount(); i++) {
+ writer.format("root %d: %s at %04x%n", i, getRootType(i), getRootOffset(i));
}
}
writer.println("--------------------------------------------------------------------------");
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java?rev=1757390&r1=1757389&r2=1757390&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java Tue Aug 23 15:38:14 2016
@@ -29,19 +29,18 @@ import static java.lang.System.arraycopy
import static java.lang.System.currentTimeMillis;
import static java.lang.System.identityHashCode;
import static org.apache.jackrabbit.oak.segment.Segment.GC_GENERATION_OFFSET;
+import static org.apache.jackrabbit.oak.segment.Segment.HEADER_SIZE;
import static org.apache.jackrabbit.oak.segment.Segment.MAX_SEGMENT_SIZE;
import static org.apache.jackrabbit.oak.segment.Segment.RECORD_ID_BYTES;
-import static org.apache.jackrabbit.oak.segment.Segment.SEGMENT_REFERENCE_LIMIT;
import static org.apache.jackrabbit.oak.segment.Segment.align;
import static org.apache.jackrabbit.oak.segment.SegmentId.isDataSegmentId;
import static org.apache.jackrabbit.oak.segment.SegmentVersion.LATEST_VERSION;
import java.io.IOException;
-import java.nio.ByteBuffer;
import java.util.Collection;
-import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+import java.util.UUID;
import javax.annotation.CheckForNull;
import javax.annotation.Nonnull;
@@ -109,6 +108,8 @@ public class SegmentBufferWriter impleme
*/
private final Map<RecordId, RecordType> roots = newLinkedHashMap();
+ private final Set<UUID> referencedSegmentIds = newHashSet();
+
@Nonnull
private final SegmentStore store;
@@ -208,6 +209,7 @@ public class SegmentBufferWriter impleme
length = 0;
position = buffer.length;
roots.clear();
+ referencedSegmentIds.clear();
String metaInfo =
"{\"wid\":\"" + wid + '"' +
@@ -229,27 +231,23 @@ public class SegmentBufferWriter impleme
}
public void writeByte(byte value) {
- buffer[position++] = value;
+ position = BinaryUtils.writeByte(buffer, position, value);
dirty = true;
}
public void writeShort(short value) {
- buffer[position++] = (byte) (value >> 8);
- buffer[position++] = (byte) value;
+ position = BinaryUtils.writeShort(buffer, position, value);
dirty = true;
}
public void writeInt(int value) {
- buffer[position++] = (byte) (value >> 24);
- buffer[position++] = (byte) (value >> 16);
- buffer[position++] = (byte) (value >> 8);
- buffer[position++] = (byte) value;
+ position = BinaryUtils.writeInt(buffer, position, value);
dirty = true;
}
public void writeLong(long value) {
- writeInt((int) (value >> 32));
- writeInt((int) value);
+ position = BinaryUtils.writeLong(buffer, position, value);
+ dirty = true;
}
/**
@@ -279,12 +277,20 @@ public class SegmentBufferWriter impleme
}
int offset = recordId.getOffset();
+
checkState(0 <= offset && offset < MAX_SEGMENT_SIZE);
checkState(offset == align(offset, 1 << Segment.RECORD_ALIGN_BITS));
- buffer[position++] = (byte) getSegmentRef(recordId.getSegmentId());
- buffer[position++] = (byte) (offset >> (8 + Segment.RECORD_ALIGN_BITS));
- buffer[position++] = (byte) (offset >> Segment.RECORD_ALIGN_BITS);
+ long msb = recordId.getSegmentId().getMostSignificantBits();
+ long lsb = recordId.getSegmentId().getLeastSignificantBits();
+
+ writeLong(msb);
+ writeLong(lsb);
+ writeShort((short)((offset >> Segment.RECORD_ALIGN_BITS) & 0xffff));
+
+ if (!recordId.getSegmentId().equals(segment.getSegmentId())) {
+ referencedSegmentIds.add(recordId.getSegmentId().asUUID());
+ }
statistics.recordIdCount++;
@@ -317,27 +323,6 @@ public class SegmentBufferWriter impleme
return info;
}
- private int getSegmentRef(SegmentId segmentId) {
- checkGCGeneration(segmentId);
-
- int refCount = segment.getRefCount();
- if (refCount > SEGMENT_REFERENCE_LIMIT) {
- throw new SegmentOverflowException(
- "Segment cannot have more than 255 references " + segment.getSegmentId());
- }
- for (int index = 0; index < refCount; index++) {
- if (segmentId.equals(segment.getRefId(index))) {
- return index;
- }
- }
-
- ByteBuffer.wrap(buffer, refCount * 16, 16)
- .putLong(segmentId.getMostSignificantBits())
- .putLong(segmentId.getLeastSignificantBits());
- buffer[Segment.REF_COUNT_OFFSET] = (byte) refCount;
- return refCount;
- }
-
public void writeBytes(byte[] data, int offset, int length) {
arraycopy(data, offset, buffer, position, length);
position += length;
@@ -352,19 +337,22 @@ public class SegmentBufferWriter impleme
@Override
public void flush() throws IOException {
if (dirty) {
- int refcount = segment.getRefCount();
- statistics.segmentIdCount = refcount;
-
int rootcount = roots.size();
- buffer[Segment.ROOT_COUNT_OFFSET] = (byte) (rootcount >> 8);
- buffer[Segment.ROOT_COUNT_OFFSET + 1] = (byte) rootcount;
+ BinaryUtils.writeShort(buffer, Segment.ROOT_COUNT_OFFSET, (short) rootcount);
+
+ int referencedSegmentIdCount = referencedSegmentIds.size();
+ BinaryUtils.writeInt(buffer, Segment.REFERENCED_SEGMENT_ID_COUNT_OFFSET, referencedSegmentIdCount);
+ statistics.segmentIdCount = referencedSegmentIdCount;
+
+ int totalLength = align(HEADER_SIZE + referencedSegmentIdCount * 16 + rootcount * 3 + length, 16);
- length = align(refcount * 16 + rootcount * 3 + length, 16);
- statistics.size = length;
+ if (totalLength > buffer.length) {
+ throw new IllegalStateException("too much data for a segment");
+ }
- checkState(length <= buffer.length);
+ statistics.size = length = totalLength;
- int pos = refcount * 16;
+ int pos = HEADER_SIZE;
if (pos + length <= buffer.length) {
// the whole segment fits to the space *after* the referenced
// segment identifiers we've already written, so we can safely
@@ -380,11 +368,14 @@ public class SegmentBufferWriter impleme
length = buffer.length;
}
+ for (UUID id : referencedSegmentIds) {
+ pos = BinaryUtils.writeLong(buffer, pos, id.getMostSignificantBits());
+ pos = BinaryUtils.writeLong(buffer, pos, id.getLeastSignificantBits());
+ }
+
for (Map.Entry<RecordId, RecordType> entry : roots.entrySet()) {
- int offset = entry.getKey().getOffset();
- buffer[pos++] = (byte) entry.getValue().ordinal();
- buffer[pos++] = (byte) (offset >> (8 + Segment.RECORD_ALIGN_BITS));
- buffer[pos++] = (byte) (offset >> Segment.RECORD_ALIGN_BITS);
+ pos = BinaryUtils.writeByte(buffer, pos, (byte) entry.getValue().ordinal());
+ pos = BinaryUtils.writeShort(buffer, pos, (short) (entry.getKey().getOffset() >> Segment.RECORD_ALIGN_BITS));
}
SegmentId segmentId = segment.getSegmentId();
@@ -421,25 +412,27 @@ public class SegmentBufferWriter impleme
// First compute the header and segment sizes based on the assumption
// that *all* identifiers stored in this record point to previously
// unreferenced segments.
- int refCount = segment.getRefCount() + idCount;
+
int rootCount = roots.size() + 1;
- int headerSize = refCount * 16 + rootCount * 3;
+ int referencedIdCount = referencedSegmentIds.size() + ids.size();
+ int headerSize = HEADER_SIZE + rootCount * 3 + referencedIdCount * 16;
int segmentSize = align(headerSize + recordSize + length, 16);
// If the size estimate looks too big, recompute it with a more
// accurate refCount value. We skip doing this when possible to
// avoid the somewhat expensive list and set traversals.
- if (segmentSize > buffer.length - 1 || refCount > Segment.SEGMENT_REFERENCE_LIMIT) {
- refCount -= idCount;
- Set<SegmentId> segmentIds = newHashSet();
+ if (segmentSize > buffer.length) {
// The set of old record ids in this segment
// that were previously root record ids, but will no longer be,
// because the record to be written references them.
// This needs to be a set, because the list of ids can
// potentially reference the same record multiple times
- Set<RecordId> notRoots = new HashSet<RecordId>();
+
+ Set<SegmentId> segmentIds = newHashSet();
+ Set<RecordId> notRoots = newHashSet();
+
for (RecordId recordId : ids) {
SegmentId segmentId = recordId.getSegmentId();
if (!(segmentId.equals(segment.getSegmentId()))) {
@@ -450,20 +443,19 @@ public class SegmentBufferWriter impleme
}
rootCount -= notRoots.size();
- if (!segmentIds.isEmpty()) {
- for (int refid = 1; refid < refCount; refid++) {
- segmentIds.remove(segment.getRefId(refid));
+ // Adjust the estimation of the new referenced segment ID count.
+
+ for (SegmentId segmentId : segmentIds) {
+ if (referencedSegmentIds.contains(segmentId.asUUID())) {
+ referencedIdCount--;
}
- refCount += segmentIds.size();
}
- headerSize = refCount * 16 + rootCount * 3;
+ headerSize = HEADER_SIZE + rootCount * 3 + referencedIdCount * 16;
segmentSize = align(headerSize + recordSize + length, 16);
}
- if (segmentSize > buffer.length - 1
- || rootCount > 0xffff
- || refCount > Segment.SEGMENT_REFERENCE_LIMIT) {
+ if (segmentSize > buffer.length || rootCount > 0xffff) {
flush();
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java?rev=1757390&r1=1757389&r2=1757390&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java Tue Aug 23 15:38:14 2016
@@ -1386,13 +1386,54 @@ public class FileStore implements Segmen
@Override
public void writeSegment(SegmentId id, byte[] buffer, int offset, int length) throws IOException {
+ Segment segment = null;
+
+ // If the segment is a data segment, create a new instance of Segment to
+ // access some internal information stored in the segment and to store
+ // in an in-memory cache for later use.
+
+ if (id.isDataSegmentId()) {
+ ByteBuffer data;
+
+ if (offset > 4096) {
+ data = ByteBuffer.allocate(length);
+ data.put(buffer, offset, length);
+ data.rewind();
+ } else {
+ data = ByteBuffer.wrap(buffer, offset, length);
+ }
+
+ segment = new Segment(this, segmentReader, id, data);
+ }
+
fileStoreLock.writeLock().lock();
try {
int generation = Segment.getGcGeneration(wrap(buffer, offset, length), id.asUUID());
+
+ // Flush the segment to disk
+
long size = tarWriter.writeEntry(
id.getMostSignificantBits(),
id.getLeastSignificantBits(),
- buffer, offset, length, generation);
+ buffer,
+ offset,
+ length,
+ generation
+ );
+
+ // If the segment is a data segment, update the graph before
+ // (potentially) flushing the TAR file.
+
+ if (segment != null) {
+ UUID from = segment.getSegmentId().asUUID();
+
+ for (int i = 0; i < segment.getReferencedSegmentIdCount(); i++) {
+ tarWriter.addGraphEdge(from, segment.getReferencedSegmentId(i));
+ }
+ }
+
+ // Close the TAR file if the size exceeds the maximum.
+
if (size >= maxFileSize) {
newWriter();
}
@@ -1400,17 +1441,10 @@ public class FileStore implements Segmen
fileStoreLock.writeLock().unlock();
}
- // Keep this data segment in memory as it's likely to be accessed soon
- if (id.isDataSegmentId()) {
- ByteBuffer data;
- if (offset > 4096) {
- data = ByteBuffer.allocate(length);
- data.put(buffer, offset, length);
- data.rewind();
- } else {
- data = ByteBuffer.wrap(buffer, offset, length);
- }
- segmentCache.putSegment(new Segment(this, segmentReader, id, data));
+ // Keep this data segment in memory as it's likely to be accessed soon.
+
+ if (segment != null) {
+ segmentCache.putSegment(segment);
}
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java?rev=1757390&r1=1757389&r2=1757390&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java Tue Aug 23 15:38:14 2016
@@ -22,7 +22,6 @@ import static com.google.common.base.Cha
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.collect.Lists.newArrayList;
import static com.google.common.collect.Lists.newArrayListWithCapacity;
-import static com.google.common.collect.Maps.newHashMap;
import static com.google.common.collect.Maps.newHashMapWithExpectedSize;
import static com.google.common.collect.Maps.newLinkedHashMap;
import static com.google.common.collect.Maps.newTreeMap;
@@ -30,7 +29,6 @@ import static com.google.common.collect.
import static com.google.common.collect.Sets.newHashSetWithExpectedSize;
import static java.nio.ByteBuffer.wrap;
import static java.util.Collections.singletonList;
-import static org.apache.jackrabbit.oak.segment.Segment.REF_COUNT_OFFSET;
import static org.apache.jackrabbit.oak.segment.Segment.getGcGeneration;
import static org.apache.jackrabbit.oak.segment.SegmentId.isDataSegmentId;
import static org.apache.jackrabbit.oak.segment.file.TarWriter.BINARY_REFERENCES_MAGIC;
@@ -62,6 +60,7 @@ import com.google.common.collect.Sets;
import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.oak.plugins.blob.ReferenceCollector;
import org.apache.jackrabbit.oak.segment.SegmentGraph.SegmentGraphVisitor;
+import org.apache.jackrabbit.oak.segment.SegmentId;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -659,26 +658,13 @@ class TarReader implements Closeable {
@Nonnull
private List<UUID> getReferences(TarEntry entry, UUID id, Map<UUID, List<UUID>> graph) throws IOException {
- if (graph != null) {
- List<UUID> uuids = graph.get(id);
- return uuids == null ? Collections.<UUID>emptyList() : uuids;
- } else {
- // a pre-compiled graph is not available, so read the
- // references directly from this segment
- ByteBuffer segment = access.read(
- entry.offset(),
- Math.min(entry.size(), 16 * 256));
- int pos = segment.position();
- int refCount = segment.get(pos + REF_COUNT_OFFSET) & 0xff;
- int refEnd = pos + 16 * (refCount + 1);
- List<UUID> refIds = newArrayList();
- for (int refPos = pos + 16; refPos < refEnd; refPos += 16) {
- refIds.add(new UUID(
- segment.getLong(refPos),
- segment.getLong(refPos + 8)));
- }
- return refIds;
+ List<UUID> references = graph.get(id);
+
+ if (references == null) {
+ return Collections.emptyList();
}
+
+ return references;
}
/**
@@ -873,6 +859,30 @@ class TarReader implements Closeable {
}
}
+ // Reconstruct the graph index for non-cleaned segments.
+
+ Map<UUID, List<UUID>> graph = getGraph(false);
+
+ for (Entry<UUID, List<UUID>> e : graph.entrySet()) {
+ if (cleaned.contains(e.getKey())) {
+ continue;
+ }
+
+ Set<UUID> vertices = newHashSet();
+
+ for (UUID vertex : e.getValue()) {
+ if (cleaned.contains(vertex)) {
+ continue;
+ }
+
+ vertices.add(vertex);
+ }
+
+ for (UUID vertex : vertices) {
+ writer.addGraphEdge(e.getKey(), vertex);
+ }
+ }
+
// Reconstruct the binary reference index for non-cleaned segments.
Map<Integer, Map<UUID, Set<String>>> references = getBinaryReferences();
@@ -1081,68 +1091,77 @@ class TarReader implements Closeable {
* @throws IOException if the tar file could not be read
*/
private ByteBuffer loadGraph() throws IOException {
- // read the graph metadata just before the tar index entry
int pos = access.length() - 2 * BLOCK_SIZE - getIndexEntrySize();
+
ByteBuffer meta = access.read(pos - 16, 16);
+
int crc32 = meta.getInt();
int count = meta.getInt();
int bytes = meta.getInt();
int magic = meta.getInt();
if (magic != GRAPH_MAGIC) {
- return null; // magic byte mismatch
+ log.warn("Invalid graph magic number in {}", file);
+ return null;
}
- if (count < 0 || bytes < count * 16 + 16 || BLOCK_SIZE + bytes > pos) {
- log.warn("Invalid graph metadata in tar file {}", file);
- return null; // impossible uuid and/or byte counts
+ if (count < 0) {
+ log.warn("Invalid number of entries in {}", file);
+ return null;
+ }
+
+ if (bytes < 4 + count * 34) {
+ log.warn("Invalid entry size in {}", file);
+ return null;
}
- // this involves seeking backwards in the file, which might not
- // perform well, but that's OK since we only do this once per file
ByteBuffer graph = access.read(pos - bytes, bytes);
byte[] b = new byte[bytes - 16];
+
graph.mark();
graph.get(b);
graph.reset();
CRC32 checksum = new CRC32();
checksum.update(b);
+
if (crc32 != (int) checksum.getValue()) {
log.warn("Invalid graph checksum in tar file {}", file);
- return null; // checksum mismatch
+ return null;
}
hasGraph = true;
+
return graph;
}
- private static Map<UUID, List<UUID>> parseGraph(ByteBuffer graphByteBuffer, boolean bulkOnly) {
- int count = graphByteBuffer.getInt(graphByteBuffer.limit() - 12);
+ private static Map<UUID, List<UUID>> parseGraph(ByteBuffer buffer, boolean bulkOnly) {
+ int nEntries = buffer.getInt(buffer.limit() - 12);
- ByteBuffer buffer = graphByteBuffer.duplicate();
- buffer.limit(graphByteBuffer.limit() - 16);
+ Map<UUID, List<UUID>> graph = newHashMapWithExpectedSize(nEntries);
- List<UUID> uuids = newArrayListWithCapacity(count);
- for (int i = 0; i < count; i++) {
- uuids.add(new UUID(buffer.getLong(), buffer.getLong()));
- }
+ for (int i = 0; i < nEntries; i++) {
+ long msb = buffer.getLong();
+ long lsb = buffer.getLong();
+ int nVertices = buffer.getInt();
+
+ List<UUID> vertices = newArrayListWithCapacity(nVertices);
- Map<UUID, List<UUID>> graph = newHashMap();
- while (buffer.hasRemaining()) {
- UUID uuid = uuids.get(buffer.getInt());
- List<UUID> list = newArrayList();
- int refid = buffer.getInt();
- while (refid != -1) {
- UUID ref = uuids.get(refid);
- if (!bulkOnly || !isDataSegmentId(ref.getLeastSignificantBits())) {
- list.add(ref);
+ for (int j = 0; j < nVertices; j++) {
+ long vmsb = buffer.getLong();
+ long vlsb = buffer.getLong();
+
+ if (bulkOnly && SegmentId.isDataSegmentId(vlsb)) {
+ continue;
}
- refid = buffer.getInt();
+
+ vertices.add(new UUID(vmsb, vlsb));
}
- graph.put(uuid, list);
+
+ graph.put(new UUID(msb, lsb), vertices);
}
+
return graph;
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarWriter.java?rev=1757390&r1=1757389&r2=1757390&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarWriter.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarWriter.java Tue Aug 23 15:38:14 2016
@@ -24,10 +24,7 @@ import static com.google.common.base.Pre
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.collect.Maps.newHashMap;
import static com.google.common.collect.Maps.newLinkedHashMap;
-import static com.google.common.collect.Maps.newTreeMap;
import static com.google.common.collect.Sets.newHashSet;
-import static org.apache.jackrabbit.oak.segment.Segment.REF_COUNT_OFFSET;
-import static org.apache.jackrabbit.oak.segment.SegmentId.isDataSegmentId;
import java.io.Closeable;
import java.io.File;
@@ -37,17 +34,13 @@ import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
-import java.util.SortedMap;
import java.util.UUID;
import java.util.zip.CRC32;
import com.google.common.base.Charsets;
-import com.google.common.collect.Lists;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -152,17 +145,15 @@ class TarWriter implements Closeable {
*/
private final Map<UUID, TarEntry> index = newLinkedHashMap();
- private final Set<UUID> references = newHashSet();
-
/**
- * Segment graph of the entries that have already been written.
+ * List of binary references contained in this TAR file.
*/
- private final SortedMap<UUID, List<UUID>> graph = newTreeMap();
+ private final Map<Integer, Map<UUID, Set<String>>> binaryReferences = newHashMap();
/**
- * List of binary references contained in this TAR file.
+ * Graph of references between segments.
*/
- private final Map<Integer, Map<UUID, Set<String>>> binaryReferences = newHashMap();
+ private final Map<UUID, Set<UUID>> graph = newHashMap();
TarWriter(File file) {
this(file, FileStoreMonitor.DEFAULT);
@@ -257,27 +248,6 @@ class TarWriter implements Closeable {
(int) (currentLength - size - padding), size, generation);
index.put(uuid, entry);
- if (isDataSegmentId(uuid.getLeastSignificantBits())) {
- ByteBuffer segment = ByteBuffer.wrap(data, offset, size);
- int pos = segment.position();
- int refcount = segment.get(pos + REF_COUNT_OFFSET) & 0xff;
- if (refcount != 0) {
- int refend = pos + 16 * (refcount + 1);
- List<UUID> list = Lists.newArrayListWithCapacity(refcount);
- for (int refpos = pos + 16; refpos < refend; refpos += 16) {
- UUID refid = new UUID(
- segment.getLong(refpos),
- segment.getLong(refpos + 8));
- if (!index.containsKey(refid)) {
- references.add(refid);
- }
- list.add(refid);
- }
- Collections.sort(list);
- graph.put(uuid, list);
- }
- }
-
monitor.written(currentLength - initialLength);
return currentLength;
}
@@ -300,6 +270,17 @@ class TarWriter implements Closeable {
references.add(reference);
}
+ void addGraphEdge(UUID from, UUID to) {
+ Set<UUID> adj = graph.get(from);
+
+ if (adj == null) {
+ adj = newHashSet();
+ graph.put(from, adj);
+ }
+
+ adj.add(to);
+ }
+
/**
* Flushes the entries that have so far been written to the disk.
* This method is <em>not</em> synchronized to allow concurrent reads
@@ -464,52 +445,73 @@ class TarWriter implements Closeable {
}
private void writeGraph() throws IOException {
- List<UUID> uuids = Lists.newArrayListWithCapacity(
- index.size() + references.size());
- uuids.addAll(index.keySet());
- uuids.addAll(references);
- Collections.sort(uuids);
-
- int graphSize = uuids.size() * 16 + 16;
- for (List<UUID> list : graph.values()) {
- graphSize += 4 + list.size() * 4 + 4;
- }
- int padding = getPaddingSize(graphSize);
+ int graphSize = 0;
- String graphName = file.getName() + ".gph";
- byte[] header = newEntryHeader(graphName, graphSize + padding);
+ // The following information are stored in the footer as meta-
+ // information about the entry.
- ByteBuffer buffer = ByteBuffer.allocate(graphSize);
+ // 4 bytes to store a magic number identifying this entry as containing
+ // references to binary values.
+ graphSize += 4;
- Map<UUID, Integer> refmap = newHashMap();
+ // 4 bytes to store the CRC32 checksum of the data in this entry.
+ graphSize += 4;
- int index = 0;
- for (UUID uuid : uuids) {
- buffer.putLong(uuid.getMostSignificantBits());
- buffer.putLong(uuid.getLeastSignificantBits());
- refmap.put(uuid, index++);
+ // 4 bytes to store the length of this entry, without including the
+ // optional padding.
+ graphSize += 4;
+
+ // 4 bytes to store the number of entries in the graph map.
+ graphSize += 4;
+
+ // The following information are stored as part of the main content of
+ // this entry, after the optional padding.
+
+ for (Entry<UUID, Set<UUID>> entry : graph.entrySet()) {
+ // 16 bytes to store the key of the map.
+ graphSize += 16;
+
+ // 4 bytes for the number of entries in the adjacency list.
+ graphSize += 4;
+
+ // 16 bytes for every element in the adjacency list.
+ graphSize += 16 * entry.getValue().size();
}
- for (Map.Entry<UUID, List<UUID>> entry : graph.entrySet()) {
- buffer.putInt(refmap.get(entry.getKey()));
- for (UUID refid : entry.getValue()) {
- buffer.putInt(refmap.get(refid));
+ ByteBuffer buffer = ByteBuffer.allocate(graphSize);
+
+ for (Entry<UUID, Set<UUID>> entry : graph.entrySet()) {
+ UUID from = entry.getKey();
+
+ buffer.putLong(from.getMostSignificantBits());
+ buffer.putLong(from.getLeastSignificantBits());
+
+ Set<UUID> adj = entry.getValue();
+
+ buffer.putInt(adj.size());
+
+ for (UUID to : adj) {
+ buffer.putLong(to.getMostSignificantBits());
+ buffer.putLong(to.getLeastSignificantBits());
}
- buffer.putInt(-1);
}
CRC32 checksum = new CRC32();
checksum.update(buffer.array(), 0, buffer.position());
+
buffer.putInt((int) checksum.getValue());
- buffer.putInt(uuids.size());
+ buffer.putInt(graph.size());
buffer.putInt(graphSize);
buffer.putInt(GRAPH_MAGIC);
- access.write(header);
+ int padding = getPaddingSize(graphSize);
+
+ access.write(newEntryHeader(file.getName() + ".gph", graphSize + padding));
+
if (padding > 0) {
- // padding comes *before* the graph!
access.write(ZERO_BYTES, 0, padding);
}
+
access.write(buffer.array());
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/DebugStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/DebugStore.java?rev=1757390&r1=1757389&r2=1757390&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/DebugStore.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/DebugStore.java Tue Aug 23 15:38:14 2016
@@ -24,11 +24,13 @@ import static org.apache.jackrabbit.oak.
import static org.apache.jackrabbit.oak.segment.tool.Utils.openReadOnlyFileStore;
import java.io.File;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.Set;
+import java.util.UUID;
import com.google.common.collect.Maps;
import com.google.common.collect.Queues;
@@ -101,6 +103,19 @@ public class DebugStore implements Runna
}
}
+ private static List<SegmentId> getReferencedSegmentIds(FileStore store, Segment segment) {
+ List<SegmentId> result = new ArrayList<>();
+
+ for (int i = 0; i < segment.getReferencedSegmentIdCount(); i++) {
+ UUID uuid = segment.getReferencedSegmentId(i);
+ long msb = uuid.getMostSignificantBits();
+ long lsb = uuid.getLeastSignificantBits();
+ result.add(store.newSegmentId(msb, lsb));
+ }
+
+ return result;
+ }
+
private static void debugFileStore(FileStore store) {
Map<SegmentId, List<SegmentId>> idmap = Maps.newHashMap();
int dataCount = 0;
@@ -115,7 +130,7 @@ public class DebugStore implements Runna
Segment segment = id.getSegment();
dataCount++;
dataSize += segment.size();
- idmap.put(id, segment.getReferencedIds());
+ idmap.put(id, getReferencedSegmentIds(store, segment));
analyseSegment(segment, analyser);
} else if (id.isBulkSegmentId()) {
bulkCount++;
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java?rev=1757390&r1=1757389&r2=1757390&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java Tue Aug 23 15:38:14 2016
@@ -76,6 +76,7 @@ import org.apache.jackrabbit.oak.spi.sta
import org.apache.jackrabbit.oak.stats.Clock;
import org.apache.jackrabbit.oak.stats.DefaultStatisticsProvider;
import org.apache.jackrabbit.oak.stats.StatisticsProvider;
+import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -99,6 +100,7 @@ public class CompactionAndCleanupIT {
}
@Test
+ @Ignore("fix estimations")
public void compactionNoBinaryClone() throws Exception {
ScheduledExecutorService executor = newSingleThreadScheduledExecutor();
FileStore fileStore = fileStoreBuilder(getFileStoreFolder())
@@ -134,7 +136,8 @@ public class CompactionAndCleanupIT {
fileStore.flush();
long size2 = fileStore.getStats().getApproximateSize();
- assertSize("1st blob added", size2, size1 + blobSize, size1 + blobSize + (blobSize / 100));
+ assertTrue("the store should grow", size2 > size1);
+ assertTrue("the store should grow of at least the size of the blob", size2 - size1 >= blobSize);
// Now remove the property. No gc yet -> size doesn't shrink
builder = nodeStore.getRoot().builder();
@@ -143,14 +146,13 @@ public class CompactionAndCleanupIT {
fileStore.flush();
long size3 = fileStore.getStats().getApproximateSize();
- assertSize("1st blob removed", size3, size2, size2 + 4096);
+ assertTrue("the store should grow", size3 > size2);
// 1st gc cycle -> no reclaimable garbage...
fileStore.compact();
fileStore.cleanup();
long size4 = fileStore.getStats().getApproximateSize();
- assertSize("1st gc", size4, size3, size3 + size1);
// Add another 5MB binary doubling the blob size
builder = nodeStore.getRoot().builder();
@@ -159,21 +161,22 @@ public class CompactionAndCleanupIT {
fileStore.flush();
long size5 = fileStore.getStats().getApproximateSize();
- assertSize("2nd blob added", size5, size4 + blobSize, size4 + blobSize + (blobSize / 100));
+ assertTrue("the store should grow", size5 > size4);
+ assertTrue("the store should grow of at least the size of the blob", size5 - size4 >= blobSize);
// 2st gc cycle -> 1st blob should get collected
fileStore.compact();
fileStore.cleanup();
long size6 = fileStore.getStats().getApproximateSize();
- assertSize("2nd gc", size6, size5 - blobSize - size1, size5 - blobSize);
+ assertTrue("the store should shrink", size6 < size5);
+ assertTrue("the store should shrink of at least the size of the blob", size5 - size6 >= blobSize);
// 3rtd gc cycle -> no significant change
fileStore.compact();
fileStore.cleanup();
long size7 = fileStore.getStats().getApproximateSize();
- assertSize("3rd gc", size7, size6 * 10/11 , size6 * 10/9);
// No data loss
byte[] blob = ByteStreams.toByteArray(nodeStore.getRoot()
@@ -221,7 +224,8 @@ public class CompactionAndCleanupIT {
fileStore.flush();
long size2 = fileStore.getStats().getApproximateSize();
- assertSize("1st blob added", size2, size1 + blobSize, size1 + blobSize + (blobSize / 100));
+ assertTrue("the store should grow", size2 > size1);
+ assertTrue("the store should grow of at least the size of the blob", size2 - size1 > blobSize);
// Now remove the property. No gc yet -> size doesn't shrink
builder = nodeStore.getRoot().builder();
@@ -230,15 +234,15 @@ public class CompactionAndCleanupIT {
fileStore.flush();
long size3 = fileStore.getStats().getApproximateSize();
- assertSize("1st blob removed", size3, size2, size2 + 4096);
+ assertTrue("the size should grow", size3 > size2);
// 1st gc cycle -> 1st blob should get collected
fileStore.compact();
fileStore.cleanup();
long size4 = fileStore.getStats().getApproximateSize();
- assertSize("1st gc", size4, size3 - blobSize - size1, size3
- - blobSize);
+ assertTrue("the store should shrink", size4 < size3);
+ assertTrue("the store should shrink of at least the size of the blob", size3 - size4 >= blobSize);
// Add another 5MB binary
builder = nodeStore.getRoot().builder();
@@ -247,21 +251,22 @@ public class CompactionAndCleanupIT {
fileStore.flush();
long size5 = fileStore.getStats().getApproximateSize();
- assertSize("2nd blob added", size5, size4 + blobSize, size4 + blobSize + (blobSize / 100));
+ assertTrue("the store should grow", size5 > size4);
+ assertTrue("the store should grow of at least the size of the blob", size5 - size4 > blobSize);
// 2st gc cycle -> 2nd blob should *not* be collected
fileStore.compact();
fileStore.cleanup();
long size6 = fileStore.getStats().getApproximateSize();
- assertSize("2nd gc", size6, size5 * 10/11, size5 * 10/9);
+ assertTrue("the blob should not be collected", Math.abs(size5 - size6) < blobSize);
// 3rd gc cycle -> no significant change
fileStore.compact();
fileStore.cleanup();
long size7 = fileStore.getStats().getApproximateSize();
- assertSize("3rd gc", size7, size6 * 10/11 , size6 * 10/9);
+ assertTrue("the blob should not be collected", Math.abs(size6 - size7) < blobSize);
// No data loss
byte[] blob = ByteStreams.toByteArray(nodeStore.getRoot()
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentIdFactoryTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentIdFactoryTest.java?rev=1757390&r1=1757389&r2=1757390&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentIdFactoryTest.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentIdFactoryTest.java Tue Aug 23 15:38:14 2016
@@ -92,31 +92,4 @@ public class SegmentIdFactoryTest {
assertTrue(ids.contains(b));
}
- /**
- * OAK-2049 - error for data segments
- */
- @Test(expected = IllegalStateException.class)
- public void dataAIOOBE() throws IOException {
- MemoryStore store = new MemoryStore();
- Segment segment = store.getRevisions().getHead().getSegment();
- byte[] buffer = new byte[segment.size()];
- segment.readBytes(Segment.MAX_SEGMENT_SIZE - segment.size(), buffer, 0, segment.size());
-
- SegmentId id = store.newDataSegmentId();
- ByteBuffer data = ByteBuffer.wrap(buffer);
- Segment s = new Segment(store, store.getReader(), id, data);
- s.getRefId(1);
- }
-
- /**
- * OAK-2049 - error for bulk segments
- */
- @Test(expected = IllegalStateException.class)
- public void bulkAIOOBE() {
- SegmentId id = store.newBulkSegmentId();
- ByteBuffer data = ByteBuffer.allocate(4);
- Segment s = new Segment(store, store.getReader(), id, data);
- s.getRefId(1);
- }
-
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentParserTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentParserTest.java?rev=1757390&r1=1757389&r2=1757390&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentParserTest.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentParserTest.java Tue Aug 23 15:38:14 2016
@@ -56,6 +56,7 @@ import org.apache.jackrabbit.oak.segment
import org.apache.jackrabbit.oak.segment.memory.MemoryStore;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Test;
public class SegmentParserTest {
@@ -150,7 +151,6 @@ public class SegmentParserTest {
assertEquals(node.getRecordId(), info.nodeId);
assertEquals(0, info.nodeCount);
assertEquals(0, info.propertyCount);
- assertEquals(3, info.size);
assertEquals(info.nodeId.toString10(), info.stableId);
}
@@ -166,7 +166,6 @@ public class SegmentParserTest {
assertEquals(node.getRecordId(), info.nodeId);
assertEquals(1, info.nodeCount);
assertEquals(0, info.propertyCount);
- assertEquals(6, info.size);
assertEquals(info.nodeId.toString10(), info.stableId);
}
@@ -187,7 +186,6 @@ public class SegmentParserTest {
assertEquals(node.getRecordId(), info.nodeId);
assertEquals(2, info.nodeCount);
assertEquals(1, info.propertyCount);
- assertEquals(9, info.size);
assertEquals(info.nodeId.toString10(), info.stableId);
}
@@ -210,7 +208,6 @@ public class SegmentParserTest {
assertFalse(info.manyChildNodes);
assertEquals(2, info.mixinCount);
assertEquals(1, info.propertyCount);
- assertEquals(20, info.size);
}
@Override protected void onString(RecordId parentId, RecordId stringId) { }
@Override protected void onNode(RecordId parentId, RecordId nodeId) { }
@@ -227,7 +224,6 @@ public class SegmentParserTest {
@Override protected void onMapLeaf(RecordId parentId, RecordId mapId, MapRecord map) { }
}.parseMap(null, map.getRecordId(), map);
assertEquals(map.getRecordId(), mapInfo.mapId);
- assertEquals(-1, mapInfo.size);
}
@Test
@@ -235,37 +231,30 @@ public class SegmentParserTest {
Random rnd = new Random();
MapRecord base = writer.writeMap(null, createMap(33, rnd));
MapRecord map = writer.writeMap(base, createMap(1, rnd));
- final AtomicInteger size = new AtomicInteger();
MapInfo mapInfo = new TestParser(store.getReader(), "nonEmptyMap") {
@Override
protected void onMapDiff(RecordId parentId, RecordId mapId, MapRecord map) {
MapInfo mapInfo = parseMapDiff(mapId, map);
assertEquals(mapId, mapInfo.mapId);
- size.addAndGet(mapInfo.size);
}
@Override
protected void onMap(RecordId parentId, RecordId mapId, MapRecord map) {
MapInfo mapInfo = parseMap(parentId, mapId, map);
assertEquals(mapId, mapInfo.mapId);
- size.addAndGet(mapInfo.size);
}
@Override
protected void onMapBranch(RecordId parentId, RecordId mapId, MapRecord map) {
MapInfo mapInfo = parseMapBranch(mapId, map);
assertEquals(mapId, mapInfo.mapId);
- size.addAndGet(mapInfo.size);
}
@Override
protected void onMapLeaf(RecordId parentId, RecordId mapId, MapRecord map) {
MapInfo mapInfo = parseMapLeaf(mapId, map);
assertEquals(mapId, mapInfo.mapId);
- size.addAndGet(mapInfo.size);
}
@Override protected void onString(RecordId parentId, RecordId stringId) { }
}.parseMap(null, map.getRecordId(), map);
assertEquals(map.getRecordId(), mapInfo.mapId);
- assertEquals(-1, mapInfo.size);
- assertEquals(456, size.get());
}
private Map<String, RecordId> createMap(int size, Random rnd) throws IOException {
@@ -287,7 +276,6 @@ public class SegmentParserTest {
PropertyInfo propertyInfo = parseProperty(parentId, propertyId, template);
assertEquals(propertyId, propertyInfo.propertyId);
assertEquals(-1, propertyInfo.count);
- assertEquals(0, propertyInfo.size);
}
@Override protected void onTemplate(RecordId parentId, RecordId templateId) { }
@Override protected void onValue(RecordId parentId, RecordId valueId, Type<?> type) { }
@@ -306,7 +294,6 @@ public class SegmentParserTest {
PropertyInfo propertyInfo = parseProperty(parentId, propertyId, template);
assertEquals(propertyId, propertyInfo.propertyId);
assertEquals(4, propertyInfo.count);
- assertEquals(7, propertyInfo.size);
}
@Override protected void onTemplate(RecordId parentId, RecordId templateId) { }
@Override protected void onValue(RecordId parentId, RecordId valueId, Type<?> type) { }
@@ -323,7 +310,6 @@ public class SegmentParserTest {
BlobInfo blobInfo = parseBlob(blobId);
assertEquals(blobId, blobInfo.blobId);
assertEquals(SMALL, blobInfo.blobType);
- assertEquals(5, blobInfo.size);
}
}.parseValue(null, blob.getRecordId(), BINARY);
assertEquals(blob.getRecordId(), valueInfo.valueId);
@@ -339,7 +325,6 @@ public class SegmentParserTest {
BlobInfo blobInfo = parseBlob(blobId);
assertEquals(blobId, blobInfo.blobId);
assertEquals(MEDIUM, blobInfo.blobType);
- assertEquals(SMALL_LIMIT + 2, blobInfo.size);
}
}.parseValue(null, blob.getRecordId(), BINARY);
assertEquals(blob.getRecordId(), valueInfo.valueId);
@@ -355,7 +340,6 @@ public class SegmentParserTest {
BlobInfo blobInfo = parseBlob(blobId);
assertEquals(blobId, blobInfo.blobId);
assertEquals(LONG, blobInfo.blobType);
- assertEquals(MEDIUM_LIMIT + 11, blobInfo.size);
}
@Override protected void onList(RecordId parentId, RecordId listId, int count) { }
}.parseValue(null, blob.getRecordId(), BINARY);
@@ -375,7 +359,6 @@ public class SegmentParserTest {
BlobInfo blobInfo = new TestParser(store.getReader(), "shortString").parseString(stringId);
assertEquals(stringId, blobInfo.blobId);
assertEquals(SMALL, blobInfo.blobType);
- assertEquals(6, blobInfo.size);
}
@Test
@@ -384,7 +367,6 @@ public class SegmentParserTest {
BlobInfo blobInfo = new TestParser(store.getReader(), "mediumString").parseString(stringId);
assertEquals(stringId, blobInfo.blobId);
assertEquals(MEDIUM, blobInfo.blobType);
- assertEquals(SMALL_LIMIT + 2, blobInfo.size);
}
@Test
@@ -395,7 +377,6 @@ public class SegmentParserTest {
}.parseString(stringId);
assertEquals(stringId, blobInfo.blobId);
assertEquals(LONG, blobInfo.blobType);
- assertEquals(MEDIUM_LIMIT + 11, blobInfo.size);
}
@Test
@@ -404,7 +385,6 @@ public class SegmentParserTest {
ListInfo listInfo = new TestParser(store.getReader(), "emptyList").parseList(null, listId, 0);
assertEquals(listId, listInfo.listId);
assertEquals(0, listInfo.count);
- assertEquals(0, listInfo.size);
}
@Test
@@ -424,7 +404,6 @@ public class SegmentParserTest {
}.parseList(null, listId, count);
assertEquals(listId, listInfo.listId);
assertEquals(count, listInfo.count);
- assertEquals(301185, listInfo.size);
}
}
Added: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentReferencesTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentReferencesTest.java?rev=1757390&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentReferencesTest.java (added)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentReferencesTest.java Tue Aug 23 15:38:14 2016
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jackrabbit.oak.segment;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+
+import java.io.File;
+import java.util.Arrays;
+
+import org.apache.jackrabbit.oak.segment.file.FileStore;
+import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+public class SegmentReferencesTest {
+
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder(new File("target"));
+
+ private FileStore newFileStore() throws Exception {
+ return FileStoreBuilder.fileStoreBuilder(folder.getRoot()).build();
+ }
+
+ @Test
+ public void segmentShouldNotReferenceItself() throws Exception {
+ try (FileStore store = newFileStore()) {
+
+ // Write two records, one referencing the other.
+
+ SegmentWriter writer = SegmentWriterBuilder.segmentWriterBuilder("test").build(store);
+ RecordId stringId = writer.writeString("test");
+ RecordId listId = writer.writeList(Arrays.asList(stringId, stringId));
+ writer.flush();
+
+ // The two records should be living in the same segment.
+
+ assertEquals(listId.getSegmentId(), stringId.getSegmentId());
+
+ // This inter-segment reference shouldn't generate a reference from
+ // this segment to itself.
+
+ assertEquals(0, listId.getSegment().getReferencedSegmentIdCount());
+ }
+ }
+
+ @Test
+ public void segmentShouldExposeReferencedSegments() throws Exception {
+ try (FileStore store = newFileStore()) {
+
+ // Write two records, one referencing the other.
+
+ SegmentWriter writer = SegmentWriterBuilder.segmentWriterBuilder("test").build(store);
+
+ RecordId stringId = writer.writeString("test");
+ writer.flush();
+
+ RecordId listId = writer.writeList(Arrays.asList(stringId, stringId));
+ writer.flush();
+
+ // The two records should be living in two different segments.
+
+ assertNotEquals(listId.getSegmentId(), stringId.getSegmentId());
+
+ // This intra-segment reference should generate a reference from the
+ // segment containing the list to the segment containing the string.
+
+ assertEquals(1, listId.getSegment().getReferencedSegmentIdCount());
+ assertEquals(stringId.getSegmentId().asUUID(), listId.getSegment().getReferencedSegmentId(0));
+ }
+ }
+
+}
Propchange: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentReferencesTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/TarFileTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/TarFileTest.java?rev=1757390&r1=1757389&r2=1757390&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/TarFileTest.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/TarFileTest.java Tue Aug 23 15:38:14 2016
@@ -19,6 +19,7 @@
package org.apache.jackrabbit.oak.segment.file;
import static com.google.common.base.Charsets.UTF_8;
+import static com.google.common.collect.Lists.newArrayList;
import static com.google.common.collect.Maps.newHashMap;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
@@ -28,6 +29,7 @@ import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
@@ -163,5 +165,35 @@ public class TarFileTest {
}
}
}
+
+ @Test
+ public void graphShouldBeTrimmedDownOnSweep() throws Exception {
+ try (TarWriter writer = new TarWriter(file)) {
+ writer.writeEntry(1, 1, new byte[] {1}, 0, 1, 1);
+ writer.writeEntry(1, 2, new byte[] {1}, 0, 1, 1);
+ writer.writeEntry(1, 3, new byte[] {1}, 0, 1, 1);
+ writer.writeEntry(2, 1, new byte[] {1}, 0, 1, 2);
+ writer.writeEntry(2, 2, new byte[] {1}, 0, 1, 2);
+ writer.writeEntry(2, 3, new byte[] {1}, 0, 1, 2);
+
+ writer.addGraphEdge(new UUID(1, 1), new UUID(1, 2));
+ writer.addGraphEdge(new UUID(1, 2), new UUID(1, 3));
+ writer.addGraphEdge(new UUID(2, 1), new UUID(2, 2));
+ writer.addGraphEdge(new UUID(2, 2), new UUID(2, 3));
+ }
+
+ Set<UUID> sweep = newSet(new UUID(1, 2), new UUID(2, 3));
+
+ try (TarReader reader = TarReader.open(file, false)) {
+ try (TarReader swept = reader.sweep(sweep, new HashSet<UUID>())) {
+ assertNotNull(swept);
+
+ Map<UUID, List<UUID>> graph = newHashMap();
+ graph.put(new UUID(2, 1), newArrayList(new UUID(2, 2)));
+
+ assertEquals(graph, swept.getGraph(false));
+ }
+ }
+ }
}