You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by fr...@apache.org on 2016/10/03 09:09:52 UTC
svn commit: r1763136 - in /jackrabbit/oak/trunk/oak-segment-tar/src:
main/java/org/apache/jackrabbit/oak/segment/
main/java/org/apache/jackrabbit/oak/segment/tool/
test/java/org/apache/jackrabbit/oak/segment/
Author: frm
Date: Mon Oct 3 09:09:52 2016
New Revision: 1763136
URL: http://svn.apache.org/viewvc?rev=1763136&view=rev
Log:
OAK-2498 - Remove the root record table from segments
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/ImmutableRecordNumbers.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/MutableRecordNumbers.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/DebugStore.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/NodeRecordTest.java
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/ImmutableRecordNumbers.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/ImmutableRecordNumbers.java?rev=1763136&r1=1763135&r2=1763136&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/ImmutableRecordNumbers.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/ImmutableRecordNumbers.java Mon Oct 3 09:09:52 2016
@@ -17,7 +17,7 @@
package org.apache.jackrabbit.oak.segment;
-import static com.google.common.collect.Maps.newHashMap;
+import static com.google.common.collect.Maps.newLinkedHashMap;
import java.util.Iterator;
import java.util.Map;
@@ -39,7 +39,7 @@ class ImmutableRecordNumbers implements
* {@code null}.
*/
ImmutableRecordNumbers(Map<Integer, RecordEntry> records) {
- this.records = newHashMap(records);
+ this.records = newLinkedHashMap(records);
}
@Override
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/MutableRecordNumbers.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/MutableRecordNumbers.java?rev=1763136&r1=1763135&r2=1763136&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/MutableRecordNumbers.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/MutableRecordNumbers.java Mon Oct 3 09:09:52 2016
@@ -29,7 +29,7 @@ class MutableRecordNumbers implements Re
private final Object lock = new Object();
- private final Map<Integer, RecordEntry> records = Maps.newHashMap();
+ private final Map<Integer, RecordEntry> records = Maps.newLinkedHashMap();
@Override
public int getOffset(int recordNumber) {
@@ -55,7 +55,7 @@ class MutableRecordNumbers implements Re
Map<Integer, RecordEntry> recordNumbers;
synchronized (lock) {
- recordNumbers = Maps.newHashMap(this.records);
+ recordNumbers = Maps.newLinkedHashMap(this.records);
}
return new RecordNumbersIterator(recordNumbers.entrySet().iterator());
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java?rev=1763136&r1=1763135&r2=1763136&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java Mon Oct 3 09:09:52 2016
@@ -18,12 +18,11 @@
*/
package org.apache.jackrabbit.oak.segment;
-import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkPositionIndexes;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.collect.Lists.newArrayListWithCapacity;
-import static com.google.common.collect.Maps.newHashMapWithExpectedSize;
+import static com.google.common.collect.Maps.newLinkedHashMap;
import static org.apache.jackrabbit.oak.commons.IOUtils.closeQuietly;
import static org.apache.jackrabbit.oak.segment.SegmentId.isDataSegmentId;
import static org.apache.jackrabbit.oak.segment.SegmentVersion.LATEST_VERSION;
@@ -119,8 +118,6 @@ public class Segment {
*/
public static final int BLOB_ID_SMALL_LIMIT = 1 << 12;
- static final int ROOT_COUNT_OFFSET = 6;
-
public static final int GC_GENERATION_OFFSET = 10;
public static final int REFERENCED_SEGMENT_ID_COUNT_OFFSET = 14;
@@ -216,7 +213,7 @@ public class Segment {
* @return An instance of {@link RecordNumbers}, never {@code null}.
*/
private RecordNumbers readRecordNumberOffsets() {
- Map<Integer, RecordEntry> recordNumberOffsets = newHashMapWithExpectedSize(getRecordNumberCount());
+ Map<Integer, RecordEntry> recordNumberOffsets = newLinkedHashMap();
int position = data.position();
@@ -314,10 +311,6 @@ public class Segment {
return id;
}
- public int getRootCount() {
- return data.getShort(ROOT_COUNT_OFFSET) & 0xffff;
- }
-
public int getReferencedSegmentIdCount() {
return data.getInt(REFERENCED_SEGMENT_ID_COUNT_OFFSET);
}
@@ -353,33 +346,6 @@ public class Segment {
return getGcGeneration(data, id.asUUID());
}
- public RecordType getRootType(int index) {
- checkArgument(index < getRootCount());
-
- int position = data.position();
-
- position += HEADER_SIZE;
- position += getReferencedSegmentIdCount() * 16;
- position += getRecordNumberCount() * 12;
- position += index * 5;
-
- return RecordType.values()[data.get(position) & 0xff];
- }
-
- public int getRootOffset(int index) {
- checkArgument(index < getRootCount());
-
- int position = data.position();
-
- position += HEADER_SIZE;
- position += getReferencedSegmentIdCount() * 16;
- position += getRecordNumberCount() * 12;
- position += index * 5;
- position += 1;
-
- return data.getInt(position);
- }
-
private volatile String info;
/**
@@ -400,7 +366,7 @@ public class Segment {
@CheckForNull
public String getSegmentInfo() {
if (info == null && id.isDataSegmentId()) {
- info = readString(getRootOffset(0));
+ info = readString(recordNumbers.iterator().next().getRecordNumber());
}
return info;
}
@@ -628,10 +594,6 @@ public class Segment {
for (Entry entry : recordNumbers) {
writer.format("record number %08x: %08x", entry.getRecordNumber(), entry.getOffset());
}
-
- for (i = 0; i < getRootCount(); i++) {
- writer.format("root %d: %s at %04x%n", i, getRootType(i), getRootOffset(i));
- }
}
writer.println("--------------------------------------------------------------------------");
int pos = data.limit() - ((length + 15) & ~15);
@@ -677,4 +639,16 @@ public class Segment {
}
}
+ public interface RecordConsumer {
+
+ void consume(int number, RecordType type, int offset);
+
+ }
+
+ public void forEachRecord(RecordConsumer consumer) {
+ for (Entry entry : recordNumbers) {
+ consumer.consume(entry.getRecordNumber(), entry.getType(), entry.getOffset());
+ }
+ }
+
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java?rev=1763136&r1=1763135&r2=1763136&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java Mon Oct 3 09:09:52 2016
@@ -23,7 +23,6 @@ import static com.google.common.base.Cha
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
-import static com.google.common.collect.Maps.newLinkedHashMap;
import static com.google.common.collect.Sets.newHashSet;
import static java.lang.System.arraycopy;
import static java.lang.System.currentTimeMillis;
@@ -37,7 +36,6 @@ import static org.apache.jackrabbit.oak.
import java.io.IOException;
import java.util.Collection;
-import java.util.Map;
import java.util.Set;
import javax.annotation.CheckForNull;
@@ -107,12 +105,6 @@ public class SegmentBufferWriter impleme
}
}
- /**
- * The set of root records (i.e. ones not referenced by other records)
- * in this segment.
- */
- private final Map<RecordId, RecordType> roots = newLinkedHashMap();
-
private MutableRecordNumbers recordNumbers = new MutableRecordNumbers();
private MutableSegmentReferences segmentReferences = new MutableSegmentReferences();
@@ -215,7 +207,6 @@ public class SegmentBufferWriter impleme
buffer[GC_GENERATION_OFFSET + 3] = (byte) generation;
length = 0;
position = buffer.length;
- roots.clear();
recordNumbers = new MutableRecordNumbers();
segmentReferences = new MutableSegmentReferences();
@@ -280,10 +271,6 @@ public class SegmentBufferWriter impleme
public void writeRecordId(RecordId recordId, boolean reference) {
checkNotNull(recordId);
- if (reference) {
- roots.remove(recordId);
- }
-
checkGCGeneration(recordId.getSegmentId());
writeInt(writeSegmentIdReference(recordId.getSegmentId()));
@@ -347,9 +334,6 @@ public class SegmentBufferWriter impleme
@Override
public void flush() throws IOException {
if (dirty) {
- int rootcount = roots.size();
- BinaryUtils.writeShort(buffer, Segment.ROOT_COUNT_OFFSET, (short) rootcount);
-
int referencedSegmentIdCount = segmentReferences.size();
BinaryUtils.writeInt(buffer, Segment.REFERENCED_SEGMENT_ID_COUNT_OFFSET, referencedSegmentIdCount);
@@ -358,7 +342,7 @@ public class SegmentBufferWriter impleme
int recordNumberCount = recordNumbers.size();
BinaryUtils.writeInt(buffer, Segment.RECORD_NUMBER_COUNT_OFFSET, recordNumberCount);
- int totalLength = align(HEADER_SIZE + referencedSegmentIdCount * 16 + rootcount * 5 + recordNumberCount * 12 + length, 16);
+ int totalLength = align(HEADER_SIZE + referencedSegmentIdCount * 16 + recordNumberCount * 12 + length, 16);
if (totalLength > buffer.length) {
throw new IllegalStateException("too much data for a segment");
@@ -393,11 +377,6 @@ public class SegmentBufferWriter impleme
pos = BinaryUtils.writeInt(buffer, pos, entry.getOffset());
}
- for (Map.Entry<RecordId, RecordType> entry : roots.entrySet()) {
- pos = BinaryUtils.writeByte(buffer, pos, (byte) entry.getValue().ordinal());
- pos = BinaryUtils.writeInt(buffer, pos, entry.getKey().getRecordNumber());
- }
-
SegmentId segmentId = segment.getSegmentId();
LOG.debug("Writing data segment: {} ", statistics);
store.writeSegment(segmentId, buffer, buffer.length - length, length);
@@ -433,10 +412,9 @@ public class SegmentBufferWriter impleme
// that *all* identifiers stored in this record point to previously
// unreferenced segments.
- int rootCount = roots.size() + 1;
int recordNumbersCount = recordNumbers.size() + 1;
int referencedIdCount = segmentReferences.size() + ids.size();
- int headerSize = HEADER_SIZE + rootCount * 5 + referencedIdCount * 16 + recordNumbersCount * 12;
+ int headerSize = HEADER_SIZE + referencedIdCount * 16 + recordNumbersCount * 12;
int segmentSize = align(headerSize + recordSize + length, 16);
// If the size estimate looks too big, recompute it with a more
@@ -452,17 +430,13 @@ public class SegmentBufferWriter impleme
// potentially reference the same record multiple times
Set<SegmentId> segmentIds = newHashSet();
- Set<RecordId> notRoots = newHashSet();
for (RecordId recordId : ids) {
SegmentId segmentId = recordId.getSegmentId();
if (!(segmentId.equals(segment.getSegmentId()))) {
segmentIds.add(segmentId);
- } else if (roots.containsKey(recordId)) {
- notRoots.add(recordId);
}
}
- rootCount -= notRoots.size();
// Adjust the estimation of the new referenced segment ID count.
@@ -472,11 +446,11 @@ public class SegmentBufferWriter impleme
}
}
- headerSize = HEADER_SIZE + rootCount * 5 + referencedIdCount * 16 + recordNumbersCount * 12;
+ headerSize = HEADER_SIZE + referencedIdCount * 16 + recordNumbersCount * 12;
segmentSize = align(headerSize + recordSize + length, 16);
}
- if (segmentSize > buffer.length || rootCount > 0xffff) {
+ if (segmentSize > buffer.length) {
flush();
}
@@ -487,9 +461,7 @@ public class SegmentBufferWriter impleme
checkState(position >= 0);
int recordNumber = recordNumbers.addRecord(type, position);
- RecordId id = new RecordId(segment.getSegmentId(), recordNumber);
- roots.put(id, type);
- return id;
+ return new RecordId(segment.getSegmentId(), recordNumber);
}
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/DebugStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/DebugStore.java?rev=1763136&r1=1763135&r2=1763136&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/DebugStore.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/DebugStore.java Mon Oct 3 09:09:52 2016
@@ -18,6 +18,7 @@
package org.apache.jackrabbit.oak.segment.tool;
import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.collect.Lists.newArrayList;
import static com.google.common.collect.Sets.newHashSet;
import static org.apache.commons.io.FileUtils.byteCountToDisplaySize;
import static org.apache.jackrabbit.oak.segment.RecordType.NODE;
@@ -35,8 +36,10 @@ import java.util.UUID;
import com.google.common.collect.Maps;
import com.google.common.collect.Queues;
import org.apache.jackrabbit.oak.segment.RecordId;
+import org.apache.jackrabbit.oak.segment.RecordType;
import org.apache.jackrabbit.oak.segment.RecordUsageAnalyser;
import org.apache.jackrabbit.oak.segment.Segment;
+import org.apache.jackrabbit.oak.segment.Segment.RecordConsumer;
import org.apache.jackrabbit.oak.segment.SegmentId;
import org.apache.jackrabbit.oak.segment.file.FileStore;
@@ -170,17 +173,27 @@ public class DebugStore implements Runna
System.out.format("%s in %6d bulk segments%n", byteCountToDisplaySize(bulkSize), bulkCount);
}
- private static void analyseSegment(Segment segment, RecordUsageAnalyser analyser) {
- for (int k = 0; k < segment.getRootCount(); k++) {
- if (segment.getRootType(k) == NODE) {
- RecordId nodeId = new RecordId(segment.getSegmentId(), segment.getRootOffset(k));
- try {
- analyser.analyseNode(nodeId);
- } catch (Exception e) {
- System.err.format("Error while processing node at %s", nodeId);
- e.printStackTrace();
+ private static void analyseSegment(final Segment segment, final RecordUsageAnalyser analyser) {
+ final List<RecordId> ids = newArrayList();
+
+ segment.forEachRecord(new RecordConsumer() {
+
+ @Override
+ public void consume(int number, RecordType type, int offset) {
+ if (type == NODE) {
+ ids.add(new RecordId(segment.getSegmentId(), number));
}
}
+
+ });
+
+ for (RecordId id : ids) {
+ try {
+ analyser.analyseNode(id);
+ } catch (Exception e) {
+ System.err.format("Error while processing node at %s", id);
+ e.printStackTrace();
+ }
}
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/NodeRecordTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/NodeRecordTest.java?rev=1763136&r1=1763135&r2=1763136&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/NodeRecordTest.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/NodeRecordTest.java Mon Oct 3 09:09:52 2016
@@ -21,7 +21,6 @@ import static org.junit.Assert.assertArr
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
import javax.annotation.Nonnull;
@@ -63,7 +62,6 @@ public class NodeRecordTest {
SegmentWriter writer = SegmentWriterBuilder.segmentWriterBuilder("test").build(store);
SegmentNodeState state = writer.writeNode(EmptyNodeState.EMPTY_NODE);
writer.flush();
- assertTrue(isRootRecord(state));
}
}
@@ -189,22 +187,4 @@ public class NodeRecordTest {
};
}
- private boolean isRootRecord(SegmentNodeState sns) {
- Segment segment = sns.getRecordId().getSegment();
-
- for (int i = 0; i < segment.getRootCount(); i++) {
- if (segment.getRootType(i) != RecordType.NODE) {
- continue;
- }
-
- if (segment.getRootOffset(i) != sns.getRecordId().getRecordNumber()) {
- continue;
- }
-
- return true;
- }
-
- return false;
- }
-
}