You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by md...@apache.org on 2016/04/20 12:05:49 UTC
svn commit: r1740103 - in /jackrabbit/oak/trunk/oak-segment-next/src:
main/java/org/apache/jackrabbit/oak/plugins/segment/
main/java/org/apache/jackrabbit/oak/plugins/segment/file/
test/java/org/apache/jackrabbit/oak/plugins/segment/file/
Author: mduerig
Date: Wed Apr 20 10:05:49 2016
New Revision: 1740103
URL: http://svn.apache.org/viewvc?rev=1740103&view=rev
Log:
OAK-3348: Cross gc sessions might introduce references to pre-compacted segments
Write gc generation into tar index
Modified:
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarEntry.java
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java
jackrabbit/oak/trunk/oak-segment-next/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStoreStatsTest.java
jackrabbit/oak/trunk/oak-segment-next/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFileTest.java
jackrabbit/oak/trunk/oak-segment-next/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriterTest.java
Modified: jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java?rev=1740103&r1=1740102&r2=1740103&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java (original)
+++ jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java Wed Apr 20 10:05:49 2016
@@ -312,11 +312,14 @@ public class Segment {
return data.getShort(ROOT_COUNT_OFFSET) & 0xffff;
}
- // could we make it part of the tar file index!?
- public int getGcGen() {
+ public static int getGcGen(ByteBuffer data) {
return data.getInt(GC_GEN_OFFSET);
}
+ public int getGcGen() {
+ return getGcGen(data);
+ }
+
public RecordType getRootType(int index) {
int refCount = getRefCount();
checkArgument(index < getRootCount());
Modified: jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java?rev=1740103&r1=1740102&r2=1740103&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java (original)
+++ jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java Wed Apr 20 10:05:49 2016
@@ -26,6 +26,7 @@ import static com.google.common.collect.
import static com.google.common.collect.Sets.newHashSet;
import static java.lang.String.format;
import static java.lang.Thread.currentThread;
+import static java.nio.ByteBuffer.wrap;
import static java.util.Collections.emptyMap;
import static java.util.Collections.singletonMap;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
@@ -1278,10 +1279,11 @@ public class FileStore implements Segmen
public void writeSegment(SegmentId id, byte[] data, int offset, int length) throws IOException {
fileStoreLock.writeLock().lock();
try {
+ int generation = Segment.getGcGen(wrap(data, offset, length));
long size = writer.writeEntry(
id.getMostSignificantBits(),
id.getLeastSignificantBits(),
- data, offset, length);
+ data, offset, length, generation);
if (size >= maxFileSize) {
newWriter();
}
Modified: jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarEntry.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarEntry.java?rev=1740103&r1=1740102&r2=1740103&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarEntry.java (original)
+++ jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarEntry.java Wed Apr 20 10:05:49 2016
@@ -25,7 +25,7 @@ import java.util.Comparator;
class TarEntry {
/** Size in bytes a tar entry takes up in the tar file */
- static final int SIZE = 24;
+ static final int SIZE = 28;
static final Comparator<TarEntry> OFFSET_ORDER = new Comparator<TarEntry>() {
@Override
@@ -65,11 +65,14 @@ class TarEntry {
private final int size;
- TarEntry(long msb, long lsb, int offset, int size) {
+ private final int generation;
+
+ TarEntry(long msb, long lsb, int offset, int size, int generation) {
this.msb = msb;
this.lsb = lsb;
this.offset = offset;
this.size = size;
+ this.generation = generation;
}
long msb() {
@@ -88,4 +91,8 @@ class TarEntry {
return size;
}
+ int generation() {
+ return generation;
+ }
+
}
Modified: jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java?rev=1740103&r1=1740102&r2=1740103&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java (original)
+++ jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java Wed Apr 20 10:05:49 2016
@@ -25,9 +25,10 @@ import static com.google.common.collect.
import static com.google.common.collect.Maps.newTreeMap;
import static com.google.common.collect.Sets.newHashSet;
import static com.google.common.collect.Sets.newHashSetWithExpectedSize;
+import static java.nio.ByteBuffer.wrap;
import static java.util.Collections.singletonList;
-import static org.apache.jackrabbit.oak.plugins.segment.Segment.GC_GEN_OFFSET;
import static org.apache.jackrabbit.oak.plugins.segment.Segment.REF_COUNT_OFFSET;
+import static org.apache.jackrabbit.oak.plugins.segment.Segment.getGcGen;
import static org.apache.jackrabbit.oak.plugins.segment.SegmentId.isDataSegmentId;
import static org.apache.jackrabbit.oak.plugins.segment.file.TarWriter.GRAPH_MAGIC;
@@ -211,10 +212,11 @@ class TarReader implements Closeable {
for (Map.Entry<UUID, byte[]> entry : entries.entrySet()) {
UUID uuid = entry.getKey();
byte[] data = entry.getValue();
+ int generation = getGcGen(wrap(data));
writer.writeEntry(
uuid.getMostSignificantBits(),
uuid.getLeastSignificantBits(),
- data, 0, data.length);
+ data, 0, data.length, generation);
}
writer.close();
}
@@ -375,7 +377,7 @@ class TarReader implements Closeable {
index.get(entry);
checksum.update(entry);
- ByteBuffer buffer = ByteBuffer.wrap(entry);
+ ByteBuffer buffer = wrap(entry);
long msb = buffer.getLong();
long lsb = buffer.getLong();
int offset = buffer.getInt();
@@ -447,7 +449,7 @@ class TarReader implements Closeable {
}
// The header checksum passes, so read the entry name and size
- ByteBuffer buffer = ByteBuffer.wrap(header);
+ ByteBuffer buffer = wrap(header);
String name = readString(buffer, 100);
buffer.position(124);
int size = readNumber(buffer, 12);
@@ -645,7 +647,8 @@ class TarReader implements Closeable {
index.getLong(position),
index.getLong(position + 8),
index.getInt(position + 16),
- index.getInt(position + 20));
+ index.getInt(position + 20),
+ index.getInt(position + 24));
position += TarEntry.SIZE;
}
Arrays.sort(entries, TarEntry.OFFSET_ORDER);
@@ -730,11 +733,6 @@ class TarReader implements Closeable {
}
}
- private int getGCGeneration(TarEntry entry) throws IOException {
- ByteBuffer segment = access.read(entry.offset(), GC_GEN_OFFSET + 4);
- return segment.getInt(GC_GEN_OFFSET);
- }
-
void mark(Set<UUID> bulkRefs, Set<UUID> reclaim, int generation) throws IOException {
Map<UUID, List<UUID>> graph = getGraph(true);
TarEntry[] entries = getEntries();
@@ -742,7 +740,7 @@ class TarReader implements Closeable {
TarEntry entry = entries[i];
UUID id = new UUID(entry.msb(), entry.lsb());
if ((!isDataSegmentId(entry.lsb()) && !bulkRefs.remove(id)) ||
- (isDataSegmentId(entry.lsb()) && getGCGeneration(entry) < generation)) {
+ (isDataSegmentId(entry.lsb()) && entry.generation() < generation)) {
// non references bulk segment or old data segment
reclaim.add(id);
} else {
@@ -817,7 +815,7 @@ class TarReader implements Closeable {
byte[] data = new byte[entry.size()];
access.read(entry.offset(), entry.size()).get(data);
writer.writeEntry(
- entry.msb(), entry.lsb(), data, 0, entry.size());
+ entry.msb(), entry.lsb(), data, 0, entry.size(), entry.generation());
}
}
writer.close();
Modified: jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java?rev=1740103&r1=1740102&r2=1740103&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java (original)
+++ jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java Wed Apr 20 10:05:49 2016
@@ -206,7 +206,7 @@ class TarWriter implements Closeable {
}
long writeEntry(
- long msb, long lsb, byte[] data, int offset, int size)
+ long msb, long lsb, byte[] data, int offset, int size, int generation)
throws IOException {
checkNotNull(data);
checkPositionIndexes(offset, offset + size, data.length);
@@ -218,11 +218,11 @@ class TarWriter implements Closeable {
byte[] header = newEntryHeader(entryName, size);
log.debug("Writing segment {} to {}", uuid, file);
- return writeEntry(uuid, header, data, offset, size);
+ return writeEntry(uuid, header, data, offset, size, generation);
}
private synchronized long writeEntry(
- UUID uuid, byte[] header, byte[] data, int offset, int size)
+ UUID uuid, byte[] header, byte[] data, int offset, int size, int generation)
throws IOException {
checkState(!closed);
if (access == null) {
@@ -242,7 +242,7 @@ class TarWriter implements Closeable {
checkState(currentLength <= Integer.MAX_VALUE);
TarEntry entry = new TarEntry(
uuid.getMostSignificantBits(), uuid.getLeastSignificantBits(),
- (int) (currentLength - size - padding), size);
+ (int) (currentLength - size - padding), size, generation);
index.put(uuid, entry);
if (isDataSegmentId(uuid.getLeastSignificantBits())) {
@@ -403,6 +403,7 @@ class TarWriter implements Closeable {
buffer.putLong(entry.lsb());
buffer.putInt(entry.offset());
buffer.putInt(entry.size());
+ buffer.putInt(entry.generation());
}
CRC32 checksum = new CRC32();
Modified: jackrabbit/oak/trunk/oak-segment-next/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStoreStatsTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-next/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStoreStatsTest.java?rev=1740103&r1=1740102&r2=1740103&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-next/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStoreStatsTest.java (original)
+++ jackrabbit/oak/trunk/oak-segment-next/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStoreStatsTest.java Wed Apr 20 10:05:49 2016
@@ -84,7 +84,7 @@ public class FileStoreStatsTest {
File file = segmentFolder.newFile();
TarWriter writer = new TarWriter(file, stats);
- writer.writeEntry(msb, lsb, data, 0, data.length);
+ writer.writeEntry(msb, lsb, data, 0, data.length, 0);
writer.close();
assertEquals(stats.getApproximateSize(), file.length());
Modified: jackrabbit/oak/trunk/oak-segment-next/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFileTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-next/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFileTest.java?rev=1740103&r1=1740102&r2=1740103&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-next/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFileTest.java (original)
+++ jackrabbit/oak/trunk/oak-segment-next/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFileTest.java Wed Apr 20 10:05:49 2016
@@ -50,7 +50,7 @@ public class TarFileTest {
TarWriter writer = new TarWriter(file);
try {
- writer.writeEntry(msb, lsb, data, 0, data.length);
+ writer.writeEntry(msb, lsb, data, 0, data.length, 0);
assertEquals(ByteBuffer.wrap(data), writer.readEntry(msb, lsb));
} finally {
writer.close();
Modified: jackrabbit/oak/trunk/oak-segment-next/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriterTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-next/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriterTest.java?rev=1740103&r1=1740102&r2=1740103&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-next/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriterTest.java (original)
+++ jackrabbit/oak/trunk/oak-segment-next/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriterTest.java Wed Apr 20 10:05:49 2016
@@ -186,7 +186,7 @@ public class TarWriterTest {
public void write(TarWriter tarWriter) throws IOException {
long msb = getSegmentId().getMostSignificantBits();
long lsb = getSegmentId().getLeastSignificantBits();
- tarWriter.writeEntry(msb, lsb, data, 0, data.length);
+ tarWriter.writeEntry(msb, lsb, data, 0, data.length, 0);
}
public UUID getUUID() {