You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by fr...@apache.org on 2017/07/27 09:54:50 UTC
svn commit: r1803156 - in /jackrabbit/oak/trunk/oak-segment-tar/src:
main/java/org/apache/jackrabbit/oak/segment/file/tar/
test/java/org/apache/jackrabbit/oak/segment/file/tar/
Author: frm
Date: Thu Jul 27 09:54:50 2017
New Revision: 1803156
URL: http://svn.apache.org/viewvc?rev=1803156&view=rev
Log:
OAK-6456 - Include the tail generation in the TAR index
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarEntry.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriter.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFileTest.java
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarEntry.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarEntry.java?rev=1803156&r1=1803155&r2=1803156&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarEntry.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarEntry.java Thu Jul 27 09:54:50 2017
@@ -27,7 +27,7 @@ import java.util.Comparator;
class TarEntry {
/** Size in bytes a tar entry takes up in the tar file */
- static final int SIZE = 28;
+ static final int SIZE = 33;
static final Comparator<TarEntry> OFFSET_ORDER = new Comparator<TarEntry>() {
@Override
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java?rev=1803156&r1=1803155&r2=1803156&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java Thu Jul 27 09:54:50 2017
@@ -633,7 +633,7 @@ class TarReader implements Closeable {
* @return An array of {@link TarEntry}.
*/
@Nonnull
- private TarEntry[] getEntries() {
+ TarEntry[] getEntries() {
TarEntry[] entries = new TarEntry[index.remaining() / TarEntry.SIZE];
int position = index.position();
for (int i = 0; position < index.limit(); i++) {
@@ -642,8 +642,12 @@ class TarReader implements Closeable {
index.getLong(position + 8),
index.getInt(position + 16),
index.getInt(position + 20),
- // TODO frm This should read both a full and a tail generation. See OAK-6456.
- newGCGeneration(index.getInt(position + 24), 0, false));
+ newGCGeneration(
+ index.getInt(position + 24),
+ index.getInt(position + 28),
+ index.get(position + 32) != 0
+ )
+ );
position += TarEntry.SIZE;
}
Arrays.sort(entries, TarEntry.OFFSET_ORDER);
@@ -750,7 +754,6 @@ class TarReader implements Closeable {
// CPU on subsequent look-ups.
TarEntry entry = entries[i];
UUID id = new UUID(entry.msb(), entry.lsb());
- // FIXME OAK-3349 cannot properly clean up under tail compaction here since we don't have access to the tail part of the generation (i.e. it is not in the tar index). As a workaround for now the implementation of shouldReclaim() could directly read the tail generation from the segment (i.e. SegmentIdProvider.newSegmentId(id.getMostSignificantBits(), id.getLeastSignificantBits()).getGcGeneration().getTail())
if (context.shouldReclaim(id, entry.generation(), references.remove(id))) {
reclaimable.add(id);
} else {
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriter.java?rev=1803156&r1=1803155&r2=1803156&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriter.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriter.java Thu Jul 27 09:54:50 2017
@@ -499,8 +499,9 @@ class TarWriter implements Closeable {
buffer.putLong(entry.lsb());
buffer.putInt(entry.offset());
buffer.putInt(entry.size());
- // TODO frm Include both the full and tail generation. See OAK-6456.
buffer.putInt(entry.generation().getFull());
+ buffer.putInt(entry.generation().getTail());
+ buffer.put((byte) (entry.generation().isTail() ? 1 : 0));
}
CRC32 checksum = new CRC32();
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFileTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFileTest.java?rev=1803156&r1=1803155&r2=1803156&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFileTest.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFileTest.java Thu Jul 27 09:54:50 2017
@@ -75,6 +75,42 @@ public class TarFileTest {
}
@Test
+ public void testGCGeneration() throws Exception {
+ UUID id = UUID.randomUUID();
+ long msb = id.getMostSignificantBits();
+ long lsb = id.getLeastSignificantBits();
+ String data = "test";
+ byte[] buffer = data.getBytes(UTF_8);
+
+ try (TarWriter writer = new TarWriter(file, new IOMonitorAdapter())) {
+ writer.writeEntry(msb, lsb, buffer, 0, buffer.length, GCGeneration.newGCGeneration(1, 2, false));
+ }
+
+ try (TarReader reader = TarReader.open(file, false, new IOMonitorAdapter())) {
+ TarEntry[] entries = reader.getEntries();
+ assertEquals(GCGeneration.newGCGeneration(1, 2, false), entries[0].generation());
+ }
+ }
+
+ @Test
+ public void testGCGenerationIsTailFlagNotErased() throws Exception {
+ UUID id = UUID.randomUUID();
+ long msb = id.getMostSignificantBits();
+ long lsb = id.getLeastSignificantBits();
+ String data = "test";
+ byte[] buffer = data.getBytes(UTF_8);
+
+ try (TarWriter writer = new TarWriter(file, new IOMonitorAdapter())) {
+ writer.writeEntry(msb, lsb, buffer, 0, buffer.length, GCGeneration.newGCGeneration(1, 2, true));
+ }
+
+ try (TarReader reader = TarReader.open(file, false, new IOMonitorAdapter())) {
+ TarEntry[] entries = reader.getEntries();
+ assertEquals(GCGeneration.newGCGeneration(1, 2, true), entries[0].generation());
+ }
+ }
+
+ @Test
public void testWriteAndReadBinaryReferences() throws Exception {
try (TarWriter writer = new TarWriter(file, new IOMonitorAdapter())) {
writer.writeEntry(0x00, 0x00, new byte[] {0x01, 0x02, 0x3}, 0, 3, generation(0));