You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by fr...@apache.org on 2016/10/18 11:10:45 UTC
svn commit: r1765414 - in
/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak:
backup/impl/ segment/ segment/file/ segment/http/ segment/memory/
Author: frm
Date: Tue Oct 18 11:10:44 2016
New Revision: 1765414
URL: http://svn.apache.org/viewvc?rev=1765414&view=rev
Log:
OAK-4947 - Inspect the segment on write to upadte the index of binary references
Removed:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/BinaryReferenceConsumer.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/BinaryReferences.java
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreBackupImpl.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreRestoreImpl.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriterBuilder.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/http/HttpStore.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/memory/MemoryStore.java
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreBackupImpl.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreBackupImpl.java?rev=1765414&r1=1765413&r2=1765414&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreBackupImpl.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreBackupImpl.java Tue Oct 18 11:10:44 2016
@@ -79,8 +79,7 @@ public class FileStoreBackupImpl impleme
backup.getReader(),
backup.getBlobStore(),
new WriterCacheManager.Default(),
- bufferWriter,
- backup.getBinaryReferenceConsumer()
+ bufferWriter
);
Compactor compactor = new Compactor(
backup.getReader(),
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreRestoreImpl.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreRestoreImpl.java?rev=1765414&r1=1765413&r2=1765414&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreRestoreImpl.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreRestoreImpl.java Tue Oct 18 11:10:44 2016
@@ -73,8 +73,7 @@ public class FileStoreRestoreImpl implem
store.getReader(),
store.getBlobStore(),
new WriterCacheManager.Default(),
- bufferWriter,
- store.getBinaryReferenceConsumer()
+ bufferWriter
);
SegmentGCOptions gcOptions = defaultGCOptions().setOffline();
Compactor compactor = new Compactor(
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java?rev=1765414&r1=1765413&r2=1765414&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java Tue Oct 18 11:10:44 2016
@@ -157,7 +157,7 @@ public class SegmentBlob extends Record
}
@CheckForNull
- static String readBlobId(@Nonnull Segment segment, int recordNumber) {
+ public static String readBlobId(@Nonnull Segment segment, int recordNumber) {
byte head = segment.readByte(recordNumber);
if ((head & 0xf0) == 0xe0) {
// 1110 xxxx: external value, small blob ID
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java?rev=1765414&r1=1765413&r2=1765414&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java Tue Oct 18 11:10:44 2016
@@ -44,7 +44,6 @@ import static org.apache.jackrabbit.oak.
import static org.apache.jackrabbit.oak.api.Type.NAME;
import static org.apache.jackrabbit.oak.api.Type.NAMES;
import static org.apache.jackrabbit.oak.api.Type.STRING;
-import static org.apache.jackrabbit.oak.segment.BinaryReferences.newReference;
import static org.apache.jackrabbit.oak.segment.MapRecord.BUCKETS_PER_LEVEL;
import static org.apache.jackrabbit.oak.segment.RecordWriters.newNodeStateWriter;
@@ -115,9 +114,6 @@ public class SegmentWriter {
private final WriteOperationHandler writeOperationHandler;
@Nonnull
- private final BinaryReferenceConsumer binaryReferenceConsumer;
-
- @Nonnull
private final SynchronizedDescriptiveStatistics nodeCompactTimeStats =
new SynchronizedDescriptiveStatistics(NODE_WRITER_STATS_WINDOW);
@@ -135,19 +131,18 @@ public class SegmentWriter {
* @param cacheManager cache manager instance for the de-duplication caches used by this writer
* @param writeOperationHandler handler for write operations.
*/
- public SegmentWriter(@Nonnull SegmentStore store,
- @Nonnull SegmentReader reader,
- @Nullable BlobStore blobStore,
- @Nonnull WriterCacheManager cacheManager,
- @Nonnull WriteOperationHandler writeOperationHandler,
- @Nonnull BinaryReferenceConsumer binaryReferenceConsumer
+ public SegmentWriter(
+ @Nonnull SegmentStore store,
+ @Nonnull SegmentReader reader,
+ @Nullable BlobStore blobStore,
+ @Nonnull WriterCacheManager cacheManager,
+ @Nonnull WriteOperationHandler writeOperationHandler
) {
this.store = checkNotNull(store);
this.reader = checkNotNull(reader);
this.blobStore = blobStore;
this.cacheManager = checkNotNull(cacheManager);
this.writeOperationHandler = checkNotNull(writeOperationHandler);
- this.binaryReferenceConsumer = checkNotNull(binaryReferenceConsumer);
}
/**
@@ -801,16 +796,12 @@ public class SegmentWriter {
String binaryReference;
if (data.length < Segment.BLOB_ID_SMALL_LIMIT) {
- binaryReference = newReference(blobId);
recordId = RecordWriters.newBlobIdWriter(data).write(writer);
} else {
RecordId refId = writeString(blobId);
- binaryReference = newReference(refId);
recordId = RecordWriters.newBlobIdWriter(refId).write(writer);
}
- binaryReferenceConsumer.consume(writer.getGeneration(), recordId.asUUID(), binaryReference);
-
return recordId;
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriterBuilder.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriterBuilder.java?rev=1765414&r1=1765413&r2=1765414&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriterBuilder.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriterBuilder.java Tue Oct 18 11:10:44 2016
@@ -148,8 +148,7 @@ public final class SegmentWriterBuilder
store.getReader(),
store.getBlobStore(),
cacheManager,
- createWriter(store, pooled),
- store.getBinaryReferenceConsumer()
+ createWriter(store, pooled)
);
}
@@ -163,8 +162,7 @@ public final class SegmentWriterBuilder
store.getReader(),
store.getBlobStore(),
cacheManager,
- createWriter(store, pooled),
- store.getBinaryReferenceConsumer()
+ createWriter(store, pooled)
);
}
@@ -178,8 +176,7 @@ public final class SegmentWriterBuilder
store.getReader(),
store.getBlobStore(),
cacheManager,
- createWriter(store, pooled),
- store.getBinaryReferenceConsumer()
+ createWriter(store, pooled)
);
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java?rev=1765414&r1=1765413&r2=1765414&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java Tue Oct 18 11:10:44 2016
@@ -32,7 +32,6 @@ import static java.util.concurrent.TimeU
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.apache.jackrabbit.oak.commons.IOUtils.humanReadableByteCount;
import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
-import static org.apache.jackrabbit.oak.segment.BinaryReferences.newReferenceReader;
import static org.apache.jackrabbit.oak.segment.SegmentId.isDataSegmentId;
import static org.apache.jackrabbit.oak.segment.SegmentWriterBuilder.segmentWriterBuilder;
import static org.apache.jackrabbit.oak.segment.file.GCListener.Status.FAILURE;
@@ -72,14 +71,15 @@ import com.google.common.base.Stopwatch;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableList;
-
import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics;
import org.apache.jackrabbit.oak.api.jmx.CacheStatsMBean;
import org.apache.jackrabbit.oak.plugins.blob.ReferenceCollector;
-import org.apache.jackrabbit.oak.segment.BinaryReferenceConsumer;
import org.apache.jackrabbit.oak.segment.Compactor;
import org.apache.jackrabbit.oak.segment.RecordId;
+import org.apache.jackrabbit.oak.segment.RecordType;
import org.apache.jackrabbit.oak.segment.Segment;
+import org.apache.jackrabbit.oak.segment.Segment.RecordConsumer;
+import org.apache.jackrabbit.oak.segment.SegmentBlob;
import org.apache.jackrabbit.oak.segment.SegmentBufferWriter;
import org.apache.jackrabbit.oak.segment.SegmentId;
import org.apache.jackrabbit.oak.segment.SegmentIdTable;
@@ -113,9 +113,6 @@ public class FileStore extends AbstractF
@Nonnull
private final SegmentWriter segmentWriter;
- @Nonnull
- private final BinaryReferenceConsumer binaryReferenceConsumer;
-
private final int maxFileSize;
@Nonnull
@@ -169,18 +166,6 @@ public class FileStore extends AbstractF
+ " is in use by another store.", ex);
}
- this.binaryReferenceConsumer = new BinaryReferenceConsumer() {
- @Override
- public void consume(int generation, UUID segmentId, String binaryReference) {
- fileStoreLock.writeLock().lock();
- try {
- tarWriter.addBinaryReference(generation, segmentId, binaryReference);
- } finally {
- fileStoreLock.writeLock().unlock();
- }
- }
- };
-
this.segmentWriter = segmentWriterBuilder("sys")
.withGeneration(new Supplier<Integer>() {
@Override
@@ -458,11 +443,6 @@ public class FileStore extends AbstractF
return segmentWriter;
}
- @Nonnull
- public BinaryReferenceConsumer getBinaryReferenceConsumer() {
- return binaryReferenceConsumer;
- }
-
@Override
@Nonnull
public TarRevisions getRevisions() {
@@ -661,11 +641,8 @@ public class FileStore extends AbstractF
// (potentially) flushing the TAR file.
if (segment != null) {
- UUID from = segment.getSegmentId().asUUID();
-
- for (int i = 0; i < segment.getReferencedSegmentIdCount(); i++) {
- tarWriter.addGraphEdge(from, segment.getReferencedSegmentId(i));
- }
+ populateTarGraph(segment);
+ populateTarBinaryReferences(segment);
}
// Close the TAR file if the size exceeds the maximum.
@@ -684,6 +661,28 @@ public class FileStore extends AbstractF
}
}
+ private void populateTarGraph(Segment segment) {
+ UUID from = segment.getSegmentId().asUUID();
+ for (int i = 0; i < segment.getReferencedSegmentIdCount(); i++) {
+ tarWriter.addGraphEdge(from, segment.getReferencedSegmentId(i));
+ }
+ }
+
+ private void populateTarBinaryReferences(final Segment segment) {
+ final int generation = segment.getGcGeneration();
+ final UUID id = segment.getSegmentId().asUUID();
+ segment.forEachRecord(new RecordConsumer() {
+
+ @Override
+ public void consume(int number, RecordType type, int offset) {
+ if (type == RecordType.BLOB_ID) {
+ tarWriter.addBinaryReference(generation, id, SegmentBlob.readBlobId(segment, number));
+ }
+ }
+
+ });
+ }
+
/**
* Switch to a new tar writer.
* This method may only be called when holding the write lock of {@link #fileStoreLock}
@@ -976,7 +975,7 @@ public class FileStore extends AbstractF
throws IOException {
if (gcOptions.isOffline()) {
BlobStore blobStore = getBlobStore();
- SegmentWriter writer = new SegmentWriter(FileStore.this, segmentReader, blobStore, new Default(), bufferWriter, binaryReferenceConsumer);
+ SegmentWriter writer = new SegmentWriter(FileStore.this, segmentReader, blobStore, new Default(), bufferWriter);
return new Compactor(segmentReader, writer, blobStore, cancel, gcOptions)
.compact(EMPTY_NODE, head, EMPTY_NODE);
} else {
@@ -1209,7 +1208,7 @@ public class FileStore extends AbstractF
int minGeneration = getGcGeneration() - gcOptions.getRetainedGenerations() + 1;
for (TarReader tarReader : tarReaders) {
- tarReader.collectBlobReferences(collector, newReferenceReader(FileStore.this), minGeneration);
+ tarReader.collectBlobReferences(collector, minGeneration);
}
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java?rev=1765414&r1=1765413&r2=1765414&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java Tue Oct 18 11:10:44 2016
@@ -55,7 +55,6 @@ import java.util.zip.CRC32;
import javax.annotation.Nonnull;
import com.google.common.base.Charsets;
-import com.google.common.base.Function;
import com.google.common.base.Predicate;
import com.google.common.collect.Sets;
import org.apache.commons.io.FileUtils;
@@ -735,13 +734,9 @@ class TarReader implements Closeable {
* Collect the references of those blobs that are reachable from any segment with a
* generation at or above {@code minGeneration}.
* @param collector
- * @param referenceDecoder
* @param minGeneration
*/
- void collectBlobReferences(
- @Nonnull ReferenceCollector collector,
- @Nonnull Function<String, String> referenceDecoder,
- int minGeneration) {
+ void collectBlobReferences(@Nonnull ReferenceCollector collector, int minGeneration) {
Map<Integer, Map<UUID, Set<String>>> generations = getBinaryReferences();
if (generations == null) {
@@ -755,7 +750,7 @@ class TarReader implements Closeable {
for (Set<String> references : entry.getValue().values()) {
for (String reference : references) {
- collector.addReference(referenceDecoder.apply(reference), null);
+ collector.addReference(reference, null);
}
}
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/http/HttpStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/http/HttpStore.java?rev=1765414&r1=1765413&r2=1765414&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/http/HttpStore.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/http/HttpStore.java Tue Oct 18 11:10:44 2016
@@ -35,8 +35,6 @@ import javax.annotation.Nonnull;
import com.google.common.base.Supplier;
import com.google.common.io.ByteStreams;
-import org.apache.jackrabbit.oak.segment.BinaryReferenceConsumer;
-import org.apache.jackrabbit.oak.segment.BinaryReferences;
import org.apache.jackrabbit.oak.segment.CachingSegmentReader;
import org.apache.jackrabbit.oak.segment.Revisions;
import org.apache.jackrabbit.oak.segment.Segment;
@@ -113,11 +111,6 @@ public class HttpStore implements Segmen
return revisions;
}
- @Nonnull
- public BinaryReferenceConsumer getBinaryReferenceConsumer() {
- return BinaryReferences.newDiscardBinaryReferenceConsumer();
- }
-
@Override
@Nonnull
public SegmentId newSegmentId(long msb, long lsb) {
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/memory/MemoryStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/memory/MemoryStore.java?rev=1765414&r1=1765413&r2=1765414&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/memory/MemoryStore.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/memory/MemoryStore.java Tue Oct 18 11:10:44 2016
@@ -29,8 +29,6 @@ import javax.annotation.Nonnull;
import com.google.common.base.Supplier;
import com.google.common.collect.Maps;
-import org.apache.jackrabbit.oak.segment.BinaryReferenceConsumer;
-import org.apache.jackrabbit.oak.segment.BinaryReferences;
import org.apache.jackrabbit.oak.segment.CachingSegmentReader;
import org.apache.jackrabbit.oak.segment.Revisions;
import org.apache.jackrabbit.oak.segment.Segment;
@@ -107,11 +105,6 @@ public class MemoryStore implements Segm
return revisions;
}
- @Nonnull
- public BinaryReferenceConsumer getBinaryReferenceConsumer() {
- return BinaryReferences.newDiscardBinaryReferenceConsumer();
- }
-
@Override
public boolean containsSegment(SegmentId id) {
return id.sameStore(this) || segments.containsKey(id);