You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by md...@apache.org on 2017/03/01 17:26:33 UTC
svn commit: r1785000 - in /jackrabbit/oak/trunk/oak-segment-tar/src:
main/java/org/apache/jackrabbit/oak/backup/impl/
main/java/org/apache/jackrabbit/oak/segment/
test/java/org/apache/jackrabbit/oak/segment/
Author: mduerig
Date: Wed Mar 1 17:26:32 2017
New Revision: 1785000
URL: http://svn.apache.org/viewvc?rev=1785000&view=rev
Log:
OAK-3690: Decouple SegmentBufferWriter from SegmentStore
Remove the SegmentStore field from SegmentBufferWriter and related classes
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreBackupImpl.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreRestoreImpl.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/RecordWriters.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriterPool.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriterBuilder.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/WriteOperationHandler.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriterPoolTest.java
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreBackupImpl.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreBackupImpl.java?rev=1785000&r1=1784999&r2=1785000&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreBackupImpl.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreBackupImpl.java Wed Mar 1 17:26:32 2017
@@ -68,7 +68,6 @@ public class FileStoreBackupImpl impleme
try {
int gen = current.getRecordId().getSegmentId().getGcGeneration();
SegmentBufferWriter bufferWriter = new SegmentBufferWriter(
- backup,
backup.getSegmentIdProvider(),
backup.getReader(),
"b",
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreRestoreImpl.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreRestoreImpl.java?rev=1785000&r1=1784999&r2=1785000&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreRestoreImpl.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreRestoreImpl.java Wed Mar 1 17:26:32 2017
@@ -62,7 +62,6 @@ public class FileStoreRestoreImpl implem
SegmentNodeState head = restore.getHead();
int gen = head.getRecordId().getSegmentId().getGcGeneration();
SegmentBufferWriter bufferWriter = new SegmentBufferWriter(
- store,
store.getSegmentIdProvider(),
store.getReader(),
"r",
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/RecordWriters.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/RecordWriters.java?rev=1785000&r1=1784999&r2=1785000&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/RecordWriters.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/RecordWriters.java Wed Mar 1 17:26:32 2017
@@ -66,8 +66,8 @@ final class RecordWriters {
this(type, size, Collections.<RecordId> emptyList());
}
- public final RecordId write(SegmentBufferWriter writer) throws IOException {
- RecordId id = writer.prepare(type, size, ids);
+ public final RecordId write(SegmentBufferWriter writer, SegmentStore store) throws IOException {
+ RecordId id = writer.prepare(type, size, ids, store);
return writeRecordContent(id, writer);
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java?rev=1785000&r1=1784999&r2=1785000&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java Wed Mar 1 17:26:32 2017
@@ -104,9 +104,6 @@ public class SegmentBufferWriter impleme
private MutableSegmentReferences segmentReferences = new MutableSegmentReferences();
@Nonnull
- private final SegmentStore store;
-
- @Nonnull
private final SegmentIdProvider idProvider;
@Nonnull
@@ -148,12 +145,10 @@ public class SegmentBufferWriter impleme
*/
private boolean dirty;
- public SegmentBufferWriter(@Nonnull SegmentStore store,
- @Nonnull SegmentIdProvider idProvider,
+ public SegmentBufferWriter(@Nonnull SegmentIdProvider idProvider,
@Nonnull SegmentReader reader,
@CheckForNull String wid,
int generation) {
- this.store = checkNotNull(store);
this.idProvider = checkNotNull(idProvider);
this.reader = checkNotNull(reader);
this.wid = (wid == null
@@ -161,8 +156,6 @@ public class SegmentBufferWriter impleme
: wid);
this.generation = generation;
- this.statistics = new Statistics();
- newSegment();
}
@Nonnull
@@ -187,7 +180,7 @@ public class SegmentBufferWriter impleme
* </ul>
* The segment meta data is guaranteed to be the first string record in a segment.
*/
- private void newSegment() {
+ private void newSegment(SegmentStore store) throws IOException {
buffer = new byte[Segment.MAX_SEGMENT_SIZE];
buffer[0] = '0';
buffer[1] = 'a';
@@ -209,17 +202,13 @@ public class SegmentBufferWriter impleme
"{\"wid\":\"" + wid + '"' +
",\"sno\":" + idProvider.getSegmentIdCount() +
",\"t\":" + currentTimeMillis() + "}";
- try {
- segment = new Segment(idProvider.newDataSegmentId(), reader, buffer, recordNumbers, segmentReferences, metaInfo);
+ segment = new Segment(idProvider.newDataSegmentId(), reader, buffer, recordNumbers, segmentReferences, metaInfo);
- statistics = new Statistics();
- statistics.id = segment.getSegmentId();
+ statistics = new Statistics();
+ statistics.id = segment.getSegmentId();
- byte[] data = metaInfo.getBytes(UTF_8);
- RecordWriters.newValueWriter(data.length, data).write(this);
- } catch (IOException e) {
- LOG.error("Unable to write meta info to segment {} {}", segment.getSegmentId(), metaInfo, e);
- }
+ byte[] data = metaInfo.getBytes(UTF_8);
+ RecordWriters.newValueWriter(data.length, data).write(this, store);
dirty = false;
}
@@ -332,7 +321,7 @@ public class SegmentBufferWriter impleme
* enough space for a record. It can also be called explicitly.
*/
@Override
- public void flush() throws IOException {
+ public void flush(@Nonnull SegmentStore store) throws IOException {
if (dirty) {
int referencedSegmentIdCount = segmentReferences.size();
BinaryUtils.writeInt(buffer, Segment.REFERENCED_SEGMENT_ID_COUNT_OFFSET, referencedSegmentIdCount);
@@ -380,7 +369,7 @@ public class SegmentBufferWriter impleme
SegmentId segmentId = segment.getSegmentId();
LOG.debug("Writing data segment: {} ", statistics);
store.writeSegment(segmentId, buffer, buffer.length - length, length);
- newSegment();
+ newSegment(store);
}
}
@@ -395,16 +384,22 @@ public class SegmentBufferWriter impleme
* the write position (records are written from the end to the beginning,
* but within a record from left to right).
*
- * @param type the record type (only used for root records)
- * @param size the size of the record, excluding the size used for the
- * record ids
- * @param ids the record ids
+ * @param type the record type (only used for root records)
+ * @param size the size of the record, excluding the size used for the
+ * record ids
+ * @param ids the record ids
+ * @param store the {@code SegmentStore} instance to write full segments to
* @return a new record id
*/
- public RecordId prepare(RecordType type, int size, Collection<RecordId> ids) throws IOException {
+ public RecordId prepare(RecordType type, int size, Collection<RecordId> ids, SegmentStore store) throws IOException {
checkArgument(size >= 0);
checkNotNull(ids);
+ if (segment == null) {
+ // Create a segment first if this is the first time this segment buffer writer is used.
+ newSegment(store);
+ }
+
int idCount = ids.size();
int recordSize = align(size + idCount * RECORD_ID_BYTES, 1 << Segment.RECORD_ALIGN_BITS);
@@ -440,7 +435,7 @@ public class SegmentBufferWriter impleme
}
if (segmentSize > buffer.length) {
- flush();
+ flush(store);
}
statistics.recordCount++;
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriterPool.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriterPool.java?rev=1785000&r1=1784999&r2=1785000&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriterPool.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriterPool.java Wed Mar 1 17:26:32 2017
@@ -69,9 +69,6 @@ public class SegmentBufferWriterPool imp
private final Set<SegmentBufferWriter> disposed = newHashSet();
@Nonnull
- private final SegmentStore store;
-
- @Nonnull
private final SegmentIdProvider idProvider;
@Nonnull
@@ -86,12 +83,10 @@ public class SegmentBufferWriterPool imp
private short writerId = -1;
public SegmentBufferWriterPool(
- @Nonnull SegmentStore store,
@Nonnull SegmentIdProvider idProvider,
@Nonnull SegmentReader reader,
@Nonnull String wid,
@Nonnull Supplier<Integer> gcGeneration) {
- this.store = checkNotNull(store);
this.idProvider = checkNotNull(idProvider);
this.reader = checkNotNull(reader);
this.wid = checkNotNull(wid);
@@ -110,7 +105,7 @@ public class SegmentBufferWriterPool imp
}
@Override
- public void flush() throws IOException {
+ public void flush(@Nonnull SegmentStore store) throws IOException {
List<SegmentBufferWriter> toFlush = newArrayList();
List<SegmentBufferWriter> toReturn = newArrayList();
@@ -145,7 +140,7 @@ public class SegmentBufferWriterPool imp
// Call flush from outside the pool monitor to avoid potential
// deadlocks of that method calling SegmentStore.writeSegment
for (SegmentBufferWriter writer : toFlush) {
- writer.flush();
+ writer.flush(store);
}
}
@@ -191,7 +186,6 @@ public class SegmentBufferWriterPool imp
SegmentBufferWriter writer = writers.remove(key);
if (writer == null) {
writer = new SegmentBufferWriter(
- store,
idProvider,
reader,
getWriterId(wid),
@@ -200,7 +194,6 @@ public class SegmentBufferWriterPool imp
} else if (writer.getGeneration() != gcGeneration.get()) {
disposed.add(writer);
writer = new SegmentBufferWriter(
- store,
idProvider,
reader,
getWriterId(wid),
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java?rev=1785000&r1=1784999&r2=1785000&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java Wed Mar 1 17:26:32 2017
@@ -146,7 +146,7 @@ public class SegmentWriter {
}
public void flush() throws IOException {
- writeOperationHandler.flush();
+ writeOperationHandler.flush(store);
}
/**
@@ -485,7 +485,7 @@ public class SegmentWriter {
return base.getRecordId();
} else {
return RecordWriters.newMapBranchWriter(entry.getHash(), asList(entry.getKey(),
- value, base.getRecordId())).write(writer);
+ value, base.getRecordId())).write(writer, store);
}
}
}
@@ -519,7 +519,7 @@ public class SegmentWriter {
checkElementIndex(size, MapRecord.MAX_SIZE);
checkPositionIndex(level, MapRecord.MAX_NUMBER_OF_LEVELS);
checkArgument(size != 0 || level == MapRecord.MAX_NUMBER_OF_LEVELS);
- return RecordWriters.newMapLeafWriter(level, entries).write(writer);
+ return RecordWriters.newMapLeafWriter(level, entries).write(writer, store);
}
private RecordId writeMapBranch(int level, int size, MapRecord... buckets) throws IOException {
@@ -531,7 +531,7 @@ public class SegmentWriter {
bucketIds.add(buckets[i].getRecordId());
}
}
- return RecordWriters.newMapBranchWriter(level, size, bitmap, bucketIds).write(writer);
+ return RecordWriters.newMapBranchWriter(level, size, bitmap, bucketIds).write(writer, store);
}
private RecordId writeMapBucket(MapRecord base, Collection<MapEntry> entries, int level)
@@ -541,7 +541,7 @@ public class SegmentWriter {
if (base != null) {
return base.getRecordId();
} else if (level == 0) {
- return RecordWriters.newMapLeafWriter().write(writer);
+ return RecordWriters.newMapLeafWriter().write(writer, store);
} else {
return null;
}
@@ -650,7 +650,7 @@ public class SegmentWriter {
private RecordId writeListBucket(List<RecordId> bucket) throws IOException {
checkArgument(bucket.size() > 1);
- return RecordWriters.newListBucketWriter(bucket).write(writer);
+ return RecordWriters.newListBucketWriter(bucket).write(writer, store);
}
private List<List<MapEntry>> splitToBuckets(Collection<MapEntry> entries, int level) {
@@ -673,12 +673,12 @@ public class SegmentWriter {
private RecordId writeValueRecord(long length, RecordId blocks) throws IOException {
long len = (length - Segment.MEDIUM_LIMIT) | (0x3L << 62);
- return RecordWriters.newValueWriter(blocks, len).write(writer);
+ return RecordWriters.newValueWriter(blocks, len).write(writer, store);
}
private RecordId writeValueRecord(int length, byte... data) throws IOException {
checkArgument(length < Segment.MEDIUM_LIMIT);
- return RecordWriters.newValueWriter(length, data).write(writer);
+ return RecordWriters.newValueWriter(length, data).write(writer, store);
}
/**
@@ -778,10 +778,10 @@ public class SegmentWriter {
RecordId recordId;
if (data.length < Segment.BLOB_ID_SMALL_LIMIT) {
- recordId = RecordWriters.newBlobIdWriter(data).write(writer);
+ recordId = RecordWriters.newBlobIdWriter(data).write(writer, store);
} else {
RecordId refId = writeString(blobId);
- recordId = RecordWriters.newBlobIdWriter(refId).write(writer);
+ recordId = RecordWriters.newBlobIdWriter(refId).write(writer, store);
}
return recordId;
@@ -791,7 +791,7 @@ public class SegmentWriter {
throws IOException {
checkNotNull(bytes);
checkPositionIndexes(offset, offset + length, bytes.length);
- return RecordWriters.newBlockWriter(bytes, offset, length).write(writer);
+ return RecordWriters.newBlockWriter(bytes, offset, length).write(writer, store);
}
private RecordId writeStream(@Nonnull InputStream stream) throws IOException {
@@ -890,9 +890,9 @@ public class SegmentWriter {
if (!type.isArray()) {
return valueIds.iterator().next();
} else if (count == 0) {
- return RecordWriters.newListWriter().write(writer);
+ return RecordWriters.newListWriter().write(writer, store);
} else {
- return RecordWriters.newListWriter(count, writeList(valueIds)).write(writer);
+ return RecordWriters.newListWriter(count, writeList(valueIds)).write(writer, store);
}
}
@@ -965,7 +965,7 @@ public class SegmentWriter {
RecordId tid = RecordWriters.newTemplateWriter(ids, propertyNames,
propertyTypes, head, primaryId, mixinIds, childNameId,
- propNamesId).write(writer);
+ propNamesId).write(writer, store);
templateCache.put(template, tid);
return tid;
}
@@ -1120,7 +1120,7 @@ public class SegmentWriter {
byte[] id = ((SegmentNodeState) state).getStableIdBytes();
stableId = writeBlock(id, 0, id.length);
}
- return newNodeStateWriter(stableId, ids).write(writer);
+ return newNodeStateWriter(stableId, ids).write(writer, store);
}
/**
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriterBuilder.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriterBuilder.java?rev=1785000&r1=1784999&r2=1785000&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriterBuilder.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriterBuilder.java Wed Mar 1 17:26:32 2017
@@ -174,7 +174,7 @@ public final class SegmentWriterBuilder
}
@Override
- public void flush() {
+ public void flush(@Nonnull SegmentStore store) {
throw new UnsupportedOperationException("Cannot write to read-only store");
}
});
@@ -199,7 +199,6 @@ public final class SegmentWriterBuilder
private WriteOperationHandler createWriter(@Nonnull FileStore store, boolean pooled) {
if (pooled) {
return new SegmentBufferWriterPool(
- store,
store.getSegmentIdProvider(),
store.getReader(),
name,
@@ -207,7 +206,6 @@ public final class SegmentWriterBuilder
);
} else {
return new SegmentBufferWriter(
- store,
store.getSegmentIdProvider(),
store.getReader(),
name,
@@ -220,7 +218,6 @@ public final class SegmentWriterBuilder
private WriteOperationHandler createWriter(@Nonnull MemoryStore store, boolean pooled) {
if (pooled) {
return new SegmentBufferWriterPool(
- store,
store.getSegmentIdProvider(),
store.getReader(),
name,
@@ -228,7 +225,6 @@ public final class SegmentWriterBuilder
);
} else {
return new SegmentBufferWriter(
- store,
store.getSegmentIdProvider(),
store.getReader(),
name,
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/WriteOperationHandler.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/WriteOperationHandler.java?rev=1785000&r1=1784999&r2=1785000&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/WriteOperationHandler.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/WriteOperationHandler.java Wed Mar 1 17:26:32 2017
@@ -58,7 +58,8 @@ interface WriteOperationHandler {
/**
* Flush any pending changes on any {@link SegmentBufferWriter} managed by this instance.
+ * @param store the {@code SegmentStore} instance to write the {@code Segment} to
* @throws IOException
*/
- void flush() throws IOException;
+ void flush(@Nonnull SegmentStore store) throws IOException;
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriterPoolTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriterPoolTest.java?rev=1785000&r1=1784999&r2=1785000&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriterPoolTest.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriterPoolTest.java Wed Mar 1 17:26:32 2017
@@ -51,7 +51,6 @@ public class SegmentBufferWriterPoolTest
private final RecordId rootId = store.getRevisions().getHead();
private final SegmentBufferWriterPool pool = new SegmentBufferWriterPool(
- store,
store.getSegmentIdProvider(),
store.getReader(),
"",
@@ -134,7 +133,7 @@ public class SegmentBufferWriterPoolTest
assertEquals(rootId, res3.get());
assertEquals(3, map1.size());
- pool.flush();
+ pool.flush(store);
ConcurrentMap<String, SegmentBufferWriter> map2 = newConcurrentMap();
Future<RecordId> res4 = execute(createOp("a", map2), 0);
@@ -164,7 +163,7 @@ public class SegmentBufferWriterPoolTest
executors[1].submit(new Callable<Void>() {
@Override
public Void call() throws Exception {
- pool.flush();
+ pool.flush(store);
return null;
}
}).get(100, MILLISECONDS);