You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by al...@apache.org on 2016/09/06 13:43:50 UTC
svn commit: r1759434 - in /jackrabbit/oak/trunk/oak-segment-tar/src:
main/java/org/apache/jackrabbit/oak/segment/file/
test/java/org/apache/jackrabbit/oak/segment/file/
Author: alexparvulescu
Date: Tue Sep 6 13:43:50 2016
New Revision: 1759434
URL: http://svn.apache.org/viewvc?rev=1759434&view=rev
Log:
OAK-4746 Move writeFile into the TarWriter
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarWriter.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreStatsTest.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/TarWriterTest.java
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java?rev=1759434&r1=1759433&r2=1759434&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java Tue Sep 6 13:43:50 2016
@@ -129,7 +129,7 @@ public class FileStore implements Segmen
private static final Pattern FILE_NAME_PATTERN =
Pattern.compile("(data|bulk)((0|[1-9][0-9]*)[0-9]{4})([a-z])?.tar");
- private static final String FILE_NAME_FORMAT = "data%05d%s.tar";
+ static final String FILE_NAME_FORMAT = "data%05d%s.tar";
private static final String LOCK_FILE_NAME = "repo.lock";
@@ -165,10 +165,6 @@ public class FileStore implements Segmen
private volatile List<TarReader> readers;
- private int writeNumber;
-
- private volatile File writeFile;
-
private volatile TarWriter tarWriter;
private final RandomAccessFile lockFile;
@@ -330,14 +326,13 @@ public class FileStore implements Segmen
this.stats = new FileStoreStats(builder.getStatsProvider(), this, size());
if (!readOnly) {
+ int writeNumber = 0;
if (indices.length > 0) {
- this.writeNumber = indices[indices.length - 1] + 1;
- } else {
- this.writeNumber = 0;
+ writeNumber = indices[indices.length - 1] + 1;
}
- this.writeFile = new File(directory, format(
- FILE_NAME_FORMAT, writeNumber, "a"));
- this.tarWriter = new TarWriter(writeFile, stats);
+ this.tarWriter = new TarWriter(directory, stats, writeNumber);
+ } else {
+ this.tarWriter = null;
}
// FIXME OAK-4621: External invocation of background operations
@@ -655,7 +650,7 @@ public class FileStore implements Segmen
fileStoreLock.readLock().lock();
try {
readersSnapshot = ImmutableList.copyOf(readers);
- writeFileSnapshotSize = writeFile != null ? writeFile.length() : 0;
+ writeFileSnapshotSize = tarWriter != null ? tarWriter.fileLength() : 0;
} finally {
fileStoreLock.readLock().unlock();
}
@@ -1452,20 +1447,15 @@ public class FileStore implements Segmen
* @throws IOException
*/
private void newWriter() throws IOException {
- if (tarWriter.isDirty()) {
- tarWriter.close();
-
+ TarWriter newWriter = tarWriter.createNextGeneration();
+ if (newWriter != tarWriter) {
+ File writeFile = tarWriter.getFile();
List<TarReader> list =
newArrayListWithCapacity(1 + readers.size());
list.add(TarReader.open(writeFile, memoryMapping));
list.addAll(readers);
readers = list;
-
- writeNumber++;
- writeFile = new File(
- directory,
- format(FILE_NAME_FORMAT, writeNumber, "a"));
- tarWriter = new TarWriter(writeFile, stats);
+ tarWriter = newWriter;
}
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarWriter.java?rev=1759434&r1=1759433&r2=1759434&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarWriter.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarWriter.java Tue Sep 6 13:43:50 2016
@@ -27,6 +27,8 @@ import static com.google.common.collect.
import static com.google.common.collect.Maps.newHashMap;
import static com.google.common.collect.Maps.newLinkedHashMap;
import static com.google.common.collect.Sets.newHashSet;
+import static java.lang.String.format;
+import static org.apache.jackrabbit.oak.segment.file.FileStore.FILE_NAME_FORMAT;
import java.io.Closeable;
import java.io.File;
@@ -110,6 +112,8 @@ class TarWriter implements Closeable {
}
}
+ private final int writeIndex;
+
/**
* The file being written. This instance is also used as an additional
* synchronization point by {@link #flush()} and {@link #close()} to
@@ -157,13 +161,20 @@ class TarWriter implements Closeable {
*/
private final Map<UUID, Set<UUID>> graph = newHashMap();
+ /**
+ * Used for maintenance operations (GC or recovery) via the TarReader and tests
+ */
TarWriter(File file) {
- this(file, FileStoreMonitor.DEFAULT);
+ this.file = file;
+ this.monitor = FileStoreMonitor.DEFAULT;
+ this.writeIndex = -1;
}
- TarWriter(File file, FileStoreMonitor monitor) {
- this.file = file;
+ TarWriter(File directory, FileStoreMonitor monitor, int writeIndex) {
+ this.file = new File(directory, format(FILE_NAME_FORMAT, writeIndex,
+ "a"));
this.monitor = monitor;
+ this.writeIndex = writeIndex;
}
/**
@@ -308,10 +319,6 @@ class TarWriter implements Closeable {
}
}
- boolean isDirty() {
- return access != null;
- }
-
/**
* Closes this tar file.
*
@@ -352,6 +359,23 @@ class TarWriter implements Closeable {
monitor.written(currentPosition - initialPosition);
}
+ /**
+ * If the current instance is dirty, this will return a new TarWriter based
+ * on the next generation of the file being written to by incrementing the
+ * internal {@link #writeIndex} counter. Otherwise it will return the
+ * current instance.
+ */
+ synchronized TarWriter createNextGeneration() throws IOException {
+ checkState(writeIndex >= 0);
+ // If nothing was written to this file, then we're already done.
+ if (access == null) {
+ return this;
+ }
+ close();
+ int newIndex = writeIndex + 1;
+ return new TarWriter(file.getParentFile(), monitor, newIndex);
+ }
+
private void writeBinaryReferences() throws IOException {
int binaryReferenceSize = 0;
@@ -625,6 +649,18 @@ class TarWriter implements Closeable {
}
}
+ synchronized long fileLength() {
+ return file.length();
+ }
+
+ synchronized File getFile() {
+ return file;
+ }
+
+ synchronized boolean isClosed() {
+ return closed;
+ }
+
//------------------------------------------------------------< Object >--
@Override
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreStatsTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreStatsTest.java?rev=1759434&r1=1759433&r2=1759434&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreStatsTest.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreStatsTest.java Tue Sep 6 13:43:50 2016
@@ -73,31 +73,30 @@ public class FileStoreStatsTest {
@Test
public void tarWriterIntegration() throws Exception{
StatisticsProvider statsProvider = new DefaultStatisticsProvider(executor);
- FileStore store = fileStoreBuilder(segmentFolder.newFolder())
+ File directory = segmentFolder.newFolder();
+ FileStore store = fileStoreBuilder(directory)
.withStatisticsProvider(statsProvider)
.build();
-
FileStoreStats stats = new FileStoreStats(statsProvider, store, 0);
-
try {
long initialSize = stats.getApproximateSize();
-
UUID id = UUID.randomUUID();
long msb = id.getMostSignificantBits();
long lsb = id.getLeastSignificantBits() & (-1 >>> 4); // OAK-1672
byte[] data = "Hello, World!".getBytes(UTF_8);
- File file = segmentFolder.newFile();
- try (TarWriter writer = new TarWriter(file, stats)) {
+ TarWriter writer = null;
+ try {
+ writer = new TarWriter(directory, stats, 0);
writer.writeEntry(msb, lsb, data, 0, data.length, 0);
+ } finally {
+ writer.close();
}
-
- assertEquals(stats.getApproximateSize() - initialSize, file.length());
-
+ assertEquals(stats.getApproximateSize() - initialSize,
+ writer.fileLength());
} finally {
store.close();
}
-
assertEquals(1, stats.getJournalWriteStatsAsCount());
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/TarWriterTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/TarWriterTest.java?rev=1759434&r1=1759433&r2=1759434&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/TarWriterTest.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/TarWriterTest.java Tue Sep 6 13:43:50 2016
@@ -19,11 +19,13 @@
package org.apache.jackrabbit.oak.segment.file;
+import static com.google.common.base.Charsets.UTF_8;
import static com.google.common.collect.Maps.newHashMap;
import static com.google.common.collect.Sets.newHashSet;
import static java.nio.ByteBuffer.allocate;
import static java.util.Collections.singleton;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
import java.io.File;
@@ -34,6 +36,7 @@ import java.util.Set;
import java.util.UUID;
import com.google.common.collect.ImmutableList;
+
import org.apache.jackrabbit.oak.segment.RecordId;
import org.apache.jackrabbit.oak.segment.Segment;
import org.apache.jackrabbit.oak.segment.SegmentId;
@@ -233,4 +236,28 @@ public class TarWriterTest {
}
}
+ @Test
+ public void createNextGenerationTest() throws IOException {
+ int counter = 2222;
+ TarWriter t0 = new TarWriter(folder.newFolder(),
+ FileStoreMonitor.DEFAULT, counter);
+
+ // not dirty, will not create a new writer
+ TarWriter t1 = t0.createNextGeneration();
+ assertEquals(t0, t1);
+ assertTrue(t1.getFile().getName().contains("" + counter));
+
+ // dirty, will create a new writer
+ UUID id = UUID.randomUUID();
+ long msb = id.getMostSignificantBits();
+ long lsb = id.getLeastSignificantBits() & (-1 >>> 4); // OAK-1672
+ byte[] data = "Hello, World!".getBytes(UTF_8);
+ t1.writeEntry(msb, lsb, data, 0, data.length, 0);
+
+ TarWriter t2 = t1.createNextGeneration();
+ assertNotEquals(t1, t2);
+ assertTrue(t1.isClosed());
+ assertTrue(t2.getFile().getName().contains("" + (counter + 1)));
+ }
+
}