You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by md...@apache.org on 2015/04/15 09:13:17 UTC
svn commit: r1673664 - in
/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file:
FileStore.java TarReader.java
Author: mduerig
Date: Wed Apr 15 07:13:17 2015
New Revision: 1673664
URL: http://svn.apache.org/r1673664
Log:
OAK-2723: FileStore does not scale because of precomputed graph on TarReader
Don't buffer the graph data in memory.
Credits to Andrei Dulvac for the initial patch
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java?rev=1673664&r1=1673663&r2=1673664&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java Wed Apr 15 07:13:17 2015
@@ -929,7 +929,10 @@ public class FileStore implements Segmen
for (UUID uuid : reader.getUUIDs()) {
graph.put(uuid, null);
}
- graph.putAll(reader.getGraph());
+ Map<UUID, List<UUID>> g = reader.getGraph();
+ if (g != null) {
+ graph.putAll(g);
+ }
return graph;
}
}
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java?rev=1673664&r1=1673663&r2=1673664&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java Wed Apr 15 07:13:17 2015
@@ -24,7 +24,6 @@ import static com.google.common.collect.
import static com.google.common.collect.Maps.newTreeMap;
import static com.google.common.collect.Sets.newHashSet;
import static com.google.common.collect.Sets.newHashSetWithExpectedSize;
-import static java.util.Collections.emptyMap;
import static java.util.Collections.singletonList;
import static org.apache.jackrabbit.oak.plugins.segment.Segment.REF_COUNT_OFFSET;
import static org.apache.jackrabbit.oak.plugins.segment.SegmentId.isDataSegmentId;
@@ -426,64 +425,16 @@ class TarReader {
}
}
- /**
- * Loads the optional pre-compiled graph entry from the given tar file.
- *
- * @return graph buffer, or {@code null} if one was not found
- * @throws IOException if the tar file could not be read
- */
- private static ByteBuffer loadGraph(
- File file, FileAccess access, ByteBuffer index) throws IOException {
- // read the graph metadata just before the tar index entry
- int pos = access.length() - 2 * BLOCK_SIZE - getEntrySize(index.remaining());
- ByteBuffer meta = access.read(pos - 16, 16);
- int crc32 = meta.getInt();
- int count = meta.getInt();
- int bytes = meta.getInt();
- int magic = meta.getInt();
-
- if (magic != GRAPH_MAGIC) {
- return null; // magic byte mismatch
- }
-
- if (count < 0 || bytes < count * 16 + 16 || BLOCK_SIZE + bytes > pos) {
- log.warn("Invalid graph metadata in tar file {}", file);
- return null; // impossible uuid and/or byte counts
- }
-
- // this involves seeking backwards in the file, which might not
- // perform well, but that's OK since we only do this once per file
- ByteBuffer graph = access.read(pos - bytes, bytes);
-
- byte[] b = new byte[bytes - 16];
- graph.mark();
- graph.get(b);
- graph.reset();
-
- CRC32 checksum = new CRC32();
- checksum.update(b);
- if (crc32 != (int) checksum.getValue()) {
- log.warn("Invalid graph checksum in tar file {}", file);
- return null; // checksum mismatch
- }
-
- return graph;
- }
-
private final File file;
private final FileAccess access;
private final ByteBuffer index;
- private final ByteBuffer graph;
-
- private TarReader(File file, FileAccess access, ByteBuffer index)
- throws IOException {
+ private TarReader(File file, FileAccess access, ByteBuffer index) {
this.file = file;
this.access = access;
this.index = index;
- this.graph = loadGraph(file, access, index);
}
long size() {
@@ -622,10 +573,8 @@ class TarReader {
*/
synchronized TarReader cleanup(Set<UUID> referencedIds, CompactionMap cm) throws IOException {
Set<UUID> cleaned = newHashSet();
- Map<UUID, List<UUID>> graph = null;
- if (this.graph != null) {
- graph = parseGraph();
- }
+ Map<UUID, List<UUID>> graph = getGraph();
+
TarEntry[] sorted = new TarEntry[index.remaining() / 24];
int position = index.position();
for (int i = 0; position < index.limit(); i++) {
@@ -778,19 +727,70 @@ class TarReader {
//-----------------------------------------------------------< private >--
+ /**
+ * Loads and parses the optional pre-compiled graph entry from the given tar
+ * file.
+ *
+ * @return the parsed graph, or {@code null} if one was not found
+ * @throws IOException if the tar file could not be read
+ */
Map<UUID, List<UUID>> getGraph() throws IOException {
+ ByteBuffer graph = loadGraph();
if (graph == null) {
- return emptyMap();
+ return null;
} else {
- return parseGraph();
+ return parseGraph(graph);
+ }
+ }
+
+ /**
+ * Loads the optional pre-compiled graph entry from the given tar file.
+ *
+ * @return graph buffer, or {@code null} if one was not found
+ * @throws IOException if the tar file could not be read
+ */
+ private ByteBuffer loadGraph() throws IOException {
+ // read the graph metadata just before the tar index entry
+ int pos = access.length() - 2 * BLOCK_SIZE - getEntrySize(index.remaining());
+ ByteBuffer meta = access.read(pos - 16, 16);
+ int crc32 = meta.getInt();
+ int count = meta.getInt();
+ int bytes = meta.getInt();
+ int magic = meta.getInt();
+
+ if (magic != GRAPH_MAGIC) {
+ return null; // magic byte mismatch
}
+
+ if (count < 0 || bytes < count * 16 + 16 || BLOCK_SIZE + bytes > pos) {
+ log.warn("Invalid graph metadata in tar file {}", file);
+ return null; // impossible uuid and/or byte counts
+ }
+
+ // this involves seeking backwards in the file, which might not
+ // perform well, but that's OK since we only do this once per file
+ ByteBuffer graph = access.read(pos - bytes, bytes);
+
+ byte[] b = new byte[bytes - 16];
+ graph.mark();
+ graph.get(b);
+ graph.reset();
+
+ CRC32 checksum = new CRC32();
+ checksum.update(b);
+ if (crc32 != (int) checksum.getValue()) {
+ log.warn("Invalid graph checksum in tar file {}", file);
+ return null; // checksum mismatch
+ }
+
+ return graph;
}
- private Map<UUID, List<UUID>> parseGraph() throws IOException {
- int count = graph.getInt(graph.limit() - 12);
+ private static Map<UUID, List<UUID>> parseGraph(ByteBuffer graphByteBuffer) {
+ int count = graphByteBuffer.getInt(graphByteBuffer.limit() - 12);
- ByteBuffer buffer = graph.duplicate();
- buffer.limit(graph.limit() - 16);
+ ByteBuffer buffer = graphByteBuffer.duplicate();
+ buffer.limit(graphByteBuffer.limit() - 16);
List<UUID> uuids = newArrayListWithCapacity(count);
for (int i = 0; i < count; i++) {