You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ju...@apache.org on 2014/04/03 06:47:05 UTC
svn commit: r1584258 - in /jackrabbit/oak/trunk:
oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/
oak-run/src/main/java/org/apache/jackrabbit/oak/run/
Author: jukka
Date: Thu Apr 3 04:47:05 2014
New Revision: 1584258
URL: http://svn.apache.org/r1584258
Log:
OAK-631: SegmentMK: Implement garbage collection
Simplify TarWriter.cleanup()
Allow cleanup to proceed even when no commits are pending
Extend the oak-run debug feature to allow inspection of existing data
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java?rev=1584258&r1=1584257&r2=1584258&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java Thu Apr 3 04:47:05 2014
@@ -310,7 +310,8 @@ public class FileStore implements Segmen
synchronized (persistedHead) {
RecordId before = persistedHead.get();
RecordId after = head.get();
- if (!after.equals(before)) {
+ boolean cleanup = cleanupNeeded.getAndSet(false);
+ if (cleanup || !after.equals(before)) {
// needs to happen outside the synchronization block below to
// avoid a deadlock with another thread flushing the writer
tracker.getWriter().flush();
@@ -324,7 +325,7 @@ public class FileStore implements Segmen
journalFile.getChannel().force(false);
persistedHead.set(after);
- if (cleanupNeeded.getAndSet(false)) {
+ if (cleanup) {
Set<UUID> ids = newHashSet();
for (SegmentId id : tracker.getReferencedSegmentIds()) {
ids.add(new UUID(
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java?rev=1584258&r1=1584257&r2=1584258&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java Thu Apr 3 04:47:05 2014
@@ -93,6 +93,8 @@ class TarWriter {
*/
private final Map<UUID, TarEntry> index = newHashMap();
+ private final Set<UUID> references = newHashSet();
+
TarWriter(File file) {
this.file = file;
}
@@ -159,6 +161,18 @@ class TarWriter {
(int) (length - size - padding), size);
index.put(uuid, entry);
+ if (isDataSegmentId(uuid.getLeastSignificantBits())) {
+ ByteBuffer segment = ByteBuffer.wrap(data, offset, size);
+ int pos = segment.position();
+ int refcount = segment.get(pos + REF_COUNT_OFFSET) & 0xff;
+ int refend = pos + 16 * (refcount + 1);
+ for (int refpos = pos + 16; refpos < refend; refpos += 16) {
+ references.add(new UUID(
+ segment.getLong(refpos),
+ segment.getLong(refpos + 8)));
+ }
+ }
+
return length;
}
@@ -303,30 +317,8 @@ class TarWriter {
}
synchronized void cleanup(Set<UUID> referencedIds) throws IOException {
- TarEntry[] sorted = index.values().toArray(new TarEntry[index.size()]);
- Arrays.sort(sorted, TarEntry.OFFSET_ORDER);
-
- for (int i = sorted.length - 1; i >= 0; i--) {
- TarEntry entry = sorted[i];
- UUID id = new UUID(entry.msb(), entry.lsb());
- if (referencedIds.remove(id) && isDataSegmentId(entry.lsb())) {
- // this is a referenced data segment, so follow the graph
- ByteBuffer segment = ByteBuffer.allocate(
- Math.min(entry.size(), 16 * 256));
- access.seek(entry.offset());
- access.readFully(segment.array());
- int pos = segment.position();
- int refcount = segment.get(pos + REF_COUNT_OFFSET) & 0xff;
- int refend = pos + 16 * (refcount + 1);
- for (int refpos = pos + 16; refpos < refend; refpos += 16) {
- referencedIds.add(new UUID(
- segment.getLong(refpos),
- segment.getLong(refpos + 8)));
- }
- }
- }
-
- access.seek(access.length());
+ referencedIds.removeAll(index.keySet());
+ referencedIds.addAll(references);
}
//------------------------------------------------------------< Object >--
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java?rev=1584258&r1=1584257&r2=1584258&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java Thu Apr 3 04:47:05 2014
@@ -27,6 +27,8 @@ import java.util.Properties;
import java.util.Queue;
import java.util.Set;
import java.util.UUID;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import javax.jcr.Repository;
@@ -44,16 +46,20 @@ import org.apache.jackrabbit.core.config
import org.apache.jackrabbit.oak.Oak;
import org.apache.jackrabbit.oak.api.ContentRepository;
import org.apache.jackrabbit.oak.benchmark.BenchmarkRunner;
+import org.apache.jackrabbit.oak.commons.PathUtils;
import org.apache.jackrabbit.oak.fixture.OakFixture;
import org.apache.jackrabbit.oak.http.OakServlet;
import org.apache.jackrabbit.oak.jcr.Jcr;
import org.apache.jackrabbit.oak.plugins.backup.FileStoreBackup;
import org.apache.jackrabbit.oak.plugins.document.DocumentMK;
import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore;
+import org.apache.jackrabbit.oak.plugins.segment.RecordId;
import org.apache.jackrabbit.oak.plugins.segment.Segment;
import org.apache.jackrabbit.oak.plugins.segment.SegmentId;
+import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeState;
import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore;
import org.apache.jackrabbit.oak.plugins.segment.file.FileStore;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
import org.apache.jackrabbit.oak.upgrade.RepositoryUpgrade;
import org.apache.jackrabbit.server.remoting.davex.JcrRemotingServlet;
@@ -172,6 +178,7 @@ public class Main {
dataCount++;
dataSize += segment.size();
idmap.put(id, segment.getReferencedIds());
+ System.out.println(id + " -> " + idmap.get(id));
} else if (id.isBulkSegmentId()) {
bulkCount++;
bulkSize += id.getSegment().size();
@@ -216,12 +223,35 @@ public class Main {
"%6dMB in %6d bulk segments%n",
bulkSize / (1024 * 1024), bulkCount);
} else {
+ Pattern pattern = Pattern.compile(
+ "([0-9a-f-]+)|([0-9a-f-]+:[0-9a-f]+)?(/.*)?");
for (int i = 1; i < args.length; i++) {
- UUID uuid = UUID.fromString(args[i]);
- SegmentId id = store.getTracker().getSegmentId(
- uuid.getMostSignificantBits(),
- uuid.getLeastSignificantBits());
- System.out.println(id.getSegment());
+ Matcher matcher = pattern.matcher(args[i]);
+ if (!matcher.matches()) {
+ System.err.println("Unknown argument: " + args[i]);
+ } else if (matcher.group(1) != null) {
+ UUID uuid = UUID.fromString(matcher.group(1));
+ SegmentId id = store.getTracker().getSegmentId(
+ uuid.getMostSignificantBits(),
+ uuid.getLeastSignificantBits());
+ System.out.println(id.getSegment());
+ } else {
+ RecordId id = store.getHead().getRecordId();
+ if (matcher.group(2) != null) {
+ id = RecordId.fromString(
+ store.getTracker(), matcher.group(2));
+ }
+ String path = "/";
+ if (matcher.group(3) != null) {
+ path = matcher.group(3);
+ }
+ NodeState node = new SegmentNodeState(id);
+ System.out.println("/ -> " + node);
+ for (String name : PathUtils.elements(path)) {
+ node = node.getChildNode(name);
+ System.out.println(" " + name + " -> " + node);
+ }
+ }
}
}
} finally {