You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ju...@apache.org on 2014/05/29 20:06:27 UTC
svn commit: r1598352 - in /jackrabbit/oak/trunk:
oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/
oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/
oak-core/src/main/java/org/apache/jackrabbit/oak/spi/state/ oak-ru...
Author: jukka
Date: Thu May 29 18:06:27 2014
New Revision: 1598352
URL: http://svn.apache.org/r1598352
Log:
OAK-1804: TarMK compaction
Move the compaction code to a separate Compactor class.
Avoid duplicating shared subtrees like those in checkpoints.
Drop the level option again as the overhead of running a full
compaction vs. a level-limited one isn't too high.
Added:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Compactor.java (with props)
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/state/ApplyDiff.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java
Added: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Compactor.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Compactor.java?rev=1598352&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Compactor.java (added)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Compactor.java Thu May 29 18:06:27 2014
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.segment;
+
+import static com.google.common.collect.Maps.newHashMap;
+import static org.apache.jackrabbit.oak.api.Type.BINARIES;
+import static org.apache.jackrabbit.oak.api.Type.BINARY;
+import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.jackrabbit.oak.api.Blob;
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.plugins.memory.BinaryPropertyState;
+import org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState;
+import org.apache.jackrabbit.oak.plugins.memory.MultiBinaryPropertyState;
+import org.apache.jackrabbit.oak.plugins.memory.PropertyStates;
+import org.apache.jackrabbit.oak.spi.state.ApplyDiff;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.esotericsoftware.minlog.Log;
+
+/**
+ * Tool for compacting segments.
+ */
+public class Compactor {
+
+ /** Logger instance */
+ private static final Logger log = LoggerFactory.getLogger(Compactor.class);
+
+ private final SegmentStore store;
+
+ private final SegmentWriter writer;
+
+ /**
+ * Map from the identifiers of old records to the identifiers of their
+ * compacted copies. Used to prevent the compaction code from duplicating
+ * things like checkpoints that share most of their content with other
+ * subtrees.
+ */
+ private final Map<RecordId, RecordId> compacted = newHashMap();
+
+ public Compactor(SegmentStore store) {
+ this.store = store;
+ this.writer = store.getTracker().getWriter();
+ }
+
+ public void compact() throws IOException {
+ log.debug("TarMK compaction");
+
+ SegmentNodeBuilder builder = writer.writeNode(EMPTY_NODE).builder();
+ SegmentNodeState before = store.getHead();
+ EmptyNodeState.compareAgainstEmptyState(
+ before, new CompactDiff(builder));
+ System.out.println(compacted.size() + " nodes compacted");
+
+ SegmentNodeState after = builder.getNodeState();
+ while (!store.setHead(before, after)) {
+ // Some other concurrent changes have been made.
+ // Rebase (and compact) those changes on top of the
+ // compacted state before retrying to set the head.
+ SegmentNodeState head = store.getHead();
+ head.compareAgainstBaseState(before, new CompactDiff(builder));
+ System.out.println(compacted.size() + " nodes compacted");
+ before = head;
+ after = builder.getNodeState();
+ }
+ }
+
+ private class CompactDiff extends ApplyDiff {
+
+ CompactDiff(NodeBuilder builder) {
+ super(builder);
+ }
+
+ @Override
+ public boolean propertyAdded(PropertyState after) {
+ return super.propertyAdded(compact(after));
+ }
+
+
+ @Override
+ public boolean propertyChanged(
+ PropertyState before, PropertyState after) {
+ return super.propertyChanged(before, compact(after));
+ }
+
+ @Override
+ public boolean childNodeAdded(String name, NodeState after) {
+ RecordId id = null;
+ if (after instanceof SegmentNodeState) {
+ id = ((SegmentNodeState) after).getRecordId();
+ RecordId compactedId = compacted.get(id);
+ if (compactedId != null) {
+ builder.setChildNode(name, new SegmentNodeState(compactedId));
+ return true;
+ }
+ }
+
+ NodeBuilder child = builder.setChildNode(name);
+ boolean success = EmptyNodeState.compareAgainstEmptyState(
+ after, new CompactDiff(child));
+
+ if (success && id != null) {
+ RecordId compactedId =
+ writer.writeNode(child.getNodeState()).getRecordId();
+ compacted.put(id, compactedId);
+ }
+
+ return success;
+ }
+
+ @Override
+ public boolean childNodeChanged(
+ String name, NodeState before, NodeState after) {
+ RecordId id = null;
+ if (after instanceof SegmentNodeState) {
+ id = ((SegmentNodeState) after).getRecordId();
+ RecordId compactedId = compacted.get(id);
+ if (compactedId != null) {
+ builder.setChildNode(name, new SegmentNodeState(compactedId));
+ return true;
+ }
+ }
+
+ NodeBuilder child = builder.getChildNode(name);
+ boolean success = after.compareAgainstBaseState(
+ before, new CompactDiff(child));
+
+ if (success && id != null) {
+ RecordId compactedId =
+ writer.writeNode(child.getNodeState()).getRecordId();
+ compacted.put(id, compactedId);
+ }
+
+ return success;
+ }
+
+ }
+
+ private PropertyState compact(PropertyState property) {
+ String name = property.getName();
+ Type<?> type = property.getType();
+ if (type == BINARY) {
+ Blob blob = compact(property.getValue(Type.BINARY));
+ return BinaryPropertyState.binaryProperty(name, blob);
+ } else if (type == BINARIES) {
+ List<Blob> blobs = new ArrayList<Blob>();
+ for (Blob blob : property.getValue(BINARIES)) {
+ blobs.add(compact(blob));
+ }
+ return MultiBinaryPropertyState.binaryPropertyFromBlob(name, blobs);
+ } else {
+ Object value = property.getValue(type);
+ return PropertyStates.createProperty(name, value, type);
+ }
+ }
+
+ private Blob compact(Blob blob) {
+ if (blob instanceof SegmentBlob) {
+ try {
+ return ((SegmentBlob) blob).clone(writer);
+ } catch (IOException e) {
+ Log.warn("Failed to clone a binary value", e);
+ }
+ }
+ return blob;
+ }
+
+}
Propchange: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Compactor.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java?rev=1598352&r1=1598351&r2=1598352&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java Thu May 29 18:06:27 2014
@@ -35,7 +35,6 @@ import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.FileLock;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
@@ -51,24 +50,14 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.jackrabbit.oak.api.Blob;
-import org.apache.jackrabbit.oak.api.PropertyState;
-import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.plugins.blob.BlobStoreBlob;
-import org.apache.jackrabbit.oak.plugins.memory.BinaryPropertyState;
-import org.apache.jackrabbit.oak.plugins.memory.MultiBinaryPropertyState;
-import org.apache.jackrabbit.oak.plugins.memory.PropertyStates;
import org.apache.jackrabbit.oak.plugins.segment.RecordId;
import org.apache.jackrabbit.oak.plugins.segment.Segment;
-import org.apache.jackrabbit.oak.plugins.segment.SegmentBlob;
import org.apache.jackrabbit.oak.plugins.segment.SegmentId;
-import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeBuilder;
import org.apache.jackrabbit.oak.plugins.segment.SegmentTracker;
import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeState;
import org.apache.jackrabbit.oak.plugins.segment.SegmentStore;
-import org.apache.jackrabbit.oak.plugins.segment.SegmentWriter;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
-import org.apache.jackrabbit.oak.spi.state.ApplyDiff;
-import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.slf4j.Logger;
@@ -339,67 +328,6 @@ public class FileStore implements Segmen
return dataFiles;
}
- public void compact(int levels) throws IOException {
- log.debug("TarMK compaction");
-
- SegmentWriter segmentWriter = tracker.getWriter();
- segmentWriter.dropCache();
-
- SegmentNodeBuilder builder =
- segmentWriter.writeNode(EMPTY_NODE).builder();
- SegmentNodeState before = getHead();
- compact(before, "/", levels, builder);
-
- SegmentNodeState after = builder.getNodeState();
- while (!setHead(before, after)) {
- // Some other concurrent changes have been made.
- // Rebase those changes on top of the compacted
- // state before retrying setting the head state.
- SegmentNodeState head = getHead();
- head.compareAgainstBaseState(before, new ApplyDiff(builder));
- before = head;
- after = builder.getNodeState();
- }
- }
-
- private void compact(SegmentNodeState state, String name, int levels,
- NodeBuilder dest) throws IOException {
- for (PropertyState ps : state.getProperties()) {
- if (Type.BINARY.tag() != ps.getType().tag()) {
- ps = PropertyStates.createProperty(ps.getName(),
- ps.getValue(ps.getType()), ps.getType());
- } else {
- List<Blob> newBlobList = new ArrayList<Blob>();
- for (Blob b : ps.getValue(Type.BINARIES)) {
- if (b instanceof SegmentBlob) {
- SegmentBlob sb = (SegmentBlob) b;
- b = sb.clone(tracker.getWriter());
- }
- newBlobList.add(b);
- }
- if (ps.isArray()) {
- ps = MultiBinaryPropertyState.binaryPropertyFromBlob(
- ps.getName(), newBlobList);
- } else {
- ps = BinaryPropertyState.binaryProperty(ps.getName(),
- newBlobList.get(0));
- }
- }
- dest.setProperty(ps);
- }
-
- for (ChildNodeEntry entry : state.getChildNodeEntries()) {
- SegmentNodeState child = (SegmentNodeState) entry.getNodeState();
- String n = entry.getName();
- if (levels > 0) {
- compact(child, name + entry.getName() + "/", levels - 1,
- dest.child(entry.getName()));
- } else {
- dest.setChildNode(n, child);
- }
- }
- }
-
public void flush() throws IOException {
synchronized (persistedHead) {
RecordId before = persistedHead.get();
@@ -672,4 +600,5 @@ public class FileStore implements Segmen
}
return index;
}
-}
\ No newline at end of file
+
+}
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/state/ApplyDiff.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/state/ApplyDiff.java?rev=1598352&r1=1598351&r2=1598352&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/state/ApplyDiff.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/state/ApplyDiff.java Thu May 29 18:06:27 2014
@@ -32,7 +32,7 @@ import org.apache.jackrabbit.oak.api.Pro
* NodeState base = ...;
* NodeState target = ...;
* NodeBuilder builder = base.builder();
- * target.compareAgainstBaseState(base, new ReapplyDiff(builder));
+ * target.compareAgainstBaseState(base, new ApplyDiff(builder));
* assertEquals(target, builder.getNodeState());
* </pre>
* <p>
@@ -47,7 +47,7 @@ import org.apache.jackrabbit.oak.api.Pro
*/
public class ApplyDiff implements NodeStateDiff {
- private final NodeBuilder builder;
+ protected final NodeBuilder builder;
public ApplyDiff(NodeBuilder builder) {
this.builder = builder;
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java?rev=1598352&r1=1598351&r2=1598352&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java Thu May 29 18:06:27 2014
@@ -38,9 +38,11 @@ import com.google.common.collect.Maps;
import com.google.common.collect.Queues;
import com.mongodb.MongoClient;
import com.mongodb.MongoClientURI;
+
import joptsimple.OptionParser;
import joptsimple.OptionSet;
import joptsimple.OptionSpec;
+
import org.apache.jackrabbit.core.RepositoryContext;
import org.apache.jackrabbit.core.config.RepositoryConfig;
import org.apache.jackrabbit.oak.Oak;
@@ -55,6 +57,7 @@ import org.apache.jackrabbit.oak.jcr.Jcr
import org.apache.jackrabbit.oak.plugins.backup.FileStoreBackup;
import org.apache.jackrabbit.oak.plugins.document.DocumentMK;
import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore;
+import org.apache.jackrabbit.oak.plugins.segment.Compactor;
import org.apache.jackrabbit.oak.plugins.segment.RecordId;
import org.apache.jackrabbit.oak.plugins.segment.Segment;
import org.apache.jackrabbit.oak.plugins.segment.SegmentId;
@@ -178,15 +181,10 @@ public class Main {
}
private static void compact(String[] args) throws IOException {
- if (args.length < 1 || args.length > 2) {
- System.err.println("usage: compact <path> [levels]");
+ if (args.length != 1) {
+ System.err.println("usage: compact <path>");
System.exit(1);
} else {
- int levels = 5;
- if (args.length == 2) {
- levels = Integer.parseInt(args[1]);
- }
-
File directory = new File(args[0]);
System.out.println("Compacting " + directory);
System.out.println(" before " + Arrays.toString(directory.list()));
@@ -194,7 +192,7 @@ public class Main {
System.out.println(" -> compacting");
FileStore store = new FileStore(directory, 256, false);
try {
- store.compact(levels);
+ new Compactor(store).compact();
} finally {
store.close();
}
@@ -208,7 +206,6 @@ public class Main {
store.close();
}
-
System.out.println(" after " + Arrays.toString(directory.list()));
}
}