You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jackrabbit.apache.org by dp...@apache.org on 2012/03/06 17:16:12 UTC
svn commit: r1297556 - in /jackrabbit/sandbox/microkernel/src:
main/java/org/apache/jackrabbit/mk/ main/java/org/apache/jackrabbit/mk/model/
main/java/org/apache/jackrabbit/mk/store/
test/java/org/apache/jackrabbit/mk/store/
Author: dpfister
Date: Tue Mar 6 16:16:11 2012
New Revision: 1297556
URL: http://svn.apache.org/viewvc?rev=1297556&view=rev
Log:
Add copying GC for revisions (WiP)
- change test to include manual GC cycle start and stop
Added:
jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/CopyingGC.java (with props)
Modified:
jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/Repository.java
jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/model/ChildNode.java
jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/DefaultRevisionStore.java
jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/RevisionStore.java
jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/store/CopyHeadRevisionTest.java
Modified: jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/Repository.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/Repository.java?rev=1297556&r1=1297555&r2=1297556&view=diff
==============================================================================
--- jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/Repository.java (original)
+++ jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/Repository.java Tue Mar 6 16:16:11 2012
@@ -16,6 +16,7 @@
*/
package org.apache.jackrabbit.mk;
+import java.io.Closeable;
import java.io.File;
import org.apache.jackrabbit.mk.model.ChildNodeEntry;
@@ -27,6 +28,7 @@ import org.apache.jackrabbit.mk.model.St
import org.apache.jackrabbit.mk.store.DefaultRevisionStore;
import org.apache.jackrabbit.mk.store.NotFoundException;
import org.apache.jackrabbit.mk.store.RevisionStore;
+import org.apache.jackrabbit.mk.util.IOUtils;
import org.apache.jackrabbit.mk.util.PathUtils;
/**
@@ -36,7 +38,7 @@ public class Repository {
private final String homeDir;
private boolean initialized;
- private final DefaultRevisionStore rs;
+ private RevisionStore rs;
public Repository(String homeDir) throws Exception {
File home = new File(homeDir == null ? "." : homeDir, ".mk");
@@ -44,8 +46,6 @@ public class Repository {
home.mkdirs();
}
this.homeDir = home.getCanonicalPath();
-
- rs = new DefaultRevisionStore();
}
/**
@@ -53,7 +53,7 @@ public class Repository {
*
* @param rs revision store, already initialized
*/
- public Repository(DefaultRevisionStore rs) {
+ public Repository(RevisionStore rs) {
this.homeDir = null;
this.rs = rs;
@@ -64,7 +64,9 @@ public class Repository {
if (initialized) {
return;
}
+ DefaultRevisionStore rs = new DefaultRevisionStore();
rs.initialize(new File(homeDir));
+ this.rs = rs;
initialized = true;
}
@@ -73,9 +75,9 @@ public class Repository {
if (!initialized) {
return;
}
-
- rs.close();
-
+ if (rs instanceof Closeable) {
+ IOUtils.closeQuietly((Closeable) rs);
+ }
initialized = false;
}
Modified: jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/model/ChildNode.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/model/ChildNode.java?rev=1297556&r1=1297555&r2=1297556&view=diff
==============================================================================
--- jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/model/ChildNode.java (original)
+++ jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/model/ChildNode.java Tue Mar 6 16:16:11 2012
@@ -22,9 +22,9 @@ package org.apache.jackrabbit.mk.model;
public class ChildNode {
private final String name;
- private final Node node;
+ private final StoredNode node;
- public ChildNode(String name, Node node) {
+ public ChildNode(String name, StoredNode node) {
this.name = name;
this.node = node;
}
@@ -33,7 +33,7 @@ public class ChildNode {
return name;
}
- public Node getNode() {
+ public StoredNode getNode() {
return node;
}
}
Added: jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/CopyingGC.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/CopyingGC.java?rev=1297556&view=auto
==============================================================================
--- jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/CopyingGC.java (added)
+++ jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/CopyingGC.java Tue Mar 6 16:16:11 2012
@@ -0,0 +1,256 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.mk.store;
+
+import java.io.Closeable;
+import java.io.InputStream;
+import java.util.Iterator;
+
+import org.apache.jackrabbit.mk.model.ChildNode;
+import org.apache.jackrabbit.mk.model.ChildNodeEntriesMap;
+import org.apache.jackrabbit.mk.model.MutableCommit;
+import org.apache.jackrabbit.mk.model.MutableNode;
+import org.apache.jackrabbit.mk.model.StoredCommit;
+import org.apache.jackrabbit.mk.model.StoredNode;
+import org.apache.jackrabbit.mk.util.IOUtils;
+
+/**
+ * Revision garbage collector that copies reachable revisions from a "from" revision
+ * store to a "to" revision store.
+ */
+public class CopyingGC implements RevisionStore, Closeable {
+
+ /**
+ * From store.
+ */
+ private RevisionStore rsFrom;
+
+ /**
+ * To store.
+ */
+ private RevisionStore rsTo;
+
+ /**
+ * Flag indicating whether a GC cycle is running.
+ */
+ private volatile boolean running;
+
+ /**
+ * Create a new instance of this class.
+ *
+ * @param rsFrom from store
+ * @param rsTo to store
+ */
+ public CopyingGC(RevisionStore rsFrom, RevisionStore rsTo) {
+ this.rsFrom = rsFrom;
+ this.rsTo = rsTo;
+ }
+
+ /**
+ * Start GC cycle.
+ *
+ * @throws Exception if an error occurs
+ */
+ public void start() throws Exception {
+ running = true;
+
+ copy(rsFrom.getHeadCommit());
+ }
+
+ /**
+ * Stop GC cycle.
+ */
+ public void stop() {
+ running = false;
+
+ // TODO: swap rsFrom/rsTo and re-initialize
+ rsFrom = rsTo;
+ rsTo = null;
+ }
+
+ public void close() {
+ if (rsFrom instanceof Closeable) {
+ IOUtils.closeQuietly((Closeable) rsFrom);
+ }
+ if (rsTo instanceof Closeable) {
+ IOUtils.closeQuietly((Closeable) rsTo);
+ }
+ }
+
+ /**
+ * Copy a commit and all the nodes belonging to it, starting at the root node.
+ *
+ * @param from source provider
+ * @param to target store
+ * @throws Exception if an error occurs
+ */
+ public void copy(StoredCommit commit) throws Exception {
+ StoredNode nodeFrom = rsFrom.getNode(commit.getRootNodeId());
+ copy(nodeFrom);
+
+ MutableCommit commitTo = new MutableCommit(commit);
+ commitTo.setParentId(rsTo.getHeadCommitId());
+
+ String revId = rsTo.putCommit(commitTo);
+ rsTo.setHeadCommitId(revId);
+ }
+
+ /**
+ * Copy a node and all its descendants into a target store
+ * @param node source node
+ * @param store target store
+ * @throws Exception if an error occurs
+ */
+ private void copy(StoredNode node) throws Exception {
+ try {
+ rsTo.getNode(node.getId());
+ return;
+ } catch (NotFoundException e) {
+ // ignore, better add a has() method
+ }
+ rsTo.putNode(new MutableNode(node, rsTo));
+
+ Iterator<ChildNode> iter = node.getChildNodes(0, -1);
+ while (iter.hasNext()) {
+ ChildNode c = iter.next();
+ copy(c.getNode());
+ }
+ }
+
+ // ---------------------------------------------------------- RevisionStore
+
+ public StoredNode getNode(String id) throws NotFoundException, Exception {
+ if (running) {
+ try {
+ return rsTo.getNode(id);
+ } catch (NotFoundException e) {
+ // ignore, better add a has() method
+ }
+ }
+ return rsFrom.getNode(id);
+ }
+
+ public StoredCommit getCommit(String id) throws NotFoundException,
+ Exception {
+
+ if (running) {
+ try {
+ return rsTo.getCommit(id);
+ } catch (NotFoundException e) {
+ // ignore, better add a has() method
+ }
+ }
+ StoredCommit commit = rsFrom.getCommit(id);
+ if (!running) {
+ return commit;
+ }
+ // synchronously copy the commit and the nodes it references to the target store
+ copy(commit);
+ return rsTo.getCommit(id);
+ }
+
+ public ChildNodeEntriesMap getCNEMap(String id) throws NotFoundException,
+ Exception {
+
+ if (running) {
+ try {
+ return rsTo.getCNEMap(id);
+ } catch (NotFoundException e) {
+ // ignore, better add a has() method
+ }
+ }
+ return rsFrom.getCNEMap(id);
+ }
+
+ public StoredNode getRootNode(String commitId) throws NotFoundException,
+ Exception {
+
+ if (running) {
+ try {
+ return rsTo.getRootNode(commitId);
+ } catch (NotFoundException e) {
+ // ignore, better add a has() method
+ }
+ }
+ return rsFrom.getRootNode(commitId);
+ }
+
+ public StoredCommit getHeadCommit() throws Exception {
+ return running ? rsTo.getHeadCommit() : rsFrom.getHeadCommit();
+ }
+
+ public String getHeadCommitId() throws Exception {
+ return running ? rsTo.getHeadCommitId() : rsFrom.getHeadCommitId();
+ }
+
+ public String putNode(MutableNode node) throws Exception {
+ return running ? rsTo.putNode(node) : rsFrom.putNode(node);
+ }
+
+ public String putCommit(MutableCommit commit) throws Exception {
+ return running ? rsTo.putCommit(commit) : rsFrom.putCommit(commit);
+ }
+
+ public String putCNEMap(ChildNodeEntriesMap map) throws Exception {
+ return running ? rsTo.putCNEMap(map) : rsFrom.putCNEMap(map);
+ }
+
+ // TODO: potentially dangerous, if lock & unlock interfere with GC start
+ public void lockHead() {
+ if (running) {
+ rsTo.lockHead();
+ } else {
+ rsFrom.lockHead();
+ }
+ }
+
+ public void setHeadCommitId(String commitId) throws Exception {
+ if (running) {
+ rsTo.setHeadCommitId(commitId);
+ } else {
+ rsFrom.setHeadCommitId(commitId);
+ }
+ }
+
+ // TODO: potentially dangerous, if lock & unlock interfere with GC start
+ public void unlockHead() {
+ if (running) {
+ rsTo.unlockHead();
+ } else {
+ rsFrom.unlockHead();
+ }
+ }
+
+ public int getBlob(String blobId, long pos, byte[] buff, int off, int length)
+ throws NotFoundException, Exception {
+
+ // Assuming that from and to store use the same BlobStore instance
+ return rsTo.getBlob(blobId, pos, buff, off, length);
+ }
+
+ public long getBlobLength(String blobId) throws NotFoundException,
+ Exception {
+
+ // Assuming that from and to store use the same BlobStore instance
+ return rsTo.getBlobLength(blobId);
+ }
+
+ public String putBlob(InputStream in) throws Exception {
+ // Assuming that from and to store use the same BlobStore instance
+ return rsTo.putBlob(in);
+ }
+}
Propchange: jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/CopyingGC.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/CopyingGC.java
------------------------------------------------------------------------------
svn:keywords = Author Date Id Revision Rev Url
Modified: jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/DefaultRevisionStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/DefaultRevisionStore.java?rev=1297556&r1=1297555&r2=1297556&view=diff
==============================================================================
--- jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/DefaultRevisionStore.java (original)
+++ jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/DefaultRevisionStore.java Tue Mar 6 16:16:11 2012
@@ -20,7 +20,6 @@ import org.apache.jackrabbit.mk.blobs.Bl
import org.apache.jackrabbit.mk.blobs.FileBlobStore;
import org.apache.jackrabbit.mk.model.ChildNodeEntriesMap;
import org.apache.jackrabbit.mk.model.MutableCommit;
-import org.apache.jackrabbit.mk.model.Node;
import org.apache.jackrabbit.mk.model.MutableNode;
import org.apache.jackrabbit.mk.model.StoredCommit;
import org.apache.jackrabbit.mk.model.StoredNode;
@@ -29,6 +28,7 @@ import org.apache.jackrabbit.mk.store.pm
import org.apache.jackrabbit.mk.util.SimpleLRUCache;
import org.apache.jackrabbit.mk.util.StringUtils;
+import java.io.Closeable;
import java.io.File;
import java.io.InputStream;
import java.util.Collections;
@@ -39,7 +39,7 @@ import java.util.concurrent.locks.Reentr
* Default revision store implementation, passing calls to a <code>PersistenceManager</code>
* and a <code>BlobStore</code>, respectively and providing caching.
*/
-public class DefaultRevisionStore implements RevisionStore {
+public class DefaultRevisionStore implements RevisionStore, Closeable {
public static final String CACHE_SIZE = "mk.cacheSize";
public static final int DEFAULT_CACHE_SIZE = 10000;
@@ -121,13 +121,13 @@ public class DefaultRevisionStore implem
}
/**
- * Convert a long value into a fixed-size byte array of size 16.
+ * Convert a long value into a fixed-size byte array of size 8.
*
* @param value value
* @return byte array
*/
- static byte[] longToBytes(long value) {
- byte[] result = new byte[16];
+ private static byte[] longToBytes(long value) {
+ byte[] result = new byte[8];
for (int i = result.length - 1; i >= 0 && value != 0; i--) {
result[i] = (byte) (value & 0xff);
@@ -135,10 +135,29 @@ public class DefaultRevisionStore implem
}
return result;
}
+
+ /**
+ * Convert a fixed-size byte array of size 8 into a long.
+ *
+ * @param value byte array
+ * @return long
+ */
+ private static long bytesToLong(byte[] value) {
+ long result = 0;
+
+ if (value.length != 8) {
+ throw new IllegalArgumentException("Value must be a byte array of size 8");
+ }
+ for (int i = 0; i < value.length; i++) {
+ result |= (value[i] & 0xff);
+ result <<= 8;
+ }
+ return result;
+ }
//--------------------------------------------------------< RevisionStore >
- public String putNode(Node node) throws Exception {
+ public String putNode(MutableNode node) throws Exception {
verifyInitialized();
PersistHook callback = null;
@@ -195,6 +214,7 @@ public class DefaultRevisionStore implem
id = StringUtils.convertBytesToHex(rawId);
} else {
rawId = StringUtils.convertHexToBytes(id);
+ headCounter = bytesToLong(rawId);
}
pm.writeCommit(rawId, commit);
Modified: jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/RevisionStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/RevisionStore.java?rev=1297556&r1=1297555&r2=1297556&view=diff
==============================================================================
--- jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/RevisionStore.java (original)
+++ jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/RevisionStore.java Tue Mar 6 16:16:11 2012
@@ -18,7 +18,7 @@ package org.apache.jackrabbit.mk.store;
import org.apache.jackrabbit.mk.model.ChildNodeEntriesMap;
import org.apache.jackrabbit.mk.model.MutableCommit;
-import org.apache.jackrabbit.mk.model.Node;
+import org.apache.jackrabbit.mk.model.MutableNode;
import java.io.InputStream;
@@ -27,7 +27,7 @@ import java.io.InputStream;
*/
public interface RevisionStore extends RevisionProvider {
- String /*id*/ putNode(Node node) throws Exception;
+ String /*id*/ putNode(MutableNode node) throws Exception;
String /*id*/ putCommit(MutableCommit commit) throws Exception;
String /*id*/ putCNEMap(ChildNodeEntriesMap map) throws Exception;
void setHeadCommitId(String commitId) throws Exception;
Modified: jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/store/CopyHeadRevisionTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/store/CopyHeadRevisionTest.java?rev=1297556&r1=1297555&r2=1297556&view=diff
==============================================================================
--- jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/store/CopyHeadRevisionTest.java (original)
+++ jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/store/CopyHeadRevisionTest.java Tue Mar 6 16:16:11 2012
@@ -16,21 +16,16 @@
*/
package org.apache.jackrabbit.mk.store;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
import java.io.File;
-import java.util.Iterator;
import org.apache.jackrabbit.mk.MicroKernelImpl;
import org.apache.jackrabbit.mk.Repository;
+import org.apache.jackrabbit.mk.api.MicroKernelException;
import org.apache.jackrabbit.mk.fs.FileUtils;
-import org.apache.jackrabbit.mk.json.fast.Jsop;
-import org.apache.jackrabbit.mk.json.fast.JsopArray;
-import org.apache.jackrabbit.mk.model.ChildNode;
-import org.apache.jackrabbit.mk.model.MutableCommit;
-import org.apache.jackrabbit.mk.model.MutableNode;
-import org.apache.jackrabbit.mk.model.Node;
-import org.apache.jackrabbit.mk.model.StoredCommit;
import org.junit.After;
-import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -57,72 +52,37 @@ public class CopyHeadRevisionTest {
@Test
public void testCopyHeadRevisionToNewStore() throws Exception {
+ String[] revs = new String[3];
+
DefaultRevisionStore rsFrom = new DefaultRevisionStore();
rsFrom.initialize(new File("target/mk1"));
-
- MicroKernelImpl mkFrom = new MicroKernelImpl(new Repository(rsFrom));
- mkFrom.commit("/", "+\"a\" : { \"c\":{}, \"d\":{} }", mkFrom.getHeadRevision(), null);
- mkFrom.commit("/", "+\"b\" : {}", mkFrom.getHeadRevision(), null);
- mkFrom.commit("/b", "+\"e\" : {}", mkFrom.getHeadRevision(), null);
DefaultRevisionStore rsTo = new DefaultRevisionStore();
rsTo.initialize(new File("target/mk2"));
- copyHeadRevision(rsFrom, rsTo);
+ CopyingGC gc = new CopyingGC(rsFrom, rsTo);
+
+ MicroKernelImpl mk = new MicroKernelImpl(new Repository(gc));
+ revs[0] = mk.commit("/", "+\"a\" : { \"c\":{}, \"d\":{} }", mk.getHeadRevision(), null);
+ revs[1] = mk.commit("/", "+\"b\" : {}", mk.getHeadRevision(), null);
- MicroKernelImpl mkTo = new MicroKernelImpl(new Repository(rsTo));
+ // Simulate a GC cycle start
+ gc.start();
- // Assert both old and new MK have same head revision
- Assert.assertEquals(mkFrom.getHeadRevision(), mkTo.getHeadRevision());
-
- // Assert both old and new MK have same contents
- Assert.assertEquals(
- mkFrom.getNodes("/", mkFrom.getHeadRevision(), 2, 0, -1),
- mkTo.getNodes("/", mkTo.getHeadRevision(), 2, 0, -1));
-
- // Assert new MK has only 2 revisions (initial and head)
- JsopArray revs = (JsopArray) Jsop.parse(mkTo.getRevisions(0, Integer.MAX_VALUE));
- Assert.assertEquals(2, revs.size());
- }
-
- /**
- * Copy the head revision (commit and nodes) from a source provider to a
- * target store.
- *
- * @param from source provider
- * @param to target store
- * @throws Exception if an error occurs
- */
- private void copyHeadRevision(RevisionProvider from, RevisionStore to)
- throws Exception {
+ revs[2] = mk.commit("/b", "+\"e\" : {}", mk.getHeadRevision(), null);
- StoredCommit commitFrom = from.getHeadCommit();
+ // Simulate a GC cycle stop
+ gc.stop();
- Node nodeFrom = from.getNode(commitFrom.getRootNodeId());
- copy(nodeFrom, to);
-
- MutableCommit commitTo = new MutableCommit(commitFrom);
- commitTo.setParentId(to.getHeadCommitId());
-
- String revId = to.putCommit(commitTo);
- to.setHeadCommitId(revId);
- }
-
- /**
- * Copy a node and all its descendants into a target store
- * @param node source node
- * @param store target store
- * @throws Exception if an error occurs
- */
- private void copy(Node node, RevisionStore store)
- throws Exception {
-
- store.putNode(new MutableNode(node, store));
+ // Assert head revision is contained after GC
+ assertEquals(mk.getHeadRevision(), revs[2]);
- Iterator<ChildNode> iter = node.getChildNodes(0, -1);
- while (iter.hasNext()) {
- ChildNode c = iter.next();
- copy(c.getNode(), store);
+ // Assert unused revision was GCed
+ try {
+ mk.getNodes("/", revs[0]);
+ fail("Revision should have been GCed: "+ revs[0]);
+ } catch (MicroKernelException e) {
+ // ignore
}
}
}