You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by th...@apache.org on 2013/12/04 13:23:03 UTC
svn commit: r1547786 - in /jackrabbit/oak/trunk/oak-core/src:
main/java/org/apache/jackrabbit/oak/plugins/mongomk/
test/java/org/apache/jackrabbit/oak/plugins/mongomk/
Author: thomasm
Date: Wed Dec 4 12:23:03 2013
New Revision: 1547786
URL: http://svn.apache.org/r1547786
Log:
OAK-377 Data store garbage collection (WIP)
Added:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/BlobReferenceIterator.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/mongomk/BlobReferenceTest.java
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/MongoNodeBuilder.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/MongoNodeState.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/MongoNodeStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/MongoRootBuilder.java
Added: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/BlobReferenceIterator.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/BlobReferenceIterator.java?rev=1547786&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/BlobReferenceIterator.java (added)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/BlobReferenceIterator.java Wed Dec 4 12:23:03 2013
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.mongomk;
+
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+
+import org.apache.jackrabbit.mk.json.JsopReader;
+import org.apache.jackrabbit.mk.json.JsopTokenizer;
+import org.apache.jackrabbit.oak.api.Blob;
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.plugins.mongomk.util.Utils;
+
+/**
+ * An iterator over all referenced binaries.
+ * <p>
+ * Only top-level referenced are returned (indirection, if any, is not resolved).
+ * The items are returned in no particular order.
+ * An item might be returned multiple times.
+ */
+public class BlobReferenceIterator implements Iterator<Blob> {
+
+ private static final int BATCH_SIZE = 1000;
+ private final MongoNodeStore nodeStore;
+ private final DocumentStore docStore;
+ private HashSet<Blob> batch = new HashSet<Blob>();
+ private Iterator<Blob> batchIterator;
+ private boolean done;
+ private String fromKey = "0000000";
+
+ public BlobReferenceIterator(MongoNodeStore nodeStore) {
+ this.nodeStore = nodeStore;
+ this.docStore = nodeStore.getDocumentStore();
+ batchIterator = batch.iterator();
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (!batchIterator.hasNext()) {
+ loadBatch();
+ }
+ return batchIterator.hasNext() || !done;
+ }
+
+ @Override
+ public Blob next() {
+ // this will load the next batch if required
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ return batchIterator.next();
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ private void loadBatch() {
+ if (done) {
+ return;
+ }
+ batch.clear();
+ // read until at least BATCH_SIZE references are available
+ while (true) {
+ boolean hasMore = loadBatchQuery();
+ if (!hasMore) {
+ done = true;
+ break;
+ }
+ if (batch.size() > BATCH_SIZE) {
+ break;
+ }
+ }
+ batchIterator = batch.iterator();
+ }
+
+ private boolean loadBatchQuery() {
+ // read about BATCH_SIZE documents
+ List<NodeDocument> list = docStore.query(Collection.NODES, fromKey, "999999", BATCH_SIZE);
+ boolean hasMore = false;
+ for (NodeDocument doc : list) {
+ if (doc.getId().equals(fromKey)) {
+ // already read
+ continue;
+ }
+ hasMore = true;
+ fromKey = doc.getId();
+ for (String key : doc.keySet()) {
+ if (!Utils.isPropertyName(key)) {
+ continue;
+ }
+ Map<Revision, String> valueMap = doc.getLocalMap(key);
+ for (String v : valueMap.values()) {
+ loadValue(v);
+ }
+ }
+ }
+ return hasMore;
+ }
+
+ private void loadValue(String v) {
+ JsopReader reader = new JsopTokenizer(v);
+ PropertyState p;
+ if (reader.matches('[')) {
+ p = MongoNodeState.readArrayProperty("x", nodeStore, reader);
+ if (p.getType() == Type.BINARIES) {
+ for (int i = 0; i < p.count(); i++) {
+ Blob b = p.getValue(Type.BINARY, i);
+ batch.add(b);
+ }
+ }
+ } else {
+ p = MongoNodeState.readProperty("x", nodeStore, reader);
+ if (p.getType() == Type.BINARY) {
+ Blob b = p.getValue(Type.BINARY);
+ batch.add(b);
+ }
+ }
+ }
+
+}
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/MongoNodeBuilder.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/MongoNodeBuilder.java?rev=1547786&r1=1547785&r2=1547786&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/MongoNodeBuilder.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/MongoNodeBuilder.java Wed Dec 4 12:23:03 2013
@@ -16,6 +16,10 @@
*/
package org.apache.jackrabbit.oak.plugins.mongomk;
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeState;
@@ -24,14 +28,22 @@ import org.apache.jackrabbit.oak.spi.sta
* A node builder implementation for MongoMK.
*/
class MongoNodeBuilder extends MemoryNodeBuilder {
+
+ /**
+ * The underlying store
+ */
+ protected final MongoNodeStore store;
+
private NodeState base;
- protected MongoNodeBuilder(MongoNodeState base) {
+ protected MongoNodeBuilder(MongoNodeStore store, MongoNodeState base) {
super(base);
+ this.store = store;
}
- private MongoNodeBuilder(MongoNodeBuilder parent, String name) {
+ private MongoNodeBuilder(MongoNodeStore store, MongoNodeBuilder parent, String name) {
super(parent, name);
+ this.store = store;
}
@Override
@@ -44,7 +56,7 @@ class MongoNodeBuilder extends MemoryNod
@Override
protected MongoNodeBuilder createChildBuilder(String name) {
- return new MongoNodeBuilder(this, name);
+ return new MongoNodeBuilder(store, this, name);
}
@Override
@@ -61,4 +73,11 @@ class MongoNodeBuilder extends MemoryNod
}
return super.moveTo(newParent, newName);
}
+
+
+ @Override
+ public Blob createBlob(InputStream stream) throws IOException {
+ return store.createBlob(stream);
+ }
+
}
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/MongoNodeState.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/MongoNodeState.java?rev=1547786&r1=1547785&r2=1547786&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/MongoNodeState.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/MongoNodeState.java Wed Dec 4 12:23:03 2013
@@ -231,6 +231,20 @@ final class MongoNodeState extends Abstr
* @return new property state
*/
PropertyState readProperty(String name, JsopReader reader) {
+ return readProperty(name, store, reader);
+ }
+
+ /**
+ * FIXME: copied from KernelNodeState.
+ *
+ * Read a {@code PropertyState} from a {@link JsopReader}.
+ *
+ * @param name the name of the property state
+ * @param store the store
+ * @param reader the reader
+ * @return new property state
+ */
+ public static PropertyState readProperty(String name, MongoNodeStore store, JsopReader reader) {
if (reader.matches(JsopReader.NUMBER)) {
String number = reader.getToken();
try {
@@ -268,12 +282,27 @@ final class MongoNodeState extends Abstr
/**
* FIXME: copied from KernelNodeState.
*
- * Read a multi valued {@code PropertyState} from a {@link JsopReader}
- * @param name The name of the property state
- * @param reader The reader
+ * Read a multi valued {@code PropertyState} from a {@link JsopReader}.
+ *
+ * @param name the name of the property state
+ * @param reader the reader
* @return new property state
*/
PropertyState readArrayProperty(String name, JsopReader reader) {
+ return readArrayProperty(name, store, reader);
+ }
+
+ /**
+ * FIXME: copied from KernelNodeState.
+ *
+ * Read a multi valued {@code PropertyState} from a {@link JsopReader}.
+ *
+ * @param name the name of the property state
+ * @param store the store
+ * @param reader the reader
+ * @return new property state
+ */
+ public static PropertyState readArrayProperty(String name, MongoNodeStore store, JsopReader reader) {
int type = PropertyType.STRING;
List<Object> values = Lists.newArrayList();
while (!reader.matches(']')) {
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/MongoNodeStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/MongoNodeStore.java?rev=1547786&r1=1547785&r2=1547786&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/MongoNodeStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/MongoNodeStore.java Wed Dec 4 12:23:03 2013
@@ -1301,4 +1301,8 @@ public final class MongoNodeStore
return blobSerializer;
}
+ Iterator<Blob> getReferencedBlobsIterator() {
+ return new BlobReferenceIterator(this);
+ }
+
}
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/MongoRootBuilder.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/MongoRootBuilder.java?rev=1547786&r1=1547785&r2=1547786&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/MongoRootBuilder.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/MongoRootBuilder.java Wed Dec 4 12:23:03 2013
@@ -38,11 +38,6 @@ class MongoRootBuilder extends MongoNode
private static final int UPDATE_LIMIT = Integer.getInteger("update.limit", 1000);
/**
- * The underlying store
- */
- private final MongoNodeStore store;
-
- /**
* The base state of this builder, possibly non-existent if this builder
* represents a new node that didn't yet exist in the base content tree.
* This differs from the base state of super since the latter one reflects
@@ -62,9 +57,8 @@ class MongoRootBuilder extends MongoNode
private int updates;
MongoRootBuilder(MongoNodeState base, MongoNodeStore store) {
- super(checkNotNull(base));
+ super(store, checkNotNull(base));
this.base = base;
- this.store = store;
this.branch = store.createBranch(base);
}
@@ -154,4 +148,5 @@ class MongoRootBuilder extends MongoNode
super.reset(branch.getHead());
updates = 0;
}
+
}
Added: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/mongomk/BlobReferenceTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/mongomk/BlobReferenceTest.java?rev=1547786&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/mongomk/BlobReferenceTest.java (added)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/mongomk/BlobReferenceTest.java Wed Dec 4 12:23:03 2013
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.mongomk;
+
+import static junit.framework.Assert.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Random;
+
+import org.apache.jackrabbit.oak.api.Blob;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.junit.Test;
+
+/**
+ * Test the BlobReferenceIterator.
+ */
+public class BlobReferenceTest {
+
+ @Test
+ public void test() throws Exception {
+ MongoNodeStore s = new MongoMK.Builder().getNodeStore();
+ NodeBuilder a = s.getRoot().builder();
+ HashSet<String> set = new HashSet<String>();
+ for (int i = 0; i < 100; i++) {
+ Blob b = a.createBlob(randomStream(i, 10));
+ set.add(b.toString());
+ a.child("c" + i).setProperty("x", b);
+ }
+ s.merge(a, EmptyHook.INSTANCE, null);
+ Iterator<Blob> it = s.getReferencedBlobsIterator();
+ while (it.hasNext()) {
+ Blob b = it.next();
+ set.remove(b.toString());
+ }
+ assertTrue(set.isEmpty());
+ }
+
+ static InputStream randomStream(int seed, int size) {
+ Random r = new Random(seed);
+ byte[] data = new byte[size];
+ r.nextBytes(data);
+ return new ByteArrayInputStream(data);
+ }
+
+}