You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jackrabbit.apache.org by th...@apache.org on 2012/02/13 16:32:40 UTC
svn commit: r1243562 - in /jackrabbit/sandbox/microkernel/src:
main/java/org/apache/jackrabbit/mk/mem/NodeImpl.java
main/java/org/apache/jackrabbit/mk/util/Sync.java
test/java/org/apache/jackrabbit/mk/util/SyncTest.java
Author: thomasm
Date: Mon Feb 13 15:32:39 2012
New Revision: 1243562
URL: http://svn.apache.org/viewvc?rev=1243562&view=rev
Log:
Support large child node lists.
Modified:
jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/mem/NodeImpl.java
jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/util/Sync.java
jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/util/SyncTest.java
Modified: jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/mem/NodeImpl.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/mem/NodeImpl.java?rev=1243562&r1=1243561&r2=1243562&view=diff
==============================================================================
--- jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/mem/NodeImpl.java (original)
+++ jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/mem/NodeImpl.java Mon Feb 13 15:32:39 2012
@@ -90,6 +90,7 @@ public class NodeImpl implements Cache.V
private int memory;
private long descendantCount;
private int descendantInlineCount;
+ private boolean hasMoreChildNodes;
private byte[] hash;
@@ -449,17 +450,20 @@ public class NodeImpl implements Cache.V
node.setPath(path);
}
if (!t.matches('}')) {
+ long parsedChildCount = 0;
+ int parsedChildren = 0;
do {
String key = t.readString();
t.read(':');
if (t.matches('{')) {
String childPath = path == null || key == null ? null : PathUtils.concat(path, key);
node.addChildNode(key, false, null, parse(map, t, revId, childPath));
+ parsedChildren++;
} else {
String value = t.readRawValue().trim();
if (key.length() > 0 && key.charAt(0) == ':') {
if (key.equals(CHILREN_COUNT)) {
- // ignore
+ parsedChildCount = Long.parseLong(value);
} else if (key.equals(HASH)) {
value = JsopTokenizer.decodeQuoted(value);
node.hash = StringUtils.convertHexToBytes(value);
@@ -474,6 +478,9 @@ public class NodeImpl implements Cache.V
}
} while (t.matches(','));
t.read('}');
+ if (parsedChildCount > 0 && parsedChildren < parsedChildCount) {
+ node.hasMoreChildNodes = true;
+ }
}
return node;
}
@@ -716,4 +723,8 @@ public class NodeImpl implements Cache.V
return false;
}
+ public boolean hasMoreChildNodes() {
+ return hasMoreChildNodes;
+ }
+
}
Modified: jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/util/Sync.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/util/Sync.java?rev=1243562&r1=1243561&r2=1243562&view=diff
==============================================================================
--- jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/util/Sync.java (original)
+++ jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/util/Sync.java Mon Feb 13 15:32:39 2012
@@ -16,6 +16,7 @@
*/
package org.apache.jackrabbit.mk.util;
+import java.util.Iterator;
import org.apache.jackrabbit.mk.api.MicroKernel;
import org.apache.jackrabbit.mk.mem.NodeImpl;
@@ -31,6 +32,8 @@ public class Sync {
private MicroKernel sourceMk, targetMk;
private String sourceRev, targetRev;
private String sourcePath, targetPath = "/";
+ private boolean useContentHashOptimization;
+ private int childNodesPerBatch = 100;
private Handler handler;
@@ -62,6 +65,42 @@ public class Sync {
targetPath = path;
}
+ /**
+ * Whether to use the content hash optimization if available.
+ *
+ * @return true if the optimization should be used
+ */
+ public boolean getUseContentHashOptimization() {
+ return useContentHashOptimization;
+ }
+
+ /**
+ * Use the content hash optimization if available.
+ *
+ * @param useContentHashOptimization the new value
+ */
+ public void setUseContentHashOptimization(boolean useContentHashOptimization) {
+ this.useContentHashOptimization = useContentHashOptimization;
+ }
+
+ /**
+ * Get the number of child nodes to request in one call.
+ *
+ * @return the number of child nodes to request
+ */
+ public int getChildNodesPerBatch() {
+ return childNodesPerBatch;
+ }
+
+ /**
+ * Set the number of child nodes to request in one call.
+ *
+ * @param childNodesPerBatch the number of child nodes to request
+ */
+ public void setChildNodesPerBatch(int childNodesPerBatch) {
+ this.childNodesPerBatch = childNodesPerBatch;
+ }
+
public void run(Handler handler) {
this.handler = handler;
visit("");
@@ -71,12 +110,11 @@ public class Sync {
String source = PathUtils.concat(sourcePath, relPath);
String target = PathUtils.concat(targetPath, relPath);
NodeImpl s = null, t = null;
- // TODO support large child node lists
if (sourceMk.nodeExists(source, sourceRev)) {
- s = NodeImpl.parse(sourceMk.getNodes(source, sourceRev, 0, 0, Integer.MAX_VALUE));
+ s = NodeImpl.parse(sourceMk.getNodes(source, sourceRev, 0, 0, childNodesPerBatch));
}
if (targetMk != null && targetMk.nodeExists(target, targetRev)) {
- t = NodeImpl.parse(targetMk.getNodes(target, targetRev, 0, 0, Integer.MAX_VALUE));
+ t = NodeImpl.parse(targetMk.getNodes(target, targetRev, 0, 0, childNodesPerBatch));
}
if (s == null || t == null) {
if (s == t) {
@@ -100,34 +138,91 @@ public class Sync {
handler.setProperty(target, name, sourceValue);
}
}
- for (int i = 0; t != null && i < t.getPropertyCount(); i++) {
- String name = t.getProperty(i);
- // if it exists in the source, it's already updated
- if (!s.hasProperty(name)) {
- handler.setProperty(target, name, null);
+ if (t != null) {
+ for (int i = 0; i < t.getPropertyCount(); i++) {
+ String name = t.getProperty(i);
+ // if it exists in the source, it's already updated
+ if (!s.hasProperty(name)) {
+ handler.setProperty(target, name, null);
+ }
}
}
// child nodes
- for (int i = 0;; i++) {
- String name = s.getChildNodeName(i);
- if (name == null) {
- break;
- }
+ Iterator<String> it = s.hasMoreChildNodes() ?
+ getAllChildNodeNames(sourceMk, source, sourceRev, childNodesPerBatch) :
+ s.getChildNodeNames(Integer.MAX_VALUE);
+ while (it.hasNext()) {
+ String name = it.next();
visit(PathUtils.concat(relPath, name));
}
- for (int i = 0; t != null; i++) {
- String name = t.getChildNodeName(i);
- if (name == null) {
- break;
- }
- // if it exists in the source, it's already updated
- if (!s.exists(name)) {
- visit(PathUtils.concat(relPath, name));
+ if (t != null) {
+ it = t.hasMoreChildNodes() ?
+ getAllChildNodeNames(targetMk, target, targetRev, childNodesPerBatch) :
+ t.getChildNodeNames(Integer.MAX_VALUE);
+ while (it.hasNext()) {
+ String name = it.next();
+ if (s.exists(name)) {
+ // if it exists in the source, it's already updated
+ } else if (s.hasMoreChildNodes() && sourceMk.nodeExists(PathUtils.concat(source, name), sourceRev)) {
+ // if it exists in the source, it's already updated
+ // (in this case, there are many child nodes)
+ } else {
+ visit(PathUtils.concat(relPath, name));
+ }
}
}
}
/**
+ * Get a child node name iterator that batches node names. This work
+ * efficiently for small and big child node lists.
+ *
+ * @param mk the implementation
+ * @param path the path
+ * @param rev the revision
+ * @param batchSize the batch size
+ * @return a child node name iterator
+ */
+ public static Iterator<String> getAllChildNodeNames(final MicroKernel mk, final String path, final String rev, final int batchSize) {
+ return new Iterator<String>() {
+
+ private long offset;
+ private Iterator<String> current;
+
+ {
+ nextBatch();
+ }
+
+ private void nextBatch() {
+ NodeImpl n = NodeImpl.parse(mk.getNodes(path, rev, 0, offset, batchSize));
+ current = n.getChildNodeNames(Integer.MAX_VALUE);
+ offset += batchSize;
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (!current.hasNext()) {
+ nextBatch();
+ }
+ return current.hasNext();
+ }
+
+ @Override
+ public String next() {
+ if (!current.hasNext()) {
+ nextBatch();
+ }
+ return current.next();
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+
+ /**
* The sync handler.
*/
public interface Handler {
Modified: jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/util/SyncTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/util/SyncTest.java?rev=1243562&r1=1243561&r2=1243562&view=diff
==============================================================================
--- jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/util/SyncTest.java (original)
+++ jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/util/SyncTest.java Mon Feb 13 15:32:39 2012
@@ -14,6 +14,9 @@
package org.apache.jackrabbit.mk.util;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import java.util.HashSet;
+import java.util.Iterator;
import org.apache.jackrabbit.mk.MultiMkTestBase;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -30,9 +33,37 @@ public class SyncTest extends MultiMkTes
}
@Test
+ public void testIterator() {
+ HashSet<String> names = new HashSet<String>();
+ for (int i = 0; i < 20; i++) {
+ mk.commit("/", "+ \"n" + i + "\": {}", mk.getHeadRevision(), "");
+ names.add("n" + i);
+ }
+ String head = mk.getHeadRevision();
+ Iterator<String> it = Sync.getAllChildNodeNames(mk, "/", head, 2);
+ while (it.hasNext()) {
+ String n = it.next();
+ assertTrue(names.remove(n));
+ }
+ assertEquals(0, names.size());
+ }
+
+ @Test
public void test() {
+ doTest(-1);
+ }
+
+ @Test
+ public void testSmallChildNodeBatchSize() {
+ doTest(1);
+ }
+
+ private void doTest(int childNodeBatchSize) {
mk.commit("/", "+ \"source\": { \"id\": 1, \"plus\": 0, \"a\": { \"x\": 10, \"y\": 20 }, \"b\": {\"z\": 100}, \"d\":{} }", mk.getHeadRevision(), "");
Sync sync = new Sync();
+ if (childNodeBatchSize > 0) {
+ sync.setChildNodesPerBatch(childNodeBatchSize);
+ }
String head = mk.getHeadRevision();
sync.setSource(mk, head, "/");
String diff = syncToString(sync);