You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by mr...@apache.org on 2017/01/30 10:14:06 UTC
svn commit: r1780887 - in /jackrabbit/oak/trunk/oak-core/src:
main/java/org/apache/jackrabbit/oak/plugins/document/
test/java/org/apache/jackrabbit/oak/plugins/document/
Author: mreutegg
Date: Mon Jan 30 10:14:05 2017
New Revision: 1780887
URL: http://svn.apache.org/viewvc?rev=1780887&view=rev
Log:
OAK-5479: Overdue document split with many cluster nodes
Added:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/ManyClusterNodesTest.java (with props)
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocument.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/SplitOperations.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentSplitTest.java
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocument.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocument.java?rev=1780887&r1=1780886&r2=1780887&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocument.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocument.java Mon Jan 30 10:14:05 2017
@@ -113,7 +113,7 @@ public final class NodeDocument extends
* A document size threshold after which a split is forced even if
* {@link #NUM_REVS_THRESHOLD} is not reached.
*/
- static final int DOC_SIZE_THRESHOLD = 256 * 1024;
+ static final int DOC_SIZE_THRESHOLD = 1024 * 1024;
/**
* Only split off at least this number of revisions.
@@ -121,12 +121,6 @@ public final class NodeDocument extends
static final int NUM_REVS_THRESHOLD = 100;
/**
- * The split ratio. Only split data to an old document when at least
- * 30% of the data can be moved.
- */
- static final float SPLIT_RATIO = 0.3f;
-
- /**
* Create an intermediate previous document when there are this many
* previous documents of equal height.
*/
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/SplitOperations.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/SplitOperations.java?rev=1780887&r1=1780886&r2=1780887&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/SplitOperations.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/SplitOperations.java Mon Jan 30 10:14:05 2017
@@ -51,7 +51,6 @@ import static org.apache.jackrabbit.oak.
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.DOC_SIZE_THRESHOLD;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.PREV_SPLIT_FACTOR;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.REVISIONS;
-import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.SPLIT_RATIO;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.SplitDocType;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.isCommitRootEntry;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.isRevisionsEntry;
@@ -377,14 +376,16 @@ class SplitOperations {
NodeDocument oldDoc = new NodeDocument(STORE);
UpdateUtils.applyChanges(oldDoc, old);
setSplitDocProps(doc, oldDoc, old, high);
- // only split if enough of the data can be moved to old document
- // or there are binaries to split off
- if (oldDoc.getMemory() > doc.getMemory() * SPLIT_RATIO
- || numValues >= numRevsThreshold
- || hasBinaryToSplit) {
- splitOps.add(old);
- } else {
- main = null;
+ splitOps.add(old);
+
+ if (numValues < numRevsThreshold) {
+ String reason;
+ if (hasBinaryToSplit) {
+ reason = "binary";
+ } else {
+ reason = "size";
+ }
+ LOG.debug("Force splitting {} ({})", id, reason);
}
}
return main;
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentSplitTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentSplitTest.java?rev=1780887&r1=1780886&r2=1780887&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentSplitTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentSplitTest.java Mon Jan 30 10:14:05 2017
@@ -53,6 +53,7 @@ import com.google.common.collect.Sets;
import static com.google.common.collect.ImmutableList.copyOf;
import static org.apache.jackrabbit.oak.plugins.document.Collection.NODES;
import static org.apache.jackrabbit.oak.plugins.document.MongoBlobGCTest.randomStream;
+import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.DOC_SIZE_THRESHOLD;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.NUM_REVS_THRESHOLD;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.PREV_SPLIT_FACTOR;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.SplitDocType;
@@ -510,7 +511,7 @@ public class DocumentSplitTest extends B
UpdateOp op = new UpdateOp(id, false);
// create some baggage from another cluster node
- for (int i = 0; i < 1000; i++) {
+ for (int i = 0; i < 4000; i++) {
Revision r = Revision.newRevision(2);
op.setMapEntry("prop", r, "some long test value with many characters");
NodeDocument.setRevision(op, r, "c");
@@ -518,14 +519,8 @@ public class DocumentSplitTest extends B
store.findAndUpdate(NODES, op);
NodeDocument doc = store.find(NODES, id);
assertNotNull(doc);
- assertTrue(doc.getMemory() > NodeDocument.DOC_SIZE_THRESHOLD);
+ assertTrue(doc.getMemory() > DOC_SIZE_THRESHOLD);
- // these will be considered for a split
- for (int i = 0; i < NUM_REVS_THRESHOLD / 2; i++) {
- Revision r = Revision.newRevision(clusterId);
- op.setMapEntry("prop", r, "value");
- NodeDocument.setRevision(op, r, "c");
- }
// some fake previous doc references to trigger UpdateOp
// for an intermediate document
TreeSet<Revision> prev = Sets.newTreeSet(StableRevisionComparator.INSTANCE);
@@ -979,6 +974,30 @@ public class DocumentSplitTest extends B
assertEquals(0, prevDocs.size());
}
+ @Test
+ public void nonSplittableBigDocument() throws Exception {
+ DocumentStore store = mk.getDocumentStore();
+ DocumentNodeStore ns = mk.getNodeStore();
+ NodeBuilder builder = ns.getRoot().builder();
+ builder.child("foo");
+ merge(ns, builder);
+
+ String id = Utils.getIdFromPath("/foo");
+ int num = 0;
+ while (store.find(NODES, id).getMemory() < DOC_SIZE_THRESHOLD) {
+ builder = ns.getRoot().builder();
+ for (int i = 0; i < 50; i++) {
+ builder.child("foo").setProperty("p" + num++,
+ "some value as payload for the document");
+ }
+ merge(ns, builder);
+ }
+
+ Iterable<UpdateOp> splitOps = store.find(NODES, id)
+ .split(ns, ns.getHeadRevision(), NO_BINARY);
+ assertEquals(0, Iterables.size(splitOps));
+ }
+
private static class TestRevisionContext implements RevisionContext {
private final RevisionContext rc;
Added: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/ManyClusterNodesTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/ManyClusterNodesTest.java?rev=1780887&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/ManyClusterNodesTest.java (added)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/ManyClusterNodesTest.java Mon Jan 30 10:14:05 2017
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.document;
+
+import java.util.List;
+
+import com.google.common.collect.Iterables;
+
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.plugins.document.memory.MemoryDocumentStore;
+import org.apache.jackrabbit.oak.plugins.document.util.Utils;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import static com.google.common.collect.Lists.newArrayList;
+import static org.apache.jackrabbit.oak.plugins.document.TestUtils.merge;
+import static org.junit.Assert.assertTrue;
+
+public class ManyClusterNodesTest {
+
+ @Rule
+ public DocumentMKBuilderProvider builderProvider = new DocumentMKBuilderProvider();
+
+ private static final int NUM_CLUSTER_NODES = 8;
+
+ private static final long MB = 1024 * 1024;
+
+ private DocumentStore ds;
+
+ private List<DocumentNodeStore> stores = newArrayList();
+
+ @Before
+ public void before() throws Exception {
+ ds = new MemoryDocumentStore();
+ for (int i = 0; i < NUM_CLUSTER_NODES; i++) {
+ stores.add(builderProvider.newBuilder()
+ .setClusterId(i + 1)
+ .setDocumentStore(ds)
+ .setAsyncDelay(0)
+ .getNodeStore());
+ stores.get(i).runBackgroundOperations();
+ }
+ }
+
+ // OAK-5479
+ @Test
+ public void manyMultiValuedPropertyChanges() throws Exception {
+ String value = "some-long-value-that-will-bloat-the-document";
+ String id = Utils.getIdFromPath("/test");
+ for (int i = 0; i < 1000; i++) {
+ DocumentNodeStore ns = stores.get(i % NUM_CLUSTER_NODES);
+ ns.runBackgroundOperations();
+ NodeBuilder builder = ns.getRoot().builder();
+ NodeBuilder test = builder.child("test");
+ PropertyState p = test.getProperty("p");
+ List<String> values = newArrayList();
+ if (p != null) {
+ Iterables.addAll(values, p.getValue(Type.STRINGS));
+ }
+ values.add(value);
+ test.setProperty("p", values, Type.STRINGS);
+ merge(ns, builder);
+ int size = ds.find(Collection.NODES, id).getMemory();
+ assertTrue(size < 8 * MB);
+ ns.runBackgroundOperations();
+ }
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/ManyClusterNodesTest.java
------------------------------------------------------------------------------
svn:eol-style = native