You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ca...@apache.org on 2018/03/01 04:17:03 UTC
svn commit: r1825619 - in /jackrabbit/oak/trunk/oak-run/src:
main/java/org/apache/jackrabbit/oak/index/indexer/document/
main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/
test/java/org/apache/jackrabbit/oak/index/indexer/document/fla...
Author: catholicon
Date: Thu Mar 1 04:17:02 2018
New Revision: 1825619
URL: http://svn.apache.org/viewvc?rev=1825619&view=rev
Log:
OAK-7290: Reindexing using --doc-traversal-mode should have configurable upper bound for mem usage
Wiring to track memory used by buffer linked list
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/NodeStateEntry.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedList.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryReader.java
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedListTest.java
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIteratorTest.java
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryWriterTest.java
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/NodeStateEntry.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/NodeStateEntry.java?rev=1825619&r1=1825618&r2=1825619&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/NodeStateEntry.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/NodeStateEntry.java Thu Mar 1 04:17:02 2018
@@ -26,10 +26,16 @@ import org.apache.jackrabbit.oak.spi.sta
public class NodeStateEntry {
private final NodeState nodeState;
private final String path;
+ private final long memUsage;
public NodeStateEntry(NodeState nodeState, String path) {
+ this(nodeState, path, 0);
+ }
+
+ public NodeStateEntry(NodeState nodeState, String path, long memUsage) {
this.nodeState = nodeState;
this.path = path;
+ this.memUsage = memUsage;
}
public NodeState getNodeState() {
@@ -40,6 +46,10 @@ public class NodeStateEntry {
return path;
}
+ public long estimatedMemUsage() {
+ return memUsage;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) return true;
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedList.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedList.java?rev=1825619&r1=1825618&r2=1825619&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedList.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedList.java Thu Mar 1 04:17:02 2018
@@ -38,6 +38,7 @@ public class FlatFileBufferLinkedList {
private ListNode tail = head;
private int size = 0;
+ private long memUsage = 0;
/**
* Add {@code item} at the tail of the list
@@ -47,6 +48,7 @@ public class FlatFileBufferLinkedList {
tail.next = new ListNode(item);
tail = tail.next;
size++;
+ memUsage += item.estimatedMemUsage();
}
/**
@@ -59,6 +61,7 @@ public class FlatFileBufferLinkedList {
head.next.isValid = false;
head.next = head.next.next;
size--;
+ memUsage -= ret.estimatedMemUsage();
if (size == 0) {
tail = head;
}
@@ -80,6 +83,10 @@ public class FlatFileBufferLinkedList {
return size == 0;
}
+ public long estimatedMemoryUsage() {
+ return memUsage;
+ }
+
/**
* Represents an item in the list.
*/
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java?rev=1825619&r1=1825618&r2=1825619&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java Thu Mar 1 04:17:02 2018
@@ -49,6 +49,10 @@ class FlatFileStoreIterator extends Abst
return buffer.size();
}
+ long getBufferMemoryUsage() {
+ return buffer.estimatedMemoryUsage();
+ }
+
@Override
protected NodeStateEntry computeNext() {
//TODO Add some checks on expected ordering
@@ -64,7 +68,8 @@ class FlatFileStoreIterator extends Abst
private NodeStateEntry computeNextEntry() {
if (buffer.size() > maxBufferSize) {
maxBufferSize = buffer.size();
- log.info("Max buffer size changed {} for path {}", maxBufferSize, current.getPath());
+ log.info("Max buffer size changed {} (estimated memory usage: {} bytes) for path {}",
+ maxBufferSize, buffer.estimatedMemoryUsage(), current.getPath());
}
if (!buffer.isEmpty()) {
return buffer.remove();
@@ -78,7 +83,7 @@ class FlatFileStoreIterator extends Abst
private NodeStateEntry wrap(NodeStateEntry baseEntry) {
NodeState state = new LazyChildrenNodeState(baseEntry.getNodeState(),
new ChildNodeStateProvider(getEntries(), baseEntry.getPath(), preferredPathElements));
- return new NodeStateEntry(state, baseEntry.getPath());
+ return new NodeStateEntry(state, baseEntry.getPath(), baseEntry.estimatedMemUsage());
}
private Iterable<NodeStateEntry> getEntries() {
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryReader.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryReader.java?rev=1825619&r1=1825618&r2=1825619&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryReader.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryReader.java Thu Mar 1 04:17:02 2018
@@ -26,6 +26,8 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.spi.state.NodeState;
+import static org.apache.jackrabbit.oak.commons.StringUtils.estimateMemoryUsage;
+
public class NodeStateEntryReader {
private final BlobDeserializer blobDeserializer;
@@ -35,7 +37,8 @@ public class NodeStateEntryReader {
public NodeStateEntry read(String line){
String[] parts = NodeStateEntryWriter.getParts(line);
- return new NodeStateEntry(parseState(parts[1]), parts[0]);
+ long memUsage = estimateMemoryUsage(parts[0]) + estimateMemoryUsage(parts[1]);
+ return new NodeStateEntry(parseState(parts[1]), parts[0], memUsage);
}
private NodeState parseState(String part) {
Modified: jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedListTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedListTest.java?rev=1825619&r1=1825618&r2=1825619&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedListTest.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedListTest.java Thu Mar 1 04:17:02 2018
@@ -136,6 +136,20 @@ public class FlatFileBufferLinkedListTes
}
@Test
+ public void memUsage() {
+ assertEquals("Empty list must be estimate 0", 0, list.estimatedMemoryUsage());
+
+ list.add(new NodeStateEntry(EMPTY_NODE, "/", 20));
+ assertEquals(20, list.estimatedMemoryUsage());
+
+ list.add(new NodeStateEntry(EMPTY_NODE, "/", 30));
+ assertEquals(50, list.estimatedMemoryUsage());
+
+ list.remove();
+ assertEquals(30, list.estimatedMemoryUsage());
+ }
+
+ @Test
public void basics() {
list.add(TEST_NODE_STATE_ENTRY);
assertEquals("Adding an item should change size", 1, list.size());
Modified: jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIteratorTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIteratorTest.java?rev=1825619&r1=1825618&r2=1825619&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIteratorTest.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIteratorTest.java Thu Mar 1 04:17:02 2018
@@ -19,10 +19,13 @@
package org.apache.jackrabbit.oak.index.indexer.document.flatfile;
+import java.util.List;
import java.util.Set;
import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry;
+import org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState;
import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.junit.Test;
@@ -31,6 +34,7 @@ import static java.util.Arrays.asList;
import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.TestUtils.createList;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
public class FlatFileStoreIteratorTest {
@@ -142,4 +146,23 @@ public class FlatFileStoreIteratorTest {
// Don't read whole tree to conclude that "j:c" doesn't exist (reading /a/b should imply that it doesn't exist)
assertEquals(1, fitr.getBufferSize());
}
+
+ @Test
+ public void bufferEstimatesMemory() {
+ List<NodeStateEntry> nseList = Lists.newArrayList(
+ new NodeStateEntry(EmptyNodeState.EMPTY_NODE, "/a", 20),
+ new NodeStateEntry(EmptyNodeState.EMPTY_NODE, "/a/b", 30)
+ );
+ FlatFileStoreIterator fitr = new FlatFileStoreIterator(nseList.iterator(), ImmutableSet.of());
+
+ NodeStateEntry entry = fitr.next();
+ NodeState entryNS = entry.getNodeState();
+ assertEquals("/a", entry.getPath());
+ assertEquals("Fetching from iterator doesn't use buffer", 0, fitr.getBufferSize());
+
+ entryNS.getChildNode("b");
+ assertEquals(1, fitr.getBufferSize());
+ assertEquals("Reaching child from node state should estimate 30 for /a/b",
+ 30, fitr.getBufferMemoryUsage());
+ }
}
\ No newline at end of file
Modified: jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryWriterTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryWriterTest.java?rev=1825619&r1=1825618&r2=1825619&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryWriterTest.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryWriterTest.java Thu Mar 1 04:17:02 2018
@@ -28,6 +28,7 @@ import org.apache.jackrabbit.oak.spi.blo
import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore;
import org.apache.jackrabbit.oak.spi.state.EqualsDiff;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.junit.Test;
import static com.google.common.collect.ImmutableList.copyOf;
@@ -141,4 +142,30 @@ public class NodeStateEntryWriterTest {
}
+ @Test
+ public void memUsage() {
+ NodeStateEntryWriter nw = new NodeStateEntryWriter(blobStore);
+ NodeBuilder b = EMPTY_NODE.builder();
+ b.setProperty("foo", "bar");
+
+ String json1 = nw.asJson(b.getNodeState());
+
+ b.setProperty("foo1", "bar1");
+ String json2 = nw.asJson(b.getNodeState());
+
+ String line1 = nw.toString(copyOf(elements("/")), json1);
+ String line2 = nw.toString(copyOf(elements("/sub-node")), json1);
+ String line3 = nw.toString(copyOf(elements("/sub-node")), json2);
+
+ NodeStateEntryReader nr = new NodeStateEntryReader(blobStore);
+
+ long size1 = nr.read(line1).estimatedMemUsage();
+ long size2 = nr.read(line2).estimatedMemUsage();
+ long size3 = nr.read(line3).estimatedMemUsage();
+
+ assertTrue("Mem usage should be more than 0", size1 > 0);
+ assertTrue("Mem usage should increase with longer path", size2 > size1);
+ assertTrue("Mem usage should increase with bigger node state", size3 > size2);
+ }
+
}
\ No newline at end of file