You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ca...@apache.org on 2018/03/01 05:01:04 UTC
svn commit: r1825623 - in /jackrabbit/oak/branches/1.8: ./
oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/
oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/
oak-run/src/test/java/org/apache/jackrab...
Author: catholicon
Date: Thu Mar 1 05:01:03 2018
New Revision: 1825623
URL: http://svn.apache.org/viewvc?rev=1825623&view=rev
Log:
OAK-7284: Reindexing using --doc-traversal-mode can hit ConcurrentModificationException during aggregation
OAK-7285: Reindexing using --doc-traversal-mode can OOM while aggregation in some cases
OAK-7290: Reindexing using --doc-traversal-mode should have configurable upper bound for mem usage
(backport r1825381,r1825448,r1825466,r1825475,r1825523,r1825525,r1825619,r1825620,r1825621 from trunk)
++++
OAK-7284: Reindexing using --doc-traversal-mode can hit ConcurrentModificationException during aggregation
++++
OAK-7285: Reindexing using --doc-traversal-mode can OOM while aggregation in some cases
Instead of using number_of_preferred_children, we now pass along preferred_childred set.
While gettingChildNode by name which is present in preffered set, we break iteration on encountering any non-preferred child.
Fixed tests to adhere to this new parameter and in particular fixed ChildNodeStateProviderTest#hasChildNode_InLimit.
The test earlier assumed that some children won't come in getChildNode if they aren't preferred.
++++
OAK-7290: Reindexing using --doc-traversal-mode should have configurable upper bound for mem usage
Default mem limit is 100MB.
The limit can be configure by setting jvm param 'oak.indexer.memLimitInMB'
Negative value implies unbounded usage (long_max)
Added:
jackrabbit/oak/branches/1.8/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedList.java
- copied, changed from r1825523, jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedList.java
jackrabbit/oak/branches/1.8/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedListTest.java
- copied, changed from r1825523, jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedListTest.java
Modified:
jackrabbit/oak/branches/1.8/ (props changed)
jackrabbit/oak/branches/1.8/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/NodeStateEntry.java
jackrabbit/oak/branches/1.8/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/ChildNodeStateProvider.java
jackrabbit/oak/branches/1.8/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java
jackrabbit/oak/branches/1.8/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java
jackrabbit/oak/branches/1.8/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java
jackrabbit/oak/branches/1.8/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryReader.java
jackrabbit/oak/branches/1.8/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/ChildNodeStateProviderTest.java
jackrabbit/oak/branches/1.8/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIteratorTest.java
jackrabbit/oak/branches/1.8/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryWriterTest.java
Propchange: jackrabbit/oak/branches/1.8/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Mar 1 05:01:03 2018
@@ -1,3 +1,3 @@
/jackrabbit/oak/branches/1.0:1665962
-/jackrabbit/oak/trunk:1820660-1820661,1820729,1820734,1820859,1820861,1820878,1820888,1820947,1821130,1821140-1821141,1821240,1821249,1821258,1821325,1821358,1821361-1821362,1821370,1821375,1821393,1821477,1821487,1821516,1821665,1821668,1821681,1822121,1822201,1822723,1822808,1822850,1822934,1823135,1823163,1823169,1824962,1825362,1825442
+/jackrabbit/oak/trunk:1820660-1820661,1820729,1820734,1820859,1820861,1820878,1820888,1820947,1821130,1821140-1821141,1821240,1821249,1821258,1821325,1821358,1821361-1821362,1821370,1821375,1821393,1821477,1821487,1821516,1821665,1821668,1821681,1822121,1822201,1822723,1822808,1822850,1822934,1823135,1823163,1823169,1824962,1825362,1825381,1825442,1825448,1825466,1825475,1825523,1825525,1825619-1825621
/jackrabbit/trunk:1345480
Modified: jackrabbit/oak/branches/1.8/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/NodeStateEntry.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/NodeStateEntry.java?rev=1825623&r1=1825622&r2=1825623&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.8/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/NodeStateEntry.java (original)
+++ jackrabbit/oak/branches/1.8/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/NodeStateEntry.java Thu Mar 1 05:01:03 2018
@@ -26,10 +26,16 @@ import org.apache.jackrabbit.oak.spi.sta
public class NodeStateEntry {
private final NodeState nodeState;
private final String path;
+ private final long memUsage;
public NodeStateEntry(NodeState nodeState, String path) {
+ this(nodeState, path, 0);
+ }
+
+ public NodeStateEntry(NodeState nodeState, String path, long memUsage) {
this.nodeState = nodeState;
this.path = path;
+ this.memUsage = memUsage;
}
public NodeState getNodeState() {
@@ -40,6 +46,10 @@ public class NodeStateEntry {
return path;
}
+ public long estimatedMemUsage() {
+ return memUsage;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) return true;
Modified: jackrabbit/oak/branches/1.8/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/ChildNodeStateProvider.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/ChildNodeStateProvider.java?rev=1825623&r1=1825622&r2=1825623&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.8/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/ChildNodeStateProvider.java (original)
+++ jackrabbit/oak/branches/1.8/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/ChildNodeStateProvider.java Thu Mar 1 05:01:03 2018
@@ -20,6 +20,7 @@
package org.apache.jackrabbit.oak.index.indexer.document.flatfile;
import java.util.Iterator;
+import java.util.Set;
import javax.annotation.Nonnull;
@@ -27,13 +28,13 @@ import com.google.common.base.Optional;
import com.google.common.collect.AbstractIterator;
import com.google.common.collect.Iterators;
import com.google.common.collect.PeekingIterator;
+import org.apache.jackrabbit.oak.commons.PathUtils;
import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry;
import org.apache.jackrabbit.oak.plugins.memory.MemoryChildNodeEntry;
import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import static com.google.common.base.Preconditions.checkState;
-import static com.google.common.collect.Iterators.limit;
import static com.google.common.collect.Iterators.size;
import static com.google.common.collect.Iterators.transform;
import static java.util.Collections.emptyIterator;
@@ -45,12 +46,12 @@ import static org.apache.jackrabbit.oak.
class ChildNodeStateProvider {
private final Iterable<NodeStateEntry> entries;
private final String path;
- private final int checkChildLimit;
+ private final Set<String> preferredPathElements;
- public ChildNodeStateProvider(Iterable<NodeStateEntry> entries, String path, int checkChildLimit) {
+ public ChildNodeStateProvider(Iterable<NodeStateEntry> entries, String path, Set<String> preferredPathElements) {
this.entries = entries;
this.path = path;
- this.checkChildLimit = checkChildLimit;
+ this.preferredPathElements = preferredPathElements;
}
public boolean hasChildNode(@Nonnull String name) {
@@ -59,7 +60,8 @@ class ChildNodeStateProvider {
@Nonnull
public NodeState getChildNode(@Nonnull String name) throws IllegalArgumentException {
- Optional<NodeStateEntry> o = Iterators.tryFind(limit(children(), checkChildLimit), p -> name.equals(name(p)));
+ boolean isPreferred = preferredPathElements.contains(name);
+ Optional<NodeStateEntry> o = Iterators.tryFind(children(isPreferred), p -> name.equals(name(p)));
return o.isPresent() ? o.get().getNodeState() : MISSING_NODE;
}
@@ -80,6 +82,10 @@ class ChildNodeStateProvider {
}
Iterator<NodeStateEntry> children() {
+ return children(false);
+ }
+
+ Iterator<NodeStateEntry> children(boolean preferred) {
PeekingIterator<NodeStateEntry> pitr = Iterators.peekingIterator(entries.iterator());
if (!pitr.hasNext()) {
return emptyIterator();
@@ -96,18 +102,23 @@ class ChildNodeStateProvider {
"after main iterator has moved past it", path);
//Prepare an iterator to fetch all child node paths i.e. immediate and there children
- Iterator<NodeStateEntry> itr = new AbstractIterator<NodeStateEntry>() {
+ return new AbstractIterator<NodeStateEntry>() {
@Override
protected NodeStateEntry computeNext() {
- if (pitr.hasNext() && isAncestor(path, pitr.peek().getPath())) {
- return pitr.next();
+ while (pitr.hasNext() && isAncestor(path, pitr.peek().getPath())) {
+ NodeStateEntry nextEntry = pitr.next();
+ String nextEntryPath = nextEntry.getPath();
+ if (isImmediateChild(nextEntryPath)) {
+ String nextEntryName = PathUtils.getName(nextEntryPath);
+ if (preferred && !preferredPathElements.contains(nextEntryName)) {
+ return endOfData();
+ }
+ return nextEntry;
+ }
}
return endOfData();
}
};
-
- //Filter out non immediate children
- return Iterators.filter(itr, (e) -> isImmediateChild(e.getPath()));
}
private static String name(NodeStateEntry p) {
Copied: jackrabbit/oak/branches/1.8/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedList.java (from r1825523, jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedList.java)
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedList.java?p2=jackrabbit/oak/branches/1.8/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedList.java&p1=jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedList.java&r1=1825523&r2=1825623&rev=1825623&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedList.java (original)
+++ jackrabbit/oak/branches/1.8/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedList.java Thu Mar 1 05:01:03 2018
@@ -38,14 +38,32 @@ public class FlatFileBufferLinkedList {
private ListNode tail = head;
private int size = 0;
+ private long memUsage = 0;
+ private final long memLimit;
+
+ FlatFileBufferLinkedList() {
+ this(Long.MAX_VALUE);
+ }
+
+ FlatFileBufferLinkedList(long memLimit) {
+ this.memLimit = memLimit;
+ }
/**
* Add {@code item} at the tail of the list
*/
public void add(@Nonnull NodeStateEntry item) {
+ Preconditions.checkArgument(item != null, "Can't add null to the list");
+ long incomingSize = item.estimatedMemUsage();
+ long memUsage = estimatedMemoryUsage();
+ Preconditions.checkState(memUsage + incomingSize <= memLimit,
+ String.format(
+ "Adding item (%s) estimated with %s bytes would increase mem usage beyond upper limit (%s)." +
+ " Current estimated mem usage is %s bytes", item.getPath(), incomingSize, memLimit, memUsage));
tail.next = new ListNode(item);
tail = tail.next;
size++;
+ this.memUsage += incomingSize;
}
/**
@@ -58,6 +76,7 @@ public class FlatFileBufferLinkedList {
head.next.isValid = false;
head.next = head.next.next;
size--;
+ memUsage -= ret.estimatedMemUsage();
if (size == 0) {
tail = head;
}
@@ -79,6 +98,10 @@ public class FlatFileBufferLinkedList {
return size == 0;
}
+ public long estimatedMemoryUsage() {
+ return memUsage;
+ }
+
/**
* Represents an item in the list.
*/
@@ -93,6 +116,7 @@ public class FlatFileBufferLinkedList {
}
ListNode(@Nonnull NodeStateEntry data) {
+ Preconditions.checkNotNull(data);
this.data = data;
this.next = null;
}
@@ -102,10 +126,12 @@ public class FlatFileBufferLinkedList {
private ListNode current;
static NodeIterator iteratorFor(@Nonnull ListNode node) {
+ Preconditions.checkNotNull(node);
return new NodeIterator(node);
}
NodeIterator(@Nonnull ListNode start) {
+ Preconditions.checkNotNull(start);
this.current = start;
}
Modified: jackrabbit/oak/branches/1.8/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java?rev=1825623&r1=1825622&r2=1825623&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.8/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java (original)
+++ jackrabbit/oak/branches/1.8/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java Thu Mar 1 05:01:03 2018
@@ -22,6 +22,7 @@ package org.apache.jackrabbit.oak.index.
import java.io.File;
import java.io.IOException;
import java.util.Collections;
+import java.util.Set;
import com.google.common.collect.Iterables;
import org.apache.commons.io.FileUtils;
@@ -30,7 +31,7 @@ import org.apache.jackrabbit.oak.spi.blo
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static com.google.common.collect.Iterables.size;
+import static java.util.Collections.unmodifiableSet;
public class FlatFileNodeStoreBuilder {
public static final String OAK_INDEXER_USE_ZIP = "oak.indexer.useZip";
@@ -41,7 +42,7 @@ public class FlatFileNodeStoreBuilder {
private final Logger log = LoggerFactory.getLogger(getClass());
private final Iterable<NodeStateEntry> nodeStates;
private final File workDir;
- private Iterable<String> preferredPathElements = Collections.emptySet();
+ private Set<String> preferredPathElements = Collections.emptySet();
private BlobStore blobStore;
private PathElementComparator comparator;
private NodeStateEntryWriter entryWriter;
@@ -60,7 +61,7 @@ public class FlatFileNodeStoreBuilder {
return this;
}
- public FlatFileNodeStoreBuilder withPreferredPathElements(Iterable<String> preferredPathElements) {
+ public FlatFileNodeStoreBuilder withPreferredPathElements(Set<String> preferredPathElements) {
this.preferredPathElements = preferredPathElements;
return this;
}
@@ -70,7 +71,7 @@ public class FlatFileNodeStoreBuilder {
comparator = new PathElementComparator(preferredPathElements);
entryWriter = new NodeStateEntryWriter(blobStore);
FlatFileStore store = new FlatFileStore(createdSortedStoreFile(), new NodeStateEntryReader(blobStore),
- size(preferredPathElements), useZip);
+ unmodifiableSet(preferredPathElements), useZip);
if (entryCount > 0) {
store.setEntryCount(entryCount);
}
Modified: jackrabbit/oak/branches/1.8/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java?rev=1825623&r1=1825622&r2=1825623&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.8/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java (original)
+++ jackrabbit/oak/branches/1.8/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java Thu Mar 1 05:01:03 2018
@@ -23,6 +23,7 @@ import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.util.Iterator;
+import java.util.Set;
import com.google.common.collect.AbstractIterator;
import com.google.common.io.Closer;
@@ -35,14 +36,14 @@ public class FlatFileStore implements It
private final Closer closer = Closer.create();
private final File storeFile;
private final NodeStateEntryReader entryReader;
- private final int checkChildLimit;
+ private final Set<String> preferredPathElements;
private final boolean compressionEnabled;
private long entryCount = -1;
- public FlatFileStore(File storeFile, NodeStateEntryReader entryReader, int checkChildLimit, boolean compressionEnabled) {
+ public FlatFileStore(File storeFile, NodeStateEntryReader entryReader, Set<String> preferredPathElements, boolean compressionEnabled) {
this.storeFile = storeFile;
this.entryReader = entryReader;
- this.checkChildLimit = checkChildLimit;
+ this.preferredPathElements = preferredPathElements;
this.compressionEnabled = compressionEnabled;
}
@@ -56,7 +57,7 @@ public class FlatFileStore implements It
@Override
public Iterator<NodeStateEntry> iterator() {
- return new FlatFileStoreIterator(createBaseIterator(), checkChildLimit);
+ return new FlatFileStoreIterator(createBaseIterator(), preferredPathElements);
}
private Iterator<NodeStateEntry> createBaseIterator() {
Modified: jackrabbit/oak/branches/1.8/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java?rev=1825623&r1=1825622&r2=1825623&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.8/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java (original)
+++ jackrabbit/oak/branches/1.8/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java Thu Mar 1 05:01:03 2018
@@ -20,11 +20,11 @@
package org.apache.jackrabbit.oak.index.indexer.document.flatfile;
import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.ListIterator;
+import java.util.Set;
import com.google.common.collect.AbstractIterator;
import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry;
+import org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileBufferLinkedList.NodeIterator;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -35,20 +35,35 @@ import static com.google.common.collect.
class FlatFileStoreIterator extends AbstractIterator<NodeStateEntry> implements Iterator<NodeStateEntry> {
private final Logger log = LoggerFactory.getLogger(getClass());
private final Iterator<NodeStateEntry> baseItr;
- private final LinkedList<NodeStateEntry> buffer = new LinkedList<>();
+ private final FlatFileBufferLinkedList buffer;
private NodeStateEntry current;
- private final int checkChildLimit;
+ private final Set<String> preferredPathElements;
private int maxBufferSize;
+ static final String BUFFER_MEM_LIMIT_CONFIG_NAME = "oak.indexer.memLimitInMB";
+ private static final int DEFAULT_BUFFER_MEM_LIMIT_IN_MB = 100;
- public FlatFileStoreIterator(Iterator<NodeStateEntry> baseItr, int checkChildLimit) {
+ public FlatFileStoreIterator(Iterator<NodeStateEntry> baseItr, Set<String> preferredPathElements) {
this.baseItr = baseItr;
- this.checkChildLimit = checkChildLimit;
+ this.preferredPathElements = preferredPathElements;
+
+ int memLimitConfig = Integer.getInteger(BUFFER_MEM_LIMIT_CONFIG_NAME, DEFAULT_BUFFER_MEM_LIMIT_IN_MB);
+ if (memLimitConfig < 0) {
+ log.info("Setting buffer memory limit unbounded", memLimitConfig);
+ this.buffer = new FlatFileBufferLinkedList();
+ } else {
+ log.info("Setting buffer memory limit to {} MBs", memLimitConfig);
+ this.buffer = new FlatFileBufferLinkedList(memLimitConfig * 1024L * 1024L);
+ }
}
int getBufferSize(){
return buffer.size();
}
+ long getBufferMemoryUsage() {
+ return buffer.estimatedMemoryUsage();
+ }
+
@Override
protected NodeStateEntry computeNext() {
//TODO Add some checks on expected ordering
@@ -64,7 +79,8 @@ class FlatFileStoreIterator extends Abst
private NodeStateEntry computeNextEntry() {
if (buffer.size() > maxBufferSize) {
maxBufferSize = buffer.size();
- log.info("Max buffer size changed {} for path {}", maxBufferSize, current.getPath());
+ log.info("Max buffer size changed {} (estimated memory usage: {} bytes) for path {}",
+ maxBufferSize, buffer.estimatedMemoryUsage(), current.getPath());
}
if (!buffer.isEmpty()) {
return buffer.remove();
@@ -77,8 +93,8 @@ class FlatFileStoreIterator extends Abst
private NodeStateEntry wrap(NodeStateEntry baseEntry) {
NodeState state = new LazyChildrenNodeState(baseEntry.getNodeState(),
- new ChildNodeStateProvider(getEntries(), baseEntry.getPath(), checkChildLimit));
- return new NodeStateEntry(state, baseEntry.getPath());
+ new ChildNodeStateProvider(getEntries(), baseEntry.getPath(), preferredPathElements));
+ return new NodeStateEntry(state, baseEntry.getPath(), baseEntry.estimatedMemUsage());
}
private Iterable<NodeStateEntry> getEntries() {
@@ -86,14 +102,13 @@ class FlatFileStoreIterator extends Abst
}
private Iterator<NodeStateEntry> queueIterator() {
- ListIterator<NodeStateEntry> qitr = buffer.listIterator();
+ NodeIterator qitr = buffer.iterator();
return new AbstractIterator<NodeStateEntry>() {
@Override
protected NodeStateEntry computeNext() {
//If queue is empty try to append by getting entry from base
if (!qitr.hasNext() && baseItr.hasNext()) {
- qitr.add(wrap(baseItr.next()));
- qitr.previous(); //Move back the itr
+ buffer.add(wrap(baseItr.next()));
}
if (qitr.hasNext()) {
return qitr.next();
Modified: jackrabbit/oak/branches/1.8/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryReader.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryReader.java?rev=1825623&r1=1825622&r2=1825623&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.8/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryReader.java (original)
+++ jackrabbit/oak/branches/1.8/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryReader.java Thu Mar 1 05:01:03 2018
@@ -26,6 +26,8 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.spi.state.NodeState;
+import static org.apache.jackrabbit.oak.commons.StringUtils.estimateMemoryUsage;
+
public class NodeStateEntryReader {
private final BlobDeserializer blobDeserializer;
@@ -35,7 +37,8 @@ public class NodeStateEntryReader {
public NodeStateEntry read(String line){
String[] parts = NodeStateEntryWriter.getParts(line);
- return new NodeStateEntry(parseState(parts[1]), parts[0]);
+ long memUsage = estimateMemoryUsage(parts[0]) + estimateMemoryUsage(parts[1]);
+ return new NodeStateEntry(parseState(parts[1]), parts[0], memUsage);
}
private NodeState parseState(String part) {
Modified: jackrabbit/oak/branches/1.8/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/ChildNodeStateProviderTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/ChildNodeStateProviderTest.java?rev=1825623&r1=1825622&r2=1825623&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.8/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/ChildNodeStateProviderTest.java (original)
+++ jackrabbit/oak/branches/1.8/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/ChildNodeStateProviderTest.java Thu Mar 1 05:01:03 2018
@@ -47,7 +47,8 @@ public class ChildNodeStateProviderTest
@Test
public void emptyCase() {
- ChildNodeStateProvider p = new ChildNodeStateProvider(emptyList(), "/a", 5);
+ Set<String> preferred = ImmutableSet.of("u", "v", "x", "y", "z");
+ ChildNodeStateProvider p = new ChildNodeStateProvider(emptyList(), "/a", preferred);
assertEquals(0, p.getChildNodeCount(1));
assertEquals(0, Iterables.size(p.getChildNodeNames()));
assertEquals(0, Iterables.size(p.getChildNodeEntries()));
@@ -59,18 +60,18 @@ public class ChildNodeStateProviderTest
public void children() {
Set<String> preferred = ImmutableSet.of("jcr:content", "x");
CountingIterable<NodeStateEntry> citr = createList(preferred, asList("/a", "/a/jcr:content", "/a/c", "/a/d", "/e", "/e/f", "/g", "/h"));
- ChildNodeStateProvider p = new ChildNodeStateProvider(citr, "/a", 100);
+ ChildNodeStateProvider p = new ChildNodeStateProvider(citr, "/a", preferred);
assertEquals(asList("jcr:content", "c", "d"), copyOf(childNames(p.children())));
assertEquals(5, citr.getCount());
citr.reset();
- p = new ChildNodeStateProvider(citr, "/e", 100);
+ p = new ChildNodeStateProvider(citr, "/e", preferred);
assertEquals(singletonList("f"), copyOf(childNames(p.children())));
assertEquals(7, citr.getCount());
- p = new ChildNodeStateProvider(citr, "/g", 100);
+ p = new ChildNodeStateProvider(citr, "/g", preferred);
assertEquals(emptyList(), copyOf(childNames(p.children())));
}
@@ -78,23 +79,23 @@ public class ChildNodeStateProviderTest
public void children2() {
Set<String> preferred = ImmutableSet.of("b");
CountingIterable<NodeStateEntry> citr = createList(preferred, asList("/a", "/a/b", "/a/b/c", "/a/b/c/d", "/e", "/e/f", "/g", "/h"));
- ChildNodeStateProvider p = new ChildNodeStateProvider(citr, "/a", 100);
+ ChildNodeStateProvider p = new ChildNodeStateProvider(citr, "/a", preferred);
assertEquals(singletonList("b"), copyOf(childNames(p.children())));
assertEquals(5, citr.getCount());
citr.reset();
- p = new ChildNodeStateProvider(citr, "/a/b", 100);
+ p = new ChildNodeStateProvider(citr, "/a/b", preferred);
assertEquals(singletonList("c"), copyOf(childNames(p.children())));
assertEquals(5, citr.getCount());
- p = new ChildNodeStateProvider(citr, "/a/b/c", 100);
+ p = new ChildNodeStateProvider(citr, "/a/b/c", preferred);
assertEquals(singletonList("d"), copyOf(childNames(p.children())));
- p = new ChildNodeStateProvider(citr, "/a/b/c/d", 100);
+ p = new ChildNodeStateProvider(citr, "/a/b/c/d", preferred);
assertEquals(emptyList(), copyOf(childNames(p.children())));
- p = new ChildNodeStateProvider(citr, "/h", 100);
+ p = new ChildNodeStateProvider(citr, "/h", preferred);
assertEquals(emptyList(), copyOf(childNames(p.children())));
}
@@ -102,18 +103,47 @@ public class ChildNodeStateProviderTest
public void hasChildNode_InLimit() {
Set<String> preferred = ImmutableSet.of("jcr:content", "x");
CountingIterable<NodeStateEntry> citr = createList(preferred, asList("/a", "/a/jcr:content", "/a/c", "/a/d", "/e", "/e/f"));
- ChildNodeStateProvider p = new ChildNodeStateProvider(citr, "/a", preferred.size());
+ ChildNodeStateProvider p = new ChildNodeStateProvider(citr, "/a", preferred);
+ citr.reset();
assertTrue(p.hasChildNode("jcr:content"));
+ assertEquals("Unexpected number of reads to get jcr:content", 2, citr.getCount());
+
+ citr.reset();
+ assertFalse(p.hasChildNode("x"));
+ assertEquals("Unexpected number reads to conclude that preferred child 'x' is missing",
+ 3, citr.getCount());
+
+ citr.reset();
assertTrue(p.hasChildNode("c"));
- assertFalse(p.hasChildNode("d"));
+ assertEquals("Unexpected number reads to get 'c'", 3, citr.getCount());
+
+ citr.reset();
+ assertTrue(p.hasChildNode("d"));
+ assertEquals("Unexpected number reads to get 'd'", 4, citr.getCount());
+
+ citr.reset();
+ assertFalse(p.hasChildNode("y"));
+ assertEquals("Unexpected number reads to conclude that non-preferred child 'x' is missing",
+ 5, citr.getCount());
+ }
+
+ @Test
+ public void allPreferredReadable() {
+ Set<String> preferred = ImmutableSet.of("x", "y");
+ CountingIterable<NodeStateEntry> citr = createList(preferred, asList("/a", "/a/x", "/a/x/1", "/a/x/2", "/a/x/3",
+ "/a/y"));
+ ChildNodeStateProvider p = new ChildNodeStateProvider(citr, "/a", preferred);
+
+ assertTrue(p.hasChildNode("x"));
+ assertTrue(p.hasChildNode("y"));
}
@Test
public void childCount() {
Set<String> preferred = ImmutableSet.of("jcr:content", "x");
CountingIterable<NodeStateEntry> citr = createList(preferred, asList("/a", "/a/jcr:content", "/a/c", "/a/d", "/e", "/e/f"));
- ChildNodeStateProvider p = new ChildNodeStateProvider(citr, "/a", preferred.size());
+ ChildNodeStateProvider p = new ChildNodeStateProvider(citr, "/a", preferred);
assertEquals(1, p.getChildNodeCount(1));
assertEquals(3, p.getChildNodeCount(2));
}
@@ -122,7 +152,7 @@ public class ChildNodeStateProviderTest
public void childNames() {
Set<String> preferred = ImmutableSet.of("jcr:content");
CountingIterable<NodeStateEntry> citr = createList(preferred, asList("/a", "/a/jcr:content", "/a/c", "/a/d", "/e", "/e/f"));
- ChildNodeStateProvider p = new ChildNodeStateProvider(citr, "/a", 100);
+ ChildNodeStateProvider p = new ChildNodeStateProvider(citr, "/a", preferred);
assertEquals(asList("jcr:content", "c", "d"), copyOf(childNames(p.children())));
assertEquals(5, citr.getCount());
@@ -133,7 +163,7 @@ public class ChildNodeStateProviderTest
Set<String> preferred = ImmutableSet.of("jcr:content");
CountingIterable<NodeStateEntry> citr = createList(preferred, asList("/a", "/a/jcr:content", "/a/jcr:content/metadata",
"/a/c", "/a/c/status","/a/d", "/e", "/e/f"));
- ChildNodeStateProvider p = new ChildNodeStateProvider(citr, "/a", 100);
+ ChildNodeStateProvider p = new ChildNodeStateProvider(citr, "/a", preferred);
assertEquals(asList("jcr:content", "c", "d"), copyOf(childNames(p.children())));
assertEquals(7, citr.getCount());
@@ -143,7 +173,7 @@ public class ChildNodeStateProviderTest
public void childEntries() {
Set<String> preferred = ImmutableSet.of("jcr:content");
CountingIterable<NodeStateEntry> citr = createList(preferred, asList("/a", "/a/jcr:content", "/a/c", "/a/d", "/e", "/e/f"));
- ChildNodeStateProvider p = new ChildNodeStateProvider(citr, "/a", 100);
+ ChildNodeStateProvider p = new ChildNodeStateProvider(citr, "/a", preferred);
Map<String, NodeState> children = new HashMap<>();
p.getChildNodeEntries().forEach(e -> children.put(e.getName(), e.getNodeState()));
Copied: jackrabbit/oak/branches/1.8/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedListTest.java (from r1825523, jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedListTest.java)
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedListTest.java?p2=jackrabbit/oak/branches/1.8/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedListTest.java&p1=jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedListTest.java&r1=1825523&r2=1825623&rev=1825623&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedListTest.java (original)
+++ jackrabbit/oak/branches/1.8/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedListTest.java Thu Mar 1 05:01:03 2018
@@ -136,6 +136,40 @@ public class FlatFileBufferLinkedListTes
}
@Test
+ public void memUsage() {
+ assertEquals("Empty list must be estimate 0", 0, list.estimatedMemoryUsage());
+
+ list.add(new NodeStateEntry(EMPTY_NODE, "/", 20));
+ assertEquals(20, list.estimatedMemoryUsage());
+
+ list.add(new NodeStateEntry(EMPTY_NODE, "/", 30));
+ assertEquals(50, list.estimatedMemoryUsage());
+
+ list.remove();
+ assertEquals(30, list.estimatedMemoryUsage());
+ }
+
+ @Test
+ public void memLimit() {
+ list = new FlatFileBufferLinkedList(10);
+ NodeStateEntry e10Bytes = new NodeStateEntry(EMPTY_NODE, "/", 10);
+ NodeStateEntry e1Byte = new NodeStateEntry(EMPTY_NODE, "/", 1);
+
+ list.add(e10Bytes); //this should succeed
+
+ list.remove();
+ list.add(e1Byte);
+ try {
+ list.add(e10Bytes);
+ } catch (IllegalStateException ise) {
+ //ignore
+ }
+
+ assertEquals("Addition beyond mem limit shouldn't get added", 1, list.size());
+ assertEquals("Addition beyond mem limit shouldn't show up in usage", 1, list.estimatedMemoryUsage());
+ }
+
+ @Test
public void basics() {
list.add(TEST_NODE_STATE_ENTRY);
assertEquals("Adding an item should change size", 1, list.size());
Modified: jackrabbit/oak/branches/1.8/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIteratorTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIteratorTest.java?rev=1825623&r1=1825622&r2=1825623&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.8/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIteratorTest.java (original)
+++ jackrabbit/oak/branches/1.8/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIteratorTest.java Thu Mar 1 05:01:03 2018
@@ -19,14 +19,19 @@
package org.apache.jackrabbit.oak.index.indexer.document.flatfile;
+import java.util.List;
import java.util.Set;
import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry;
+import org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState;
+import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.junit.Test;
import static java.util.Arrays.asList;
+import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileStoreIterator.BUFFER_MEM_LIMIT_CONFIG_NAME;
import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.TestUtils.createList;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -40,7 +45,7 @@ public class FlatFileStoreIteratorTest {
CountingIterable<NodeStateEntry> citr = createList(preferred, asList("/a", "/a/jcr:content", "/a/jcr:content/metadata",
"/a/d", "/e"));
- FlatFileStoreIterator fitr = new FlatFileStoreIterator(citr.iterator(), preferred.size());
+ FlatFileStoreIterator fitr = new FlatFileStoreIterator(citr.iterator(), preferred);
NodeStateEntry a = fitr.next();
assertEquals("/a", a.getPath());
@@ -75,7 +80,7 @@ public class FlatFileStoreIteratorTest {
CountingIterable<NodeStateEntry> citr = createList(preferred, asList("/a", "/a/jcr:content", "/a/jcr:content/metadata",
"/a/d", "/e"));
- FlatFileStoreIterator fitr = new FlatFileStoreIterator(citr.iterator(), preferred.size());
+ FlatFileStoreIterator fitr = new FlatFileStoreIterator(citr.iterator(), preferred);
NodeStateEntry a = fitr.next();
assertEquals("/a", a.getPath());
@@ -98,4 +103,139 @@ public class FlatFileStoreIteratorTest {
}
}
+ // OAK-7284
+ @Test
+ public void comodificationException() {
+ Set<String> preferred = ImmutableSet.of("j:c");
+
+ CountingIterable<NodeStateEntry> citr = createList(preferred, asList("/a", "/a/j:c", "/a/j:c/j:c", "/a/b"));
+
+ FlatFileStoreIterator fitr = new FlatFileStoreIterator(citr.iterator(), preferred);
+
+ NodeStateEntry a = fitr.next();
+ assertEquals("/a", a.getPath());
+
+ NodeState aNS = a.getNodeState();
+
+ // fake aggregate rule like "j:c/*"
+ for (ChildNodeEntry cne : aNS.getChildNodeEntries()) {
+ NodeState childNS = cne.getNodeState();
+ // read preferred names for aggregation sub-tree nodes
+ for (String prefName : preferred) {
+ childNS.getChildNode(prefName);
+ }
+ }
+ }
+
+ // OAK-7285
+ @Test
+ public void getChildNodeLimitedByNonPreferred() {
+ // have more than 1 preferred names
+ Set<String> preferred = ImmutableSet.of("j:c", "md");
+
+ CountingIterable<NodeStateEntry> citr = createList(preferred, asList("/a", "/a/b", "/a/c"));
+
+ FlatFileStoreIterator fitr = new FlatFileStoreIterator(citr.iterator(), preferred);
+
+ NodeStateEntry a = fitr.next();
+ assertEquals("/a", a.getPath());
+
+ NodeState aNS = a.getNodeState();
+ aNS.getChildNode("j:c");
+
+ // Don't read whole tree to conclude that "j:c" doesn't exist (reading /a/b should imply that it doesn't exist)
+ assertEquals(1, fitr.getBufferSize());
+ }
+
+ @Test
+ public void bufferEstimatesMemory() {
+ List<NodeStateEntry> nseList = Lists.newArrayList(
+ new NodeStateEntry(EmptyNodeState.EMPTY_NODE, "/a", 20),
+ new NodeStateEntry(EmptyNodeState.EMPTY_NODE, "/a/b", 30)
+ );
+ FlatFileStoreIterator fitr = new FlatFileStoreIterator(nseList.iterator(), ImmutableSet.of());
+
+ NodeStateEntry entry = fitr.next();
+ NodeState entryNS = entry.getNodeState();
+ assertEquals("/a", entry.getPath());
+ assertEquals("Fetching from iterator doesn't use buffer", 0, fitr.getBufferMemoryUsage());
+
+ entryNS.getChildNode("b");
+ assertEquals(1, fitr.getBufferSize());
+ assertEquals("Reaching child from node state should estimate 30 for /a/b",
+ 30, fitr.getBufferMemoryUsage());
+ }
+
+ @Test
+ public void memUsageConfig() {
+ String configuredValue = System.clearProperty(BUFFER_MEM_LIMIT_CONFIG_NAME);
+ try {
+ NodeStateEntry root = new NodeStateEntry(EmptyNodeState.EMPTY_NODE, "/");
+ NodeStateEntry e1Byte = new NodeStateEntry(EmptyNodeState.EMPTY_NODE, "/a/b", 1);
+ NodeStateEntry e1MB = new NodeStateEntry(EmptyNodeState.EMPTY_NODE, "/a", 1 * 1024 * 1024);
+ NodeStateEntry e100MB = new NodeStateEntry(EmptyNodeState.EMPTY_NODE, "/a", 100 * 1024 * 1024);
+
+ {
+ //default configured limit
+ List<NodeStateEntry> list = Lists.newArrayList(root, e100MB, e1Byte);
+ FlatFileStoreIterator fitr = new FlatFileStoreIterator(list.iterator(), ImmutableSet.of());
+ NodeState rootNS = fitr.next().getNodeState();
+ NodeState aNS = rootNS.getChildNode("a");//default is 100MB, this should work
+ try {
+ aNS.getChildNode("b");
+ fail("Reading beyond default 100MB must fail");
+ } catch (IllegalStateException ise) {
+ //ignore
+ }
+ }
+
+ {
+ System.setProperty(BUFFER_MEM_LIMIT_CONFIG_NAME, "1");
+
+ List<NodeStateEntry> list = Lists.newArrayList(root, e1MB, e1Byte);
+ FlatFileStoreIterator fitr = new FlatFileStoreIterator(list.iterator(), ImmutableSet.of());
+ NodeState rootNS = fitr.next().getNodeState();
+ NodeState aNS = rootNS.getChildNode("a");//configured limit is 10 bytes, this should work
+ try {
+ aNS.getChildNode("b");
+ fail("Reading beyond configured 1MB must fail");
+ } catch (IllegalStateException ise) {
+ //ignore
+ }
+ }
+
+ {
+ // illegal config behaves as default
+ System.setProperty(BUFFER_MEM_LIMIT_CONFIG_NAME, "1A");
+
+ List<NodeStateEntry> list = Lists.newArrayList(root, e100MB, e1Byte);
+ FlatFileStoreIterator fitr = new FlatFileStoreIterator(list.iterator(), ImmutableSet.of());
+ NodeState rootNS = fitr.next().getNodeState();
+ NodeState aNS = rootNS.getChildNode("a");//default is 100MB, this should work
+ try {
+ aNS.getChildNode("b");
+ fail("Reading beyond default 100MB must fail");
+ } catch (IllegalStateException ise) {
+ //ignore
+ }
+ }
+
+ {
+ // negative value for unbounded buffer
+ System.setProperty(BUFFER_MEM_LIMIT_CONFIG_NAME, "-1");
+
+ List<NodeStateEntry> list = Lists.newArrayList(root, e100MB, e1Byte);
+ FlatFileStoreIterator fitr = new FlatFileStoreIterator(list.iterator(), ImmutableSet.of());
+ NodeState rootNS = fitr.next().getNodeState();
+ NodeState aNS = rootNS.getChildNode("a");
+ aNS.getChildNode("b");//configure negative value - mem usage limit should be unbounded (long_max)
+ }
+ } finally {
+ if (configuredValue == null) {
+ System.clearProperty(BUFFER_MEM_LIMIT_CONFIG_NAME);
+ } else {
+ System.setProperty(BUFFER_MEM_LIMIT_CONFIG_NAME, configuredValue);
+ }
+ }
+ }
}
\ No newline at end of file
Modified: jackrabbit/oak/branches/1.8/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryWriterTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryWriterTest.java?rev=1825623&r1=1825622&r2=1825623&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.8/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryWriterTest.java (original)
+++ jackrabbit/oak/branches/1.8/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryWriterTest.java Thu Mar 1 05:01:03 2018
@@ -28,6 +28,7 @@ import org.apache.jackrabbit.oak.spi.blo
import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore;
import org.apache.jackrabbit.oak.spi.state.EqualsDiff;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.junit.Test;
import static com.google.common.collect.ImmutableList.copyOf;
@@ -141,4 +142,30 @@ public class NodeStateEntryWriterTest {
}
+ @Test
+ public void memUsage() {
+ NodeStateEntryWriter nw = new NodeStateEntryWriter(blobStore);
+ NodeBuilder b = EMPTY_NODE.builder();
+ b.setProperty("foo", "bar");
+
+ String json1 = nw.asJson(b.getNodeState());
+
+ b.setProperty("foo1", "bar1");
+ String json2 = nw.asJson(b.getNodeState());
+
+ String line1 = nw.toString(copyOf(elements("/")), json1);
+ String line2 = nw.toString(copyOf(elements("/sub-node")), json1);
+ String line3 = nw.toString(copyOf(elements("/sub-node")), json2);
+
+ NodeStateEntryReader nr = new NodeStateEntryReader(blobStore);
+
+ long size1 = nr.read(line1).estimatedMemUsage();
+ long size2 = nr.read(line2).estimatedMemUsage();
+ long size3 = nr.read(line3).estimatedMemUsage();
+
+ assertTrue("Mem usage should be more than 0", size1 > 0);
+ assertTrue("Mem usage should increase with longer path", size2 > size1);
+ assertTrue("Mem usage should increase with bigger node state", size3 > size2);
+ }
+
}
\ No newline at end of file