You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ca...@apache.org on 2018/03/01 04:17:43 UTC
svn commit: r1825621 - in /jackrabbit/oak/trunk/oak-run/src:
main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java
test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIteratorTest.java
Author: catholicon
Date: Thu Mar 1 04:17:43 2018
New Revision: 1825621
URL: http://svn.apache.org/viewvc?rev=1825621&view=rev
Log:
OAK-7290: Reindexing using --doc-traversal-mode should have configurable upper bound for mem usage
Wire linked list mem limit facility in FlatFileStoreIterator.
Default mem limit is 100MB.
The limit can be configure by setting jvm param 'oak.indexer.memLimitInMB'
Negative value implies unbounded usage (long_max)
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIteratorTest.java
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java?rev=1825621&r1=1825620&r2=1825621&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java Thu Mar 1 04:17:43 2018
@@ -35,14 +35,25 @@ import static com.google.common.collect.
class FlatFileStoreIterator extends AbstractIterator<NodeStateEntry> implements Iterator<NodeStateEntry> {
private final Logger log = LoggerFactory.getLogger(getClass());
private final Iterator<NodeStateEntry> baseItr;
- private final FlatFileBufferLinkedList buffer = new FlatFileBufferLinkedList();
+ private final FlatFileBufferLinkedList buffer;
private NodeStateEntry current;
private final Set<String> preferredPathElements;
private int maxBufferSize;
+ static final String BUFFER_MEM_LIMIT_CONFIG_NAME = "oak.indexer.memLimitInMB";
+ private static final int DEFAULT_BUFFER_MEM_LIMIT_IN_MB = 100;
public FlatFileStoreIterator(Iterator<NodeStateEntry> baseItr, Set<String> preferredPathElements) {
this.baseItr = baseItr;
this.preferredPathElements = preferredPathElements;
+
+ int memLimitConfig = Integer.getInteger(BUFFER_MEM_LIMIT_CONFIG_NAME, DEFAULT_BUFFER_MEM_LIMIT_IN_MB);
+ if (memLimitConfig < 0) {
+ log.info("Setting buffer memory limit unbounded", memLimitConfig);
+ this.buffer = new FlatFileBufferLinkedList();
+ } else {
+ log.info("Setting buffer memory limit to {} MBs", memLimitConfig);
+ this.buffer = new FlatFileBufferLinkedList(memLimitConfig * 1024L * 1024L);
+ }
}
int getBufferSize(){
Modified: jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIteratorTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIteratorTest.java?rev=1825621&r1=1825620&r2=1825621&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIteratorTest.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIteratorTest.java Thu Mar 1 04:17:43 2018
@@ -31,10 +31,10 @@ import org.apache.jackrabbit.oak.spi.sta
import org.junit.Test;
import static java.util.Arrays.asList;
+import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileStoreIterator.BUFFER_MEM_LIMIT_CONFIG_NAME;
import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.TestUtils.createList;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
public class FlatFileStoreIteratorTest {
@@ -158,11 +158,84 @@ public class FlatFileStoreIteratorTest {
NodeStateEntry entry = fitr.next();
NodeState entryNS = entry.getNodeState();
assertEquals("/a", entry.getPath());
- assertEquals("Fetching from iterator doesn't use buffer", 0, fitr.getBufferSize());
+ assertEquals("Fetching from iterator doesn't use buffer", 0, fitr.getBufferMemoryUsage());
entryNS.getChildNode("b");
assertEquals(1, fitr.getBufferSize());
assertEquals("Reaching child from node state should estimate 30 for /a/b",
30, fitr.getBufferMemoryUsage());
}
+
+ @Test
+ public void memUsageConfig() {
+ String configuredValue = System.clearProperty(BUFFER_MEM_LIMIT_CONFIG_NAME);
+ try {
+ NodeStateEntry root = new NodeStateEntry(EmptyNodeState.EMPTY_NODE, "/");
+ NodeStateEntry e1Byte = new NodeStateEntry(EmptyNodeState.EMPTY_NODE, "/a/b", 1);
+ NodeStateEntry e1MB = new NodeStateEntry(EmptyNodeState.EMPTY_NODE, "/a", 1 * 1024 * 1024);
+ NodeStateEntry e100MB = new NodeStateEntry(EmptyNodeState.EMPTY_NODE, "/a", 100 * 1024 * 1024);
+
+ {
+ //default configured limit
+ List<NodeStateEntry> list = Lists.newArrayList(root, e100MB, e1Byte);
+ FlatFileStoreIterator fitr = new FlatFileStoreIterator(list.iterator(), ImmutableSet.of());
+ NodeState rootNS = fitr.next().getNodeState();
+ NodeState aNS = rootNS.getChildNode("a");//default is 100MB, this should work
+ try {
+ aNS.getChildNode("b");
+ fail("Reading beyond default 100MB must fail");
+ } catch (IllegalStateException ise) {
+ //ignore
+ }
+ }
+
+ {
+ System.setProperty(BUFFER_MEM_LIMIT_CONFIG_NAME, "1");
+
+ List<NodeStateEntry> list = Lists.newArrayList(root, e1MB, e1Byte);
+ FlatFileStoreIterator fitr = new FlatFileStoreIterator(list.iterator(), ImmutableSet.of());
+ NodeState rootNS = fitr.next().getNodeState();
+ NodeState aNS = rootNS.getChildNode("a");//configured limit is 10 bytes, this should work
+ try {
+ aNS.getChildNode("b");
+ fail("Reading beyond configured 1MB must fail");
+ } catch (IllegalStateException ise) {
+ //ignore
+ }
+ }
+
+ {
+ // illegal config behaves as default
+ System.setProperty(BUFFER_MEM_LIMIT_CONFIG_NAME, "1A");
+
+ List<NodeStateEntry> list = Lists.newArrayList(root, e100MB, e1Byte);
+ FlatFileStoreIterator fitr = new FlatFileStoreIterator(list.iterator(), ImmutableSet.of());
+ NodeState rootNS = fitr.next().getNodeState();
+ NodeState aNS = rootNS.getChildNode("a");//default is 100MB, this should work
+ try {
+ aNS.getChildNode("b");
+ fail("Reading beyond default 100MB must fail");
+ } catch (IllegalStateException ise) {
+ //ignore
+ }
+ }
+
+ {
+ // negative value for unbounded buffer
+ System.setProperty(BUFFER_MEM_LIMIT_CONFIG_NAME, "-1");
+
+ List<NodeStateEntry> list = Lists.newArrayList(root, e100MB, e1Byte);
+ FlatFileStoreIterator fitr = new FlatFileStoreIterator(list.iterator(), ImmutableSet.of());
+ NodeState rootNS = fitr.next().getNodeState();
+ NodeState aNS = rootNS.getChildNode("a");
+ aNS.getChildNode("b");//configure negative value - mem usage limit should be unbounded (long_max)
+ }
+ } finally {
+ if (configuredValue == null) {
+ System.clearProperty(BUFFER_MEM_LIMIT_CONFIG_NAME);
+ } else {
+ System.setProperty(BUFFER_MEM_LIMIT_CONFIG_NAME, configuredValue);
+ }
+ }
+ }
}
\ No newline at end of file