You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by th...@apache.org on 2020/05/08 06:57:56 UTC
svn commit: r1877497 - in /jackrabbit/oak/trunk/oak-run/src:
main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/
main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/
test/java/org/apache/jackrabbit/oak/index/...
Author: thomasm
Date: Fri May 8 06:57:56 2020
New Revision: 1877497
URL: http://svn.apache.org/viewvc?rev=1877497&view=rev
Log:
OAK-9052 Reindexing using --doc-traversal-mode may need a lot of memory
Added:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/NodeStateEntryList.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedList.java
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedListTest.java
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedList.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/LazyChildrenNodeState.java
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedListTest.java
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIteratorTest.java
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedList.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedList.java?rev=1877497&r1=1877496&r2=1877497&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedList.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedList.java Fri May 8 06:57:56 2020
@@ -17,21 +17,20 @@
* under the License.
*/
-package org.apache.jackrabbit.oak.index.indexer.document.flatfile;
+package org.apache.jackrabbit.oak.index.indexer.document.flatfile.linkedList;
import com.google.common.base.Preconditions;
import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry;
import org.jetbrains.annotations.NotNull;
import java.util.Iterator;
-import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileBufferLinkedList.NodeIterator.iteratorFor;
/**
* Linked list implementation which supports multiple iterators. The iterator's state
* is backed by an actual node in the list. So, modification in the list show up in
* iterator (assuming thread safely/volatility) getting handled outside of the class.
*/
-public class FlatFileBufferLinkedList {
+public class FlatFileBufferLinkedList implements NodeStateEntryList {
private ListNode head = new ListNode();
private ListNode tail = head;
@@ -40,17 +39,14 @@ public class FlatFileBufferLinkedList {
private long memUsage = 0;
private final long memLimit;
- FlatFileBufferLinkedList() {
+ public FlatFileBufferLinkedList() {
this(Long.MAX_VALUE);
}
- FlatFileBufferLinkedList(long memLimit) {
+ public FlatFileBufferLinkedList(long memLimit) {
this.memLimit = memLimit;
}
- /**
- * Add {@code item} at the tail of the list
- */
public void add(@NotNull NodeStateEntry item) {
Preconditions.checkArgument(item != null, "Can't add null to the list");
long incomingSize = item.estimatedMemUsage();
@@ -65,10 +61,6 @@ public class FlatFileBufferLinkedList {
this.memUsage += incomingSize;
}
- /**
- * Remove the first item from the list
- * @return {@code NodeStateEntry} data in the removed item
- */
public NodeStateEntry remove() {
Preconditions.checkState(!isEmpty(), "Cannot remove item from empty list");
NodeStateEntry ret = head.next.data;
@@ -82,25 +74,31 @@ public class FlatFileBufferLinkedList {
return ret;
}
- /**
- * @return {@link NodeIterator} object which would iterate the whole list
- */
- public NodeIterator iterator() {
- return iteratorFor(head);
+ @Override
+ public Iterator<NodeStateEntry> iterator() {
+ return NodeIterator.iteratorFor(head);
}
+ @Override
public int size() {
return size;
}
+ @Override
public boolean isEmpty() {
return size == 0;
}
+ @Override
public long estimatedMemoryUsage() {
return memUsage;
}
+ @Override
+ public void close() {
+ // ignore
+ }
+
/**
* Represents an item in the list.
*/
@@ -141,9 +139,11 @@ public class FlatFileBufferLinkedList {
@Override
public NodeStateEntry next() {
- Preconditions.checkState(current.isValid, "Can't call next from a removed node");
+ Preconditions.checkState(hasNext(), "No next");
current = current.next;
+ Preconditions.checkState(current.isValid, "Can't call next from a removed node");
return current.data;
}
}
+
}
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java?rev=1877497&r1=1877496&r2=1877497&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java Fri May 8 06:57:56 2020
@@ -70,7 +70,7 @@ public class FlatFileNodeStoreBuilder {
logFlags();
comparator = new PathElementComparator(preferredPathElements);
entryWriter = new NodeStateEntryWriter(blobStore);
- FlatFileStore store = new FlatFileStore(createdSortedStoreFile(), new NodeStateEntryReader(blobStore),
+ FlatFileStore store = new FlatFileStore(blobStore, createdSortedStoreFile(), new NodeStateEntryReader(blobStore),
unmodifiableSet(preferredPathElements), useZip);
if (entryCount > 0) {
store.setEntryCount(entryCount);
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java?rev=1877497&r1=1877496&r2=1877497&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java Fri May 8 06:57:56 2020
@@ -29,18 +29,21 @@ import com.google.common.collect.Abstrac
import com.google.common.io.Closer;
import org.apache.commons.io.LineIterator;
import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry;
+import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileStoreUtils.createReader;
public class FlatFileStore implements Iterable<NodeStateEntry>, Closeable{
private final Closer closer = Closer.create();
+ private final BlobStore blobStore;
private final File storeFile;
private final NodeStateEntryReader entryReader;
private final Set<String> preferredPathElements;
private final boolean compressionEnabled;
private long entryCount = -1;
- public FlatFileStore(File storeFile, NodeStateEntryReader entryReader, Set<String> preferredPathElements, boolean compressionEnabled) {
+ public FlatFileStore(BlobStore blobStore, File storeFile, NodeStateEntryReader entryReader, Set<String> preferredPathElements, boolean compressionEnabled) {
+ this.blobStore = blobStore;
this.storeFile = storeFile;
this.entryReader = entryReader;
this.preferredPathElements = preferredPathElements;
@@ -57,7 +60,8 @@ public class FlatFileStore implements It
@Override
public Iterator<NodeStateEntry> iterator() {
- return new FlatFileStoreIterator(createBaseIterator(), preferredPathElements);
+ String fileName = new File(storeFile.getParent(), "linkedList").getAbsolutePath();
+ return new FlatFileStoreIterator(blobStore, fileName, createBaseIterator(), preferredPathElements);
}
private Iterator<NodeStateEntry> createBaseIterator() {
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java?rev=1877497&r1=1877496&r2=1877497&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java Fri May 8 06:57:56 2020
@@ -19,36 +19,51 @@
package org.apache.jackrabbit.oak.index.indexer.document.flatfile;
+import static com.google.common.collect.Iterators.concat;
+import static com.google.common.collect.Iterators.singletonIterator;
+
import java.util.Iterator;
import java.util.Set;
-import com.google.common.collect.AbstractIterator;
import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry;
-import org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileBufferLinkedList.NodeIterator;
+import org.apache.jackrabbit.oak.index.indexer.document.flatfile.linkedList.FlatFileBufferLinkedList;
+import org.apache.jackrabbit.oak.index.indexer.document.flatfile.linkedList.NodeStateEntryList;
+import org.apache.jackrabbit.oak.index.indexer.document.flatfile.linkedList.PersistedLinkedList;
+import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static com.google.common.collect.Iterators.concat;
-import static com.google.common.collect.Iterators.singletonIterator;
+import com.google.common.collect.AbstractIterator;
class FlatFileStoreIterator extends AbstractIterator<NodeStateEntry> implements Iterator<NodeStateEntry> {
private final Logger log = LoggerFactory.getLogger(getClass());
private final Iterator<NodeStateEntry> baseItr;
- private final FlatFileBufferLinkedList buffer;
+ private final NodeStateEntryList buffer;
private NodeStateEntry current;
private final Set<String> preferredPathElements;
private int maxBufferSize;
static final String BUFFER_MEM_LIMIT_CONFIG_NAME = "oak.indexer.memLimitInMB";
- private static final int DEFAULT_BUFFER_MEM_LIMIT_IN_MB = 100;
- public FlatFileStoreIterator(Iterator<NodeStateEntry> baseItr, Set<String> preferredPathElements) {
+ // by default, use the PersistedLinkedList
+ private static final int DEFAULT_BUFFER_MEM_LIMIT_IN_MB = 0;
+
+ public FlatFileStoreIterator(BlobStore blobStore, String fileName, Iterator<NodeStateEntry> baseItr, Set<String> preferredPathElements) {
+ this(blobStore, fileName, baseItr, preferredPathElements,
+ Integer.getInteger(BUFFER_MEM_LIMIT_CONFIG_NAME, DEFAULT_BUFFER_MEM_LIMIT_IN_MB));
+ }
+
+ public FlatFileStoreIterator(BlobStore blobStore, String fileName, Iterator<NodeStateEntry> baseItr, Set<String> preferredPathElements, int memLimitConfig) {
this.baseItr = baseItr;
this.preferredPathElements = preferredPathElements;
- int memLimitConfig = Integer.getInteger(BUFFER_MEM_LIMIT_CONFIG_NAME, DEFAULT_BUFFER_MEM_LIMIT_IN_MB);
- if (memLimitConfig < 0) {
- log.info("Setting buffer memory limit unbounded", memLimitConfig);
+ if (memLimitConfig == 0) {
+ log.info("Using a key-value store buffer: {}", fileName);
+ NodeStateEntryReader reader = new NodeStateEntryReader(blobStore);
+ NodeStateEntryWriter writer = new NodeStateEntryWriter(blobStore);
+ this.buffer = new PersistedLinkedList(fileName, writer, reader);
+ } else if (memLimitConfig < 0) {
+ log.info("Setting buffer memory limit unbounded");
this.buffer = new FlatFileBufferLinkedList();
} else {
log.info("Setting buffer memory limit to {} MBs", memLimitConfig);
@@ -83,7 +98,8 @@ class FlatFileStoreIterator extends Abst
maxBufferSize, buffer.estimatedMemoryUsage(), current.getPath());
}
if (!buffer.isEmpty()) {
- return buffer.remove();
+ NodeStateEntry e = buffer.remove();
+ return wrapIfNeeded(e);
}
if (baseItr.hasNext()) {
return wrap(baseItr.next());
@@ -102,7 +118,7 @@ class FlatFileStoreIterator extends Abst
}
private Iterator<NodeStateEntry> queueIterator() {
- NodeIterator qitr = buffer.iterator();
+ Iterator<NodeStateEntry> qitr = buffer.iterator();
return new AbstractIterator<NodeStateEntry>() {
@Override
protected NodeStateEntry computeNext() {
@@ -111,10 +127,24 @@ class FlatFileStoreIterator extends Abst
buffer.add(wrap(baseItr.next()));
}
if (qitr.hasNext()) {
- return qitr.next();
+ return wrapIfNeeded(qitr.next());
}
+ buffer.close();
return endOfData();
}
};
}
+
+ private NodeStateEntry wrapIfNeeded(NodeStateEntry e) {
+ if (buffer instanceof PersistedLinkedList) {
+ // for the PersistedLinkedList, the entries from the iterators are
+ // de-serialized and don't contain the LazyChildrenNodeState -
+ // so we need to wrap them
+ return wrap(e);
+ }
+ // if not a PersistedLinkedList, no wrapping is needed, as the in-memory linked list
+ // already contains the LazyChildrenNodeState
+ // (actually wrapping would work just fine - it's just not needed)
+ return e;
+ }
}
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/LazyChildrenNodeState.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/LazyChildrenNodeState.java?rev=1877497&r1=1877496&r2=1877497&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/LazyChildrenNodeState.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/LazyChildrenNodeState.java Fri May 8 06:57:56 2020
@@ -32,7 +32,7 @@ class LazyChildrenNodeState implements N
private final NodeState delegate;
private final ChildNodeStateProvider childProvider;
- LazyChildrenNodeState(NodeState delegate, ChildNodeStateProvider childProvider) {
+ public LazyChildrenNodeState(NodeState delegate, ChildNodeStateProvider childProvider) {
this.delegate = delegate;
this.childProvider = childProvider;
}
Added: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/NodeStateEntryList.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/NodeStateEntryList.java?rev=1877497&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/NodeStateEntryList.java (added)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/NodeStateEntryList.java Fri May 8 06:57:56 2020
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.index.indexer.document.flatfile.linkedList;
+
+import java.util.Iterator;
+
+import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry;
+import org.apache.jackrabbit.oak.index.indexer.document.flatfile.linkedList.FlatFileBufferLinkedList.NodeIterator;
+import org.jetbrains.annotations.NotNull;
+
+public interface NodeStateEntryList {
+
+ /**
+ * Add {@code item} at the tail of the list
+ */
+ public void add(@NotNull NodeStateEntry item);
+
+ /**
+ * Remove the first item from the list
+ * @return {@code NodeStateEntry} data in the removed item
+ */
+ public NodeStateEntry remove();
+
+ public long estimatedMemoryUsage();
+
+ public int size();
+
+ /**
+ * @return {@link NodeIterator} object which would iterate the whole list
+ */
+ public Iterator<NodeStateEntry> iterator();
+
+ public boolean isEmpty();
+
+ public void close();
+
+}
Added: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedList.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedList.java?rev=1877497&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedList.java (added)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedList.java Fri May 8 06:57:56 2020
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.index.indexer.document.flatfile.linkedList;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry;
+import org.apache.jackrabbit.oak.index.indexer.document.flatfile.NodeStateEntryReader;
+import org.apache.jackrabbit.oak.index.indexer.document.flatfile.NodeStateEntryWriter;
+import org.h2.mvstore.MVMap;
+import org.h2.mvstore.MVStore;
+import org.h2.mvstore.MVStoreTool;
+import org.jetbrains.annotations.NotNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * A persistent linked list that internally uses the MVStore.
+ */
+public class PersistedLinkedList implements NodeStateEntryList {
+
+ private final static Logger LOG = LoggerFactory.getLogger(PersistedLinkedList.class);
+
+ private static final String COMPACT_STORE_MILLIS_NAME = "oak.indexer.linkedList.compactMillis";
+
+ private final NodeStateEntryWriter writer;
+ private final NodeStateEntryReader reader;
+ private final String storeFileName;
+ private final int compactStoreMillis = Integer.getInteger(
+ COMPACT_STORE_MILLIS_NAME,
+ 60 * 1000);
+
+ private MVStore store;
+ private MVMap<Long, String> map;
+ private long headIndex;
+ private long tailIndex;
+ private long size;
+ private long lastLog;
+ private long lastCompact;
+
+ public PersistedLinkedList(String fileName, NodeStateEntryWriter writer, NodeStateEntryReader reader) {
+ LOG.info("Opening store " + fileName);
+ this.storeFileName = fileName;
+ File oldFile = new File(fileName);
+ if (oldFile.exists()) {
+ LOG.info("Deleting " + fileName);
+ try {
+ FileUtils.forceDelete(oldFile);
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+ openStore();
+ this.writer = writer;
+ this.reader = reader;
+ lastCompact = System.currentTimeMillis();
+ }
+
+ private void openStore() {
+ store = MVStore.open(storeFileName);
+ map = store.openMap("list");
+ }
+
+ @Override
+ public void add(@NotNull NodeStateEntry item) {
+ Preconditions.checkArgument(item != null, "Can't add null to the list");
+ String s = writer.toString(item);
+ map.put(tailIndex++, s);
+ size++;
+ long now = System.currentTimeMillis();
+ boolean compactNow = now >= lastCompact + compactStoreMillis;
+ long sizeBytes = store.getFileStore().size();
+ if (compactNow || now >= lastLog + 10000) {
+ LOG.info("Entries: " + size + " map size: " + map.sizeAsLong() + " file size: "
+ + sizeBytes + " bytes");
+ lastLog = now;
+ }
+ if (compactNow && sizeBytes > 10L * 1000 * 1000) {
+ // compact once a minute, if larger than 10 MB
+ LOG.info("Compacting...");
+ store.close();
+ MVStoreTool.compact(storeFileName, true);
+ openStore();
+ lastCompact = System.currentTimeMillis();
+ LOG.info("New size=" + store.getFileStore().size() + " bytes");
+ }
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return size == 0;
+ }
+
+ @Override
+ public Iterator<NodeStateEntry> iterator() {
+ return new NodeIterator(headIndex);
+ }
+
+ @Override
+ public NodeStateEntry remove() {
+ Preconditions.checkState(!isEmpty(), "Cannot remove item from empty list");
+ NodeStateEntry ret = get(headIndex);
+ map.remove(headIndex);
+ headIndex++;
+ size--;
+ if (size == 0) {
+ map.clear();
+ }
+ return ret;
+ }
+
+ private NodeStateEntry get(long index) {
+ String s = map.get(index);
+ return reader.read(s);
+ }
+
+ @Override
+ public int size() {
+ return (int) size;
+ }
+
+ @Override
+ public void close() {
+ store.close();
+ }
+
+ @Override
+ public long estimatedMemoryUsage() {
+ return 0;
+ }
+
+ /**
+ * A node iterator over this list.
+ */
+ class NodeIterator implements Iterator<NodeStateEntry> {
+
+ private long index;
+
+ NodeIterator(long index) {
+ this.index = index;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return index < tailIndex;
+ }
+
+ @Override
+ public NodeStateEntry next() {
+ if (index < headIndex || index >= tailIndex) {
+ throw new IllegalStateException();
+ }
+ return get(index++);
+ }
+
+ }
+
+}
Modified: jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedListTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedListTest.java?rev=1877497&r1=1877496&r2=1877497&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedListTest.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedListTest.java Fri May 8 06:57:56 2020
@@ -17,11 +17,11 @@
* under the License.
*/
-package org.apache.jackrabbit.oak.index.indexer.document.flatfile;
+package org.apache.jackrabbit.oak.index.indexer.document.flatfile.linkedList;
import com.google.common.collect.Iterators;
import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry;
-import org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileBufferLinkedList.NodeIterator;
+import org.apache.jackrabbit.oak.index.indexer.document.flatfile.linkedList.FlatFileBufferLinkedList;
import org.junit.Before;
import org.junit.Test;
@@ -31,12 +31,15 @@ import static org.junit.Assert.assertFal
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import java.io.IOException;
+import java.util.Iterator;
+
public class FlatFileBufferLinkedListTest {
- private FlatFileBufferLinkedList list = null;
- private static final NodeStateEntry TEST_NODE_STATE_ENTRY = new NodeStateEntry(EMPTY_NODE, "/");
+
+ protected NodeStateEntryList list;
@Before
- public void setup() {
+ public void setup() throws IOException {
list = new FlatFileBufferLinkedList();
}
@@ -49,7 +52,7 @@ public class FlatFileBufferLinkedListTes
//ignore
}
- list.add(TEST_NODE_STATE_ENTRY);
+ list.add(testNode("/"));
}
@Test
@@ -61,61 +64,66 @@ public class FlatFileBufferLinkedListTes
//ignore
}
- list.add(TEST_NODE_STATE_ENTRY);
- assertEquals("Should get item on removal", TEST_NODE_STATE_ENTRY, list.remove());
+ list.add(testNode("/"));
+ assertEquals("Should get item on removal", testNode("/"), list.remove());
}
@Test
public void iterator() {
assertEquals("empty list must be 0-sized", 0, Iterators.size(list.iterator()));
- list.add(TEST_NODE_STATE_ENTRY);
+ list.add(testNode("/"));
assertEquals("single entry list must be 1-sized", 1, Iterators.size(list.iterator()));
assertEquals("single entry list must be 1-sized on separate iterators",
1, Iterators.size(list.iterator()));
- list.add(TEST_NODE_STATE_ENTRY);
+ list.add(testNode("/a"));
assertEquals("2 entries in list must be 2-sized", 2, Iterators.size(list.iterator()));
assertEquals("2 entries in list must be 2-sized on separate iterators",
2, Iterators.size(list.iterator()));
- NodeIterator iter2 = list.iterator();
- NodeIterator iter1 = list.iterator();
- iter2.next();
+ Iterator<NodeStateEntry> iter2 = list.iterator();
+ Iterator<NodeStateEntry> iter1 = list.iterator();
+ assertEquals("/", iter2.next().toString());
assertEquals("2 entries in list must be 1-sized after consuming an item",
1, Iterators.size(iter2));
assertEquals("2 entries in list must be 2-sized even if some other iterator consumed an item",
2, Iterators.size(iter1));
- list.add(TEST_NODE_STATE_ENTRY);
+ list.add(testNode("/b"));
iter1 = list.iterator();
iter2 = list.iterator();
- iter1.next();//move iter to point at node being removed below
- iter2.next();iter2.next();// move iter beyond node being removed - this should remain valid
+ //move iter to point at node being removed below
+ assertEquals("/", iter1.next().toString());
+ // move iter beyond node being removed - this should remain valid
+ assertEquals("/", iter2.next().toString());
+ assertEquals("/a", iter2.next().toString());
- list.remove();
+ assertEquals("/", list.remove().toString());
+ assertEquals("/a", list.remove().toString());
try {
iter1.next();
fail("Iterator state once removed from list can't be traversed");
} catch (IllegalStateException ise) {
//ignore
}
- assertEquals(TEST_NODE_STATE_ENTRY, iter2.next());//this should work
+ //this should work
+ assertEquals(testNode("/b"), iter2.next());
assertEquals("2 entries in list must be 1-sized after removal of an iterm",
- 2, Iterators.size(list.iterator()));
+ 1, Iterators.size(list.iterator()));
}
@Test
public void size() {
assertEquals("empty list must be 0-sized", 0, list.size());
- list.add(TEST_NODE_STATE_ENTRY);
+ list.add(testNode("/"));
assertEquals("single entry list must be 1-sized", 1, list.size());
assertEquals("single entry list must be 1-sized on separate iterators", 1, list.size());
- list.add(TEST_NODE_STATE_ENTRY);
+ list.add(testNode("/"));
assertEquals("2 entries in list must be 2-sized", 2, list.size());
assertEquals("2 entries in list must be 2-sized on separate iterators", 2, list.size());
@@ -128,7 +136,7 @@ public class FlatFileBufferLinkedListTes
public void isEmpty() {
assertTrue("Empty list should be empty", list.isEmpty());
- list.add(TEST_NODE_STATE_ENTRY);
+ list.add(testNode("/"));
assertFalse("Non-empty list should be non-empty", list.isEmpty());
list.remove();
@@ -171,8 +179,12 @@ public class FlatFileBufferLinkedListTes
@Test
public void basics() {
- list.add(TEST_NODE_STATE_ENTRY);
+ list.add(testNode("/"));
assertEquals("Adding an item should change size", 1, list.size());
assertTrue("Adding an item should be available", list.iterator().hasNext());
}
+
+ private NodeStateEntry testNode(String n) {
+ return new NodeStateEntry(EMPTY_NODE, n);
+ }
}
\ No newline at end of file
Modified: jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIteratorTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIteratorTest.java?rev=1877497&r1=1877496&r2=1877497&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIteratorTest.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIteratorTest.java Fri May 8 06:57:56 2020
@@ -19,6 +19,7 @@
package org.apache.jackrabbit.oak.index.indexer.document.flatfile;
+import java.util.Iterator;
import java.util.List;
import java.util.Set;
@@ -26,6 +27,7 @@ import com.google.common.collect.Immutab
import com.google.common.collect.Lists;
import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry;
import org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState;
+import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.junit.Test;
@@ -39,13 +41,24 @@ import static org.junit.Assert.fail;
public class FlatFileStoreIteratorTest {
+ protected FlatFileStoreIterator newInMemoryFlatFileStore(
+ Iterator<NodeStateEntry> it, Set<String> set, int memMB) {
+ BlobStore blobStore = null;
+ return new FlatFileStoreIterator(blobStore, "target/test", it, set, memMB);
+ }
+
+ protected FlatFileStoreIterator newFlatFileStore(Iterator<NodeStateEntry> it, Set<String> set) {
+ BlobStore blobStore = null;
+ return new FlatFileStoreIterator(blobStore, "target/test", it, set);
+ }
+
@Test
public void simpleTraversal() {
Set<String> preferred = ImmutableSet.of("jcr:content");
CountingIterable<NodeStateEntry> citr = createList(preferred, asList("/a", "/a/jcr:content", "/a/jcr:content/metadata",
"/a/d", "/e"));
- FlatFileStoreIterator fitr = new FlatFileStoreIterator(citr.iterator(), preferred);
+ FlatFileStoreIterator fitr = newFlatFileStore(citr.iterator(), preferred);
NodeStateEntry a = fitr.next();
assertEquals("/a", a.getPath());
@@ -80,7 +93,7 @@ public class FlatFileStoreIteratorTest {
CountingIterable<NodeStateEntry> citr = createList(preferred, asList("/a", "/a/jcr:content", "/a/jcr:content/metadata",
"/a/d", "/e"));
- FlatFileStoreIterator fitr = new FlatFileStoreIterator(citr.iterator(), preferred);
+ FlatFileStoreIterator fitr = newFlatFileStore(citr.iterator(), preferred);
NodeStateEntry a = fitr.next();
assertEquals("/a", a.getPath());
@@ -110,7 +123,7 @@ public class FlatFileStoreIteratorTest {
CountingIterable<NodeStateEntry> citr = createList(preferred, asList("/a", "/a/j:c", "/a/j:c/j:c", "/a/b"));
- FlatFileStoreIterator fitr = new FlatFileStoreIterator(citr.iterator(), preferred);
+ FlatFileStoreIterator fitr = newFlatFileStore(citr.iterator(), preferred);
NodeStateEntry a = fitr.next();
assertEquals("/a", a.getPath());
@@ -135,7 +148,7 @@ public class FlatFileStoreIteratorTest {
CountingIterable<NodeStateEntry> citr = createList(preferred, asList("/a", "/a/b", "/a/c"));
- FlatFileStoreIterator fitr = new FlatFileStoreIterator(citr.iterator(), preferred);
+ FlatFileStoreIterator fitr = newFlatFileStore(citr.iterator(), preferred);
NodeStateEntry a = fitr.next();
assertEquals("/a", a.getPath());
@@ -153,7 +166,7 @@ public class FlatFileStoreIteratorTest {
new NodeStateEntry(EmptyNodeState.EMPTY_NODE, "/a", 20),
new NodeStateEntry(EmptyNodeState.EMPTY_NODE, "/a/b", 30)
);
- FlatFileStoreIterator fitr = new FlatFileStoreIterator(nseList.iterator(), ImmutableSet.of());
+ FlatFileStoreIterator fitr = newInMemoryFlatFileStore(nseList.iterator(), ImmutableSet.of(), 100);
NodeStateEntry entry = fitr.next();
NodeState entryNS = entry.getNodeState();
@@ -167,8 +180,7 @@ public class FlatFileStoreIteratorTest {
}
@Test
- public void memUsageConfig() {
- String configuredValue = System.clearProperty(BUFFER_MEM_LIMIT_CONFIG_NAME);
+ public void memUsageConfig100() {
try {
NodeStateEntry root = new NodeStateEntry(EmptyNodeState.EMPTY_NODE, "/");
NodeStateEntry e1Byte = new NodeStateEntry(EmptyNodeState.EMPTY_NODE, "/a/b", 1);
@@ -176,9 +188,10 @@ public class FlatFileStoreIteratorTest {
NodeStateEntry e100MB = new NodeStateEntry(EmptyNodeState.EMPTY_NODE, "/a", 100 * 1024 * 1024);
{
- //default configured limit
+ // 100 MB limit
+ int mb = 100;
List<NodeStateEntry> list = Lists.newArrayList(root, e100MB, e1Byte);
- FlatFileStoreIterator fitr = new FlatFileStoreIterator(list.iterator(), ImmutableSet.of());
+ FlatFileStoreIterator fitr = newInMemoryFlatFileStore(list.iterator(), ImmutableSet.of(), mb);
NodeState rootNS = fitr.next().getNodeState();
NodeState aNS = rootNS.getChildNode("a");//default is 100MB, this should work
try {
@@ -190,10 +203,11 @@ public class FlatFileStoreIteratorTest {
}
{
+ int mb = 1;
System.setProperty(BUFFER_MEM_LIMIT_CONFIG_NAME, "1");
List<NodeStateEntry> list = Lists.newArrayList(root, e1MB, e1Byte);
- FlatFileStoreIterator fitr = new FlatFileStoreIterator(list.iterator(), ImmutableSet.of());
+ FlatFileStoreIterator fitr = newInMemoryFlatFileStore(list.iterator(), ImmutableSet.of(), mb);
NodeState rootNS = fitr.next().getNodeState();
NodeState aNS = rootNS.getChildNode("a");//configured limit is 10 bytes, this should work
try {
@@ -205,37 +219,17 @@ public class FlatFileStoreIteratorTest {
}
{
- // illegal config behaves as default
- System.setProperty(BUFFER_MEM_LIMIT_CONFIG_NAME, "1A");
-
- List<NodeStateEntry> list = Lists.newArrayList(root, e100MB, e1Byte);
- FlatFileStoreIterator fitr = new FlatFileStoreIterator(list.iterator(), ImmutableSet.of());
- NodeState rootNS = fitr.next().getNodeState();
- NodeState aNS = rootNS.getChildNode("a");//default is 100MB, this should work
- try {
- aNS.getChildNode("b");
- fail("Reading beyond default 100MB must fail");
- } catch (IllegalStateException ise) {
- //ignore
- }
- }
-
- {
// negative value for unbounded buffer
- System.setProperty(BUFFER_MEM_LIMIT_CONFIG_NAME, "-1");
+ int mb = -1;
List<NodeStateEntry> list = Lists.newArrayList(root, e100MB, e1Byte);
- FlatFileStoreIterator fitr = new FlatFileStoreIterator(list.iterator(), ImmutableSet.of());
+ FlatFileStoreIterator fitr = newInMemoryFlatFileStore(list.iterator(), ImmutableSet.of(), mb);
NodeState rootNS = fitr.next().getNodeState();
NodeState aNS = rootNS.getChildNode("a");
aNS.getChildNode("b");//configure negative value - mem usage limit should be unbounded (long_max)
}
} finally {
- if (configuredValue == null) {
- System.clearProperty(BUFFER_MEM_LIMIT_CONFIG_NAME);
- } else {
- System.setProperty(BUFFER_MEM_LIMIT_CONFIG_NAME, configuredValue);
- }
+ // ignore
}
}
}
\ No newline at end of file
Added: jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedListTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedListTest.java?rev=1877497&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedListTest.java (added)
+++ jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedListTest.java Fri May 8 06:57:56 2020
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.index.indexer.document.flatfile.linkedList;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.jackrabbit.oak.index.indexer.document.flatfile.NodeStateEntryReader;
+import org.apache.jackrabbit.oak.index.indexer.document.flatfile.NodeStateEntryWriter;
+import org.apache.jackrabbit.oak.spi.blob.BlobStore;
+import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+public class PersistedLinkedListTest extends FlatFileBufferLinkedListTest {
+
+ @Rule
+ public final TemporaryFolder folder = new TemporaryFolder(new File("target"));
+
+ @Before
+ public void setup() throws IOException {
+ String fileName = folder.newFile().getAbsolutePath();
+ BlobStore blobStore = new MemoryBlobStore();
+ NodeStateEntryReader reader = new NodeStateEntryReader(blobStore);
+ NodeStateEntryWriter writer = new NodeStateEntryWriter(blobStore);
+ list = new PersistedLinkedList(fileName, writer, reader);
+ }
+
+ @After
+ public void tearDown() {
+ list.close();
+ }
+
+ @Test
+ @Override
+ public void memUsage() {
+ // estimated memory usage is alway 0 with the persisted list
+ }
+
+}