You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by da...@apache.org on 2024/01/25 19:16:20 UTC
(jackrabbit-oak) 03/50: OAK-10438 - Remove Mongo download strategy MULTITHREADED_TRAVERSE_WITH_SORT (#1250)
This is an automated email from the ASF dual-hosted git repository.
daim pushed a commit to branch DetailedGC/OAK-10199
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit a43373554d17ae178b15a4a198b41cf69bbf9a7e
Author: Nuno Santos <ns...@adobe.com>
AuthorDate: Thu Dec 14 21:57:35 2023 +0100
OAK-10438 - Remove Mongo download strategy MULTITHREADED_TRAVERSE_WITH_SORT (#1250)
---
.../jackrabbit/oak/index/IndexerSupport.java | 2 -
.../indexer/document/DocumentStoreIndexerBase.java | 10 +-
.../document/flatfile/DefaultMemoryManager.java | 252 ---------
.../flatfile/FlatFileNodeStoreBuilder.java | 77 +--
.../indexer/document/flatfile/MemoryManager.java | 76 ---
.../document/flatfile/MemoryManagerClient.java | 39 --
.../indexer/document/flatfile/MergeRunner.java | 358 -------------
.../MultithreadedTraverseWithSortStrategy.java | 562 ---------------------
.../document/flatfile/TraverseAndSortTask.java | 352 -------------
.../oak/indexversion/PurgeOldIndexVersion.java | 1 -
.../oak/run/cli/BlobStoreFixtureProvider.java | 1 -
.../flatfile/FlatFileNodeStoreBuilderTest.java | 21 -
.../document/flatfile/FlatFileStoreTest.java | 504 +-----------------
.../indexer/document/flatfile/MergeRunnerTest.java | 301 -----------
.../MultithreadedTraverseWithSortStrategyTest.java | 128 -----
.../document/flatfile/TraverseAndSortTaskTest.java | 92 ----
16 files changed, 11 insertions(+), 2765 deletions(-)
diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/IndexerSupport.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/IndexerSupport.java
index 4c8f213ea6..51446a9d38 100644
--- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/IndexerSupport.java
+++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/IndexerSupport.java
@@ -32,8 +32,6 @@ import org.apache.felix.inventory.Format;
import org.apache.jackrabbit.guava.common.base.Predicate;
import org.apache.jackrabbit.oak.api.CommitFailedException;
import org.apache.jackrabbit.oak.commons.PathUtils;
-import org.apache.jackrabbit.oak.index.indexer.document.flatfile.NodeStateEntryWriter;
-import org.apache.jackrabbit.oak.index.indexer.document.flatfile.NodeStateHolder;
import org.apache.jackrabbit.oak.plugins.index.IndexConstants;
import org.apache.jackrabbit.oak.plugins.index.importer.AsyncLaneSwitcher;
import org.apache.jackrabbit.oak.plugins.index.importer.IndexDefinitionUpdater;
diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java
index d180252f6b..8d7c68c6a2 100644
--- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java
+++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java
@@ -27,17 +27,13 @@ import org.apache.jackrabbit.oak.api.CommitFailedException;
import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser;
import org.apache.jackrabbit.oak.index.IndexHelper;
import org.apache.jackrabbit.oak.index.IndexerSupport;
-import org.apache.jackrabbit.oak.index.indexer.document.flatfile.DefaultMemoryManager;
import org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileNodeStoreBuilder;
import org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileStore;
-import org.apache.jackrabbit.oak.index.indexer.document.flatfile.MemoryManager;
import org.apache.jackrabbit.oak.index.indexer.document.incrementalstore.IncrementalStoreBuilder;
import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStore;
-import org.apache.jackrabbit.oak.plugins.document.Collection;
import org.apache.jackrabbit.oak.plugins.document.DocumentNodeState;
import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore;
import org.apache.jackrabbit.oak.plugins.document.RevisionVector;
-import org.apache.jackrabbit.oak.plugins.document.mongo.DocumentStoreSplitter;
import org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentStore;
import org.apache.jackrabbit.oak.plugins.document.mongo.TraversingRange;
import org.apache.jackrabbit.oak.plugins.document.util.MongoConnection;
@@ -160,15 +156,11 @@ public abstract class DocumentStoreIndexerBase implements Closeable {
DocumentNodeState rootDocumentState = (DocumentNodeState) checkpointedState;
DocumentNodeStore nodeStore = (DocumentNodeStore) indexHelper.getNodeStore();
- DocumentStoreSplitter splitter = new DocumentStoreSplitter(getMongoDocumentStore());
- List<Long> lastModifiedBreakPoints = splitter.split(Collection.NODES, 0L, 10);
FlatFileNodeStoreBuilder builder = null;
int backOffTimeInMillis = 5000;
- MemoryManager memoryManager = new DefaultMemoryManager();
while (storeList.isEmpty() && executionCount <= MAX_DOWNLOAD_ATTEMPTS) {
try {
- builder = new FlatFileNodeStoreBuilder(indexHelper.getWorkDir(), memoryManager)
- .withLastModifiedBreakPoints(lastModifiedBreakPoints)
+ builder = new FlatFileNodeStoreBuilder(indexHelper.getWorkDir())
.withBlobStore(indexHelper.getGCBlobStore())
.withPreferredPathElements((preferredPathElements != null) ? preferredPathElements : indexer.getRelativeIndexedNodeNames())
.addExistingDataDumpDir(indexerSupport.getExistingDataDumpDir())
diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/DefaultMemoryManager.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/DefaultMemoryManager.java
deleted file mode 100644
index 5dc3fe54f0..0000000000
--- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/DefaultMemoryManager.java
+++ /dev/null
@@ -1,252 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.jackrabbit.oak.index.indexer.document.flatfile;
-
-import org.apache.commons.codec.binary.Base64;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.management.Notification;
-import javax.management.NotificationEmitter;
-import javax.management.NotificationListener;
-import javax.management.openmbean.CompositeData;
-import java.lang.management.MemoryNotificationInfo;
-import java.lang.management.MemoryPoolMXBean;
-import java.lang.management.MemoryUsage;
-import java.util.Optional;
-import java.util.Random;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.Phaser;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicLong;
-
-import static java.lang.management.ManagementFactory.getMemoryMXBean;
-import static java.lang.management.ManagementFactory.getMemoryPoolMXBeans;
-import static java.lang.management.MemoryType.HEAP;
-import static org.apache.commons.io.FileUtils.ONE_GB;
-import static org.apache.jackrabbit.oak.commons.IOUtils.humanReadableByteCount;
-import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileNodeStoreBuilder.OAK_INDEXER_MAX_SORT_MEMORY_IN_GB;
-import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileNodeStoreBuilder.OAK_INDEXER_MAX_SORT_MEMORY_IN_GB_DEFAULT;
-
-public class DefaultMemoryManager implements MemoryManager {
-
- private static final String OAK_INDEXER_MIN_MEMORY = "oak.indexer.minMemoryForWork";
-
- private final Logger log = LoggerFactory.getLogger(getClass());
- private final AtomicBoolean sufficientMemory = new AtomicBoolean(true);
-
- /**
- * Max memory to be used if jmx based memory monitoring is not available. This value is not considered if jmx based
- * monitoring is available.
- */
- private final long maxMemoryBytes;
- /**
- * When jmx based memory monitoring is available, this value indicates minimum memory which should be free/available for this
- * task to proceed.
- */
- private final long minMemoryBytes;
- private final AtomicLong memoryUsed;
- private final MemoryPoolMXBean pool;
- private final ConcurrentHashMap<String, MemoryManagerClient> clients;
- private final MemoryManager.Type type;
- private final Random random;
-
- public DefaultMemoryManager() {
- this(Integer.getInteger(OAK_INDEXER_MIN_MEMORY, 2) * ONE_GB,
- Integer.getInteger(OAK_INDEXER_MAX_SORT_MEMORY_IN_GB, OAK_INDEXER_MAX_SORT_MEMORY_IN_GB_DEFAULT) * ONE_GB);
- }
-
- public DefaultMemoryManager(long minMemoryInBytes, long maxMemoryInBytes) {
- pool = getMemoryPool();
- memoryUsed = new AtomicLong(0);
- clients = new ConcurrentHashMap<>();
- random = ThreadLocalRandom.current();
- if (pool == null) {
- maxMemoryBytes = maxMemoryInBytes;
- minMemoryBytes = 0;
- type = Type.SELF_MANAGED;
- log.warn("Unable to setup monitoring of available memory. " +
- "Would use configured maxMemory limit of {} GB", maxMemoryInBytes/ONE_GB);
- } else {
- maxMemoryBytes = 0;
- MemoryUsage usage = pool.getCollectionUsage();
- long maxAvailable = usage.getMax();
- minMemoryBytes = minMemoryInBytes < maxAvailable ? minMemoryInBytes : (long)(0.5*maxAvailable);
- if (minMemoryBytes != minMemoryInBytes) {
- log.warn("Provided minimum memory {} GB more than available memory ({})." +
- "Overriding configuration and setting min memory to 50% of available memory ({}).",
- minMemoryInBytes/ONE_GB, humanReadableByteCount(maxAvailable), humanReadableByteCount(minMemoryBytes));
- }
- type = Type.JMX_BASED;
- configureMemoryListener();
- }
- logFlags();
- }
-
- private void configureMemoryListener() {
- NotificationEmitter emitter = (NotificationEmitter) getMemoryMXBean();
- MemoryListener listener = new MemoryListener();
- emitter.addNotificationListener(listener, null, null);
- MemoryUsage usage = pool.getCollectionUsage();
- long maxMemory = usage.getMax();
- log.info("Setting up a listener to monitor pool '{}' and trigger batch save " +
- "if memory drop below {} GB (max {})", pool.getName(), minMemoryBytes/ONE_GB, humanReadableByteCount(maxMemory));
- pool.setCollectionUsageThreshold(minMemoryBytes);
- }
-
- @Override
- public Type getType() {
- return type;
- }
-
- @Override
- public boolean isMemoryLow() {
- if (type == Type.SELF_MANAGED) {
- return memoryUsed.get() > maxMemoryBytes;
- }
- return !sufficientMemory.get();
- }
-
- @Override
- public void changeMemoryUsedBy(long memory) {
- if (type != Type.SELF_MANAGED) {
- throw new UnsupportedOperationException("Not a self managed memory manager");
- }
- memoryUsed.addAndGet(memory);
- }
-
- @Override
- public Optional<String> registerClient(MemoryManagerClient client) {
- if (type != Type.JMX_BASED) {
- throw new UnsupportedOperationException("Not a JMX based memory manager");
- }
- if (!sufficientMemory.get()) {
- log.info("Can't register new client now. Not enough memory.");
- return Optional.empty();
- }
- int retryCount = 0;
- String registrationID = generateRegistrationID();
- while (retryCount < 5) {
- MemoryManagerClient oldClient = clients.putIfAbsent(registrationID, client);
- if (oldClient == null) {
- log.debug("Registered client with registration_id={}", registrationID);
- return Optional.of(registrationID);
- }
- retryCount++;
- registrationID = generateRegistrationID();
- }
- return Optional.empty();
- }
-
- @Override
- public boolean deregisterClient(String registrationID) {
- if (type != Type.JMX_BASED) {
- throw new UnsupportedOperationException("Not a JMX based memory manager");
- }
- boolean removed;
- if (clients.remove(registrationID) != null) {
- log.debug("Client with registration_id={} deregistered", registrationID);
- removed = true;
- } else {
- log.warn("No client found with registration_id={}", registrationID);
- removed = false;
- }
- return removed;
- }
-
- private String generateRegistrationID() {
- byte[] r = new byte[8];
- random.nextBytes(r);
- return Base64.encodeBase64String(r) + "-" + System.currentTimeMillis();
- }
-
- private long getAvailableMemory(MemoryUsage usage) {
- return usage.getMax() - usage.getUsed();
- }
-
- private void checkMemory(MemoryUsage usage) {
- long avail = getAvailableMemory(usage);
- if (avail > minMemoryBytes) {
- sufficientMemory.set(true);
- log.info("Available memory level {} is good.", humanReadableByteCount(avail));
- } else {
- boolean couldSet = sufficientMemory.compareAndSet(true, false);
- if (couldSet) {
- Phaser phaser = new Phaser();
- clients.forEach((r, c) -> c.memoryLow(phaser));
- log.info("Available memory level {} (required {}) is low. Enabling flag to trigger batch save",
- humanReadableByteCount(avail), minMemoryBytes / ONE_GB);
- new Thread(() -> {
- log.info("Waiting for all tasks to finish dumping their data");
- phaser.awaitAdvance(phaser.getPhase());
- log.info("All tasks have finished dumping their data");
- sufficientMemory.set(true);
- }, "Wait-For-Dump").start();
- }
- }
- }
-
- //Taken from GCMemoryBarrier
- private class MemoryListener implements NotificationListener {
- @Override
- public void handleNotification(Notification notification,
- Object handback) {
- if (notification
- .getType()
- .equals(MemoryNotificationInfo.MEMORY_COLLECTION_THRESHOLD_EXCEEDED)) {
- if (sufficientMemory.get()) {
- CompositeData cd = (CompositeData) notification
- .getUserData();
- MemoryNotificationInfo info = MemoryNotificationInfo
- .from(cd);
- checkMemory(info.getUsage());
- }
- }
- }
- }
-
- private static MemoryPoolMXBean getMemoryPool() {
- long maxSize = 0;
- MemoryPoolMXBean maxPool = null;
- for (MemoryPoolMXBean pool : getMemoryPoolMXBeans()) {
- if (HEAP == pool.getType()
- && pool.isCollectionUsageThresholdSupported()) {
- // Get usage after a GC, which is more stable, if available
- long poolSize = pool.getCollectionUsage().getMax();
- // Keep the pool with biggest size, by default it should be Old Gen Space
- if (poolSize > maxSize) {
- maxPool = pool;
- }
- }
- }
- return maxPool;
- }
-
- private void logFlags() {
- if (type.equals(Type.JMX_BASED)) {
- log.info("Min heap memory (GB) to be required : {} ({})", minMemoryBytes / ONE_GB, OAK_INDEXER_MIN_MEMORY);
- }
- if (type.equals(Type.SELF_MANAGED)) {
- log.info("Max heap memory (GB) to be used for merge sort : {} ({})", maxMemoryBytes / ONE_GB, OAK_INDEXER_MAX_SORT_MEMORY_IN_GB);
- }
- }
-
-
-}
diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java
index fbe4d1b525..0d31b99eb6 100644
--- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java
+++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java
@@ -20,7 +20,6 @@
package org.apache.jackrabbit.oak.index.indexer.document.flatfile;
import com.mongodb.client.MongoDatabase;
-import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.jackrabbit.guava.common.collect.Iterables;
import org.apache.jackrabbit.oak.commons.Compression;
@@ -68,14 +67,7 @@ public class FlatFileNodeStoreBuilder {
private static final String FLAT_FILE_STORE_DIR_NAME_PREFIX = "flat-fs-";
/**
- * System property name for sort strategy. If this is true, we use {@link MultithreadedTraverseWithSortStrategy}, else
- * {@link StoreAndSortStrategy} strategy is used.
- * NOTE - System property {@link #OAK_INDEXER_SORT_STRATEGY_TYPE} takes precedence over this one.
- */
- public static final String OAK_INDEXER_TRAVERSE_WITH_SORT = "oak.indexer.traverseWithSortStrategy";
- /**
- * System property name for sort strategy. This takes precedence over {@link #OAK_INDEXER_TRAVERSE_WITH_SORT}.
- * Allowed values are the values from enum {@link SortStrategyType}
+ * System property name for sort strategy. Allowed values are the values from enum {@link SortStrategyType}
*/
public static final String OAK_INDEXER_SORT_STRATEGY_TYPE = "oak.indexer.sortStrategyType";
/**
@@ -83,42 +75,12 @@ public class FlatFileNodeStoreBuilder {
*/
public static final String OAK_INDEXER_SORTED_FILE_PATH = "oak.indexer.sortedFilePath";
- /**
- * Default value for {@link #PROP_THREAD_POOL_SIZE}
- */
- static final int DEFAULT_NUMBER_OF_DATA_DUMP_THREADS = 8;
- /**
- * System property for specifying number of threads for parallel download when using {@link MultithreadedTraverseWithSortStrategy}
- */
- static final String PROP_THREAD_POOL_SIZE = "oak.indexer.dataDumpThreadPoolSize";
-
- /**
- * Default value for {@link #PROP_MERGE_THREAD_POOL_SIZE}
- */
- static final int DEFAULT_NUMBER_OF_MERGE_TASK_THREADS = 1;
- /**
- * System property for specifying number of threads for parallel merge when using {@link MultithreadedTraverseWithSortStrategy}
- */
- static final String PROP_MERGE_THREAD_POOL_SIZE = "oak.indexer.mergeTaskThreadPoolSize";
-
- /**
- * Default value for {@link #PROP_MERGE_TASK_BATCH_SIZE}
- */
- static final int DEFAULT_NUMBER_OF_FILES_PER_MERGE_TASK = 64;
- /**
- * System property for specifying number of files for batch merge task when using {@link MultithreadedTraverseWithSortStrategy}
- */
- static final String PROP_MERGE_TASK_BATCH_SIZE = "oak.indexer.mergeTaskBatchSize";
-
/**
* Value of this system property indicates max memory that should be used if jmx based memory monitoring is not available.
*/
public static final String OAK_INDEXER_MAX_SORT_MEMORY_IN_GB = "oak.indexer.maxSortMemoryInGB";
public static final int OAK_INDEXER_MAX_SORT_MEMORY_IN_GB_DEFAULT = 2;
- static final String OAK_INDEXER_DUMP_THRESHOLD_IN_MB = "oak.indexer.dumpThresholdInMB";
- static final int OAK_INDEXER_DUMP_THRESHOLD_IN_MB_DEFAULT = 16;
private final Logger log = LoggerFactory.getLogger(getClass());
- private List<Long> lastModifiedBreakPoints;
private final File workDir;
private final List<File> existingDataDumpDirs = new ArrayList<>();
private Set<String> preferredPathElements = Collections.emptySet();
@@ -127,15 +89,12 @@ public class FlatFileNodeStoreBuilder {
private NodeStateEntryTraverserFactory nodeStateEntryTraverserFactory;
private long entryCount = 0;
private File flatFileStoreDir;
- private final MemoryManager memoryManager;
- private long dumpThreshold = Integer.getInteger(OAK_INDEXER_DUMP_THRESHOLD_IN_MB, OAK_INDEXER_DUMP_THRESHOLD_IN_MB_DEFAULT) * FileUtils.ONE_MB;
private Predicate<String> pathPredicate = path -> true;
private final Compression algorithm = IndexStoreUtils.compressionAlgorithm();
- private final boolean useTraverseWithSort = Boolean.parseBoolean(System.getProperty(OAK_INDEXER_TRAVERSE_WITH_SORT, "false"));
private final String sortStrategyTypeString = System.getProperty(OAK_INDEXER_SORT_STRATEGY_TYPE);
- private final SortStrategyType sortStrategyType = sortStrategyTypeString != null ? SortStrategyType.valueOf(sortStrategyTypeString) :
- (useTraverseWithSort ? SortStrategyType.TRAVERSE_WITH_SORT : SortStrategyType.PIPELINED);
+ private final SortStrategyType sortStrategyType = sortStrategyTypeString != null ?
+ SortStrategyType.valueOf(sortStrategyTypeString) : SortStrategyType.PIPELINED;
private RevisionVector rootRevision = null;
private DocumentNodeStore nodeStore = null;
private MongoDocumentStore mongoDocumentStore = null;
@@ -153,30 +112,14 @@ public class FlatFileNodeStoreBuilder {
* System property {@link #OAK_INDEXER_SORT_STRATEGY_TYPE} if set to this value would result in {@link TraverseWithSortStrategy} being used.
*/
TRAVERSE_WITH_SORT,
- /**
- * System property {@link #OAK_INDEXER_SORT_STRATEGY_TYPE} if set to this value would result in {@link MultithreadedTraverseWithSortStrategy} being used.
- */
- MULTITHREADED_TRAVERSE_WITH_SORT,
-
/**
* System property {@link #OAK_INDEXER_SORT_STRATEGY_TYPE} if set to this value would result in {@link PipelinedStrategy} being used.
*/
PIPELINED
}
- public FlatFileNodeStoreBuilder(File workDir, MemoryManager memoryManager) {
- this.workDir = workDir;
- this.memoryManager = memoryManager;
- }
-
public FlatFileNodeStoreBuilder(File workDir) {
this.workDir = workDir;
- this.memoryManager = new DefaultMemoryManager();
- }
-
- public FlatFileNodeStoreBuilder withLastModifiedBreakPoints(List<Long> lastModifiedBreakPoints) {
- this.lastModifiedBreakPoints = lastModifiedBreakPoints;
- return this;
}
public FlatFileNodeStoreBuilder withBlobStore(BlobStore blobStore) {
@@ -184,11 +127,6 @@ public class FlatFileNodeStoreBuilder {
return this;
}
- public FlatFileNodeStoreBuilder withDumpThreshold(long dumpThreshold) {
- this.dumpThreshold = dumpThreshold;
- return this;
- }
-
public FlatFileNodeStoreBuilder withPreferredPathElements(Set<String> preferredPathElements) {
this.preferredPathElements = preferredPathElements;
return this;
@@ -360,7 +298,7 @@ public class FlatFileNodeStoreBuilder {
return null;
}
- IndexStoreSortStrategy createSortStrategy(File dir) throws IOException {
+ IndexStoreSortStrategy createSortStrategy(File dir) {
switch (sortStrategyType) {
case STORE_AND_SORT:
log.info("Using StoreAndSortStrategy.");
@@ -372,11 +310,6 @@ public class FlatFileNodeStoreBuilder {
log.warn("TraverseWithSortStrategy is deprecated and will be removed in the near future. Use PipelinedStrategy instead.");
return new TraverseWithSortStrategy(nodeStateEntryTraverserFactory, preferredPathElements, entryWriter, dir,
algorithm, pathPredicate, checkpoint);
- case MULTITHREADED_TRAVERSE_WITH_SORT:
- log.info("Using MultithreadedTraverseWithSortStrategy");
- log.warn("MultithreadedTraverseWithSortStrategy is deprecated and will be removed in the near future. Use PipelinedStrategy instead.");
- return new MultithreadedTraverseWithSortStrategy(nodeStateEntryTraverserFactory, lastModifiedBreakPoints, preferredPathElements,
- blobStore, dir, existingDataDumpDirs, algorithm, memoryManager, dumpThreshold, pathPredicate, checkpoint);
case PIPELINED:
log.info("Using PipelinedStrategy");
List<PathFilter> pathFilters = indexDefinitions.stream().map(IndexDefinition::getPathFilter).collect(Collectors.toList());
@@ -391,7 +324,7 @@ public class FlatFileNodeStoreBuilder {
log.info("Preferred path elements are {}", Iterables.toString(preferredPathElements));
log.info("Compression enabled while sorting : {} ({})", IndexStoreUtils.compressionEnabled(), OAK_INDEXER_USE_ZIP);
log.info("LZ4 enabled for compression algorithm : {} ({})", IndexStoreUtils.useLZ4(), OAK_INDEXER_USE_LZ4);
- log.info("Sort strategy : {} ({})", sortStrategyType, OAK_INDEXER_TRAVERSE_WITH_SORT);
+ log.info("Sort strategy : {} ({})", sortStrategyType, OAK_INDEXER_SORT_STRATEGY_TYPE);
}
File createStoreDir() throws IOException {
diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MemoryManager.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MemoryManager.java
deleted file mode 100644
index 8bd153bdf5..0000000000
--- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MemoryManager.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.jackrabbit.oak.index.indexer.document.flatfile;
-
-import java.util.Optional;
-
-/**
- * Implementations of this interface can be used to get updates about remaining heap memory.
- */
-public interface MemoryManager {
- /**
- * Indicates type of memory management this instance does.
- */
- enum Type {
- /**
- * Uses JMX memory management mbeans for tracking available memory.
- */
- JMX_BASED,
- /**
- * Maintains memory usage itself. Different users can share this instance and indicate the increase/decrease in
- * their memory usage by calling {@link #changeMemoryUsedBy(long)}, and check if the total memory used by all
- * users of this calls causes the available memory to become low using {@link #isMemoryLow()}
- */
- SELF_MANAGED
- }
-
- Type getType();
-
- /**
- * Register a client with this memory manager. All registered clients are informed when available memory level is low.
- * If memory level is already low, client is not registered.
- * NOTE - this method should only be used with {@link Type#JMX_BASED} instance types, otherwise it throws {@link UnsupportedOperationException}
- * @param client client to register
- * @return an optional containing registration id if registration was successful, empty optional otherwise
- */
- Optional<String> registerClient(MemoryManagerClient client);
-
- /**
- * Deregister a client with the given registrationID.
- * NOTE - this method should only be used with {@link Type#JMX_BASED} instance types, otherwise it throws {@link UnsupportedOperationException}
- * @param registrationID registration id of client to deregister
- * @return true if deregistration was successful, false otherwise.
- */
- boolean deregisterClient(String registrationID);
-
- /**
- * Checks if available memory is low.
- * NOTE - this method should only be used with {@link Type#SELF_MANAGED} instance types, otherwise it throws {@link UnsupportedOperationException}
- * @return true if available memory is low, false otherwise.
- */
- boolean isMemoryLow();
-
- /**
- * Adds the provided memory value to existing memory usage. Callers of this method could also provide negative values
- * to indicate reduction in memory usage.
- * NOTE - this method should only be used with {@link Type#SELF_MANAGED} instance types, otherwise it throws {@link UnsupportedOperationException}
- */
- void changeMemoryUsedBy(long memory);
-
-}
diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MemoryManagerClient.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MemoryManagerClient.java
deleted file mode 100644
index b7c19dc347..0000000000
--- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MemoryManagerClient.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.jackrabbit.oak.index.indexer.document.flatfile;
-
-import java.util.concurrent.Phaser;
-
-/**
- * Client interface for {@link MemoryManager}.
- */
-@FunctionalInterface
-public interface MemoryManagerClient {
-
- /**
- * This method is invoked by {@link MemoryManager} which this client registers to, when the available heap memory is low.
- * It is expected that the client performs some clean up action to help with dealing with this shortage of memory. Client needs to
- * signal the {@link MemoryManager} when that action completes. The passed phaser needs to be used for that purpose.
- * Client should register with this phaser before performing the clean up and arrive and deregister itself after clean up is done.
- * If no cleanup is possible/required, just register and arrive and deregister immediately.
- * @param phaser phaser used to coordinate with {@link MemoryManager}
- */
- void memoryLow(Phaser phaser);
-
-}
diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MergeRunner.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MergeRunner.java
deleted file mode 100644
index f24687d4e2..0000000000
--- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MergeRunner.java
+++ /dev/null
@@ -1,358 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.jackrabbit.oak.index.indexer.document.flatfile;
-
-import org.apache.jackrabbit.guava.common.collect.Lists;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.comparator.SizeFileComparator;
-import org.apache.jackrabbit.oak.commons.Compression;
-import org.apache.jackrabbit.oak.commons.sort.ExternalSort;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedWriter;
-import java.io.EOFException;
-import java.io.File;
-import java.io.IOException;
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.List;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.Phaser;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.function.Function;
-
-import static org.apache.jackrabbit.guava.common.base.Charsets.UTF_8;
-import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileStoreUtils.createWriter;
-
-
-/**
- * Class responsible for -
- * <ol>
- * <li>Watching {@link #sortedFiles} for new sorted files</li>
- * <li>Submitting those files in batch to an {@link ExecutorService}</li>
- * <li>Collecting the results (sorted files) created by those tasks</li>
- * <li>Merge the result with any left over files to create a single sorted file</li>
- * </ol>
- * Strategy -
- * <ol>
- * <li>Wait for n files
- * <li>construct new list of files to be merged by checking if its already merged
- * and create intermediate merge file
- * (if final merge) merge all intermediate merge files and create sorted file
- * <li>add all merged files to merged list
- * </ol>
- *
- * <h3>Merge task explanation -</h3>
- *
- * SORTED_FILE_QUEUE=MultithreadedTraverseWithSortStrategy#sortedFiles
- * MERGED_FILE_LIST=MergeRunner#mergedFiles
- * UNMERGED_FILE_LIST=MergeRunner#unmergedFiles
- * <ol>
- * <li>Have a BlockingQueue of sorted files (SORTED_FILE_QUEUE) that need to be executed for merge. Each of the task has been assigned a list of files.</li>
- * <li>Task thread (TraverseAndSortTask) on completion adds sorted files to this queue</li>
- * <li>Another monitoring thread (MultithreadedTraverseWithSortStrategy#MergeRunner) is consuming from this SORTED_FILE_QUEUE and submitting those
- * part of the files in batch (batch file size is configurable via java system property {@link FlatFileNodeStoreBuilder#PROP_MERGE_TASK_BATCH_SIZE}
- * to executor service for merge
- * <ol>
- * <li>The monitoring thread pulls any sorted file and add it in SORTED_FILE_QUEUE to the UNMERGED_FILE_LIST</li>
- * <li>When UNMERGED_FILE_LIST grows larger than two times the batch merge size, a merge task is submitted for merge
- * with the smaller half portion of the UNMERGED_FILE_LIST</li>
- * <li>Files submitted for merge will be removed from UNMERGED_FILE_LIST and added to MERGED_FILE_LIST</li>
- * </ol>
- * </li>
- * <li>A poison pill is added to SORTED_FILE_QUEUE upon download completion</li>
- * <li>Once poison pill occurs, the monitoring thread stops submitting new merge task and proceed to final merging
- * <ol>
- * <li>Final merge waits for all existing tasks finish</li>
- * <li>All files left in UNMERGED_FILE_LIST and all previously task results are collected to be merged</li>
- * </ol>
- * </li>
- * <li>
- * We use a phaser (Merge#mergePhaser) for coordination between main thread and the monitoring thread. This phaser has one phase -
- * <ol>
- * <li>Waiting for a single final merged file to be created</li>
- * </ol>
- * </li>
- * <li>
- * We use another phaser for coordination between monitoring thread (MergeRunner) and the merge task executor (MergeTask). This phaser has one phase -
- * <ol>
- * <li>Waiting for all merge tasks complete</li>
- * </ol>
- * </li>
- * </ol>
- *
- * <h3>Force Stop Explanation -</h3>
- * <ol>
- * <li>On receiving MERGE_FORCE_STOP_POISON_PILL in SORTED_FILE_QUEUE from parent thread, all un-started tasks will be skipped.
- * <li>No more tasks will be created on MERGE_FORCE_STOP_POISON_PILL message is received.
- * <li>Running merge will continue to finish.
- * <li>Final merge will not be performed.
- * <li>This will not result in lose of data under conditions that -
- * <ol>
- * <li>Files will not be removed until merge task is completed. Parent thread should add unmerged files to the SORTED_FILE_QUEUE on retry.
- * <li>Merged files will still be under the merge folder. Parent thread should add those files to the SORTED_FILE_QUEUE on retry.
- * </ol>
- * </li>
- * </ol>
- */
-public class MergeRunner implements Runnable {
- private static final Logger log = LoggerFactory.getLogger(MergeRunner.class);
- private final Charset charset = UTF_8;
- private final Compression algorithm;
- private final ArrayList<File> mergedFiles = Lists.newArrayList();
- private final ArrayList<File> unmergedFiles = Lists.newArrayList();
- private ExecutorService executorService;
- private final int threadPoolSize;
- private final int batchMergeSize;
- private final Comparator<? super File> fileSizeComparator = new SizeFileComparator();
-
- /**
- * The end result file after merging all sorted files.
- */
- private final File sortedFile;
-
- /**
- * Directory where intermediate merged files will be created.
- */
- private final File mergeDir;
-
- /**
- * Comparator used for comparing node states for creating sorted files.
- */
- private final Comparator<NodeStateHolder> comparator;
- private final BlockingQueue<File> sortedFiles;
- private final ConcurrentLinkedQueue<Throwable> throwables;
-
- /**
- * Phaser used for coordination with the traverse/download and sort tasks. Advance of this phaser indicates that a single
- * merged and sorted file has been created.
- */
- private final Phaser phaser;
-
- /**
- * This poison pill is added to {@link #sortedFiles} to indicate that download phase has completed successfully and
- * merge should advance to the final merge stage.
- */
- public static final File MERGE_POISON_PILL = new File("");
-
- /**
- * This poison pill is added to {@link #sortedFiles} to indicate that download phase has failed and
- * merge should shut down immediately and not advance to the final merge stage.
- */
- public static final File MERGE_FORCE_STOP_POISON_PILL = new File("merge-force-stop-poison-pill");
- private final AtomicBoolean mergeCancelled;
-
- /**
- * Constructor.
- * @param sortedFiles thread safe list containing files to be merged.
- * @param mergeDir directory where sorted files will be created.
- * @param comparator comparator used to help with sorting of node state entries.
- * @param algorithm string representation of the compression algorithm, use "none" for disable compression.
- */
- MergeRunner(File sortedFile, BlockingQueue<File> sortedFiles, File mergeDir, Comparator<NodeStateHolder> comparator,
- Phaser phaser, int batchMergeSize, int threadPoolSize, Compression algorithm) {
- this.mergeDir = mergeDir;
- this.algorithm = algorithm;
- this.sortedFiles = sortedFiles;
- this.sortedFile = sortedFile;
- this.throwables = new ConcurrentLinkedQueue<>();
- this.comparator = comparator;
- this.phaser = phaser;
- this.batchMergeSize = batchMergeSize;
- this.threadPoolSize = threadPoolSize;
- this.mergeCancelled = new AtomicBoolean(false);
- }
-
- private boolean merge(List<File> files, File outputFile) {
- log.debug("performing merge for {} with size {} {}", outputFile.getName(), files.size(), files);
- try (BufferedWriter writer = createWriter(outputFile, algorithm)) {
- Function<String, NodeStateHolder> func1 = (line) -> line == null ? null : new SimpleNodeStateHolder(line);
- Function<NodeStateHolder, String> func2 = holder -> holder == null ? null : holder.getLine();
- ExternalSort.mergeSortedFiles(files,
- writer,
- comparator,
- charset,
- true, //distinct
- algorithm,
- func2,
- func1
- );
- } catch (IOException e) {
- log.error("Merge failed with IOException", e);
- return false;
- }
- log.debug("merge complete for {} with {}", outputFile.getName(), files);
- return true;
- }
-
- private boolean finalMerge() {
- List<File> mergeTarget = new ArrayList<>();
- int count = 0;
- while(!unmergedFiles.isEmpty()) {
- count++;
- mergeTarget.clear();
- mergeTarget.addAll(getSmallestUnmergedFiles(batchMergeSize));
- markAsMerged(mergeTarget);
- File outputFile = new File(mergeDir, String.format("final-%s", count));
- if (unmergedFiles.isEmpty()) {
- outputFile = sortedFile;
- }
-
- log.info("running final batch merge task for {} with {}", outputFile.getName(), mergeTarget);
- if (!merge(mergeTarget, outputFile)) {
- return false;
- } else if (outputFile.equals(sortedFile)) {
- return true;
- }
- unmergedFiles.add(outputFile);
- }
- return false;
- }
-
- private List<File> getSmallestUnmergedFiles(int size) {
- ArrayList<File> result = new ArrayList<>(unmergedFiles);
- result.remove(MERGE_POISON_PILL);
- result.sort(fileSizeComparator);
- int endIdx = size > result.size() ? result.size() : size;
- return result.subList(0, endIdx);
- }
-
- private void markAsMerged(List<File> target) {
- mergedFiles.addAll(target);
- unmergedFiles.removeAll(target);
- }
-
- @Override
- public void run() {
- this.executorService = Executors.newFixedThreadPool(threadPoolSize);
- try {
- FileUtils.forceMkdir(mergeDir);
- } catch (IOException e) {
- log.error("failed to create merged directory {}", mergeDir.getAbsolutePath());
- }
- Phaser mergeTaskPhaser = new Phaser(1);
- List<Future<File>> results = Lists.newArrayList();
- int count = 0;
-
- while (true) {
- try {
- File f = sortedFiles.take();
- unmergedFiles.add(f);
- log.debug("added sorted file {} to the unmerged list", f.getName());
- if (f.equals(MERGE_POISON_PILL) || f.equals(MERGE_FORCE_STOP_POISON_PILL)) {
- break;
- }
- // add another batchMergeSize so that we choose the smallest of a larger range
- if (unmergedFiles.size() >= 2*batchMergeSize) {
- count++;
- List<File> mergeTarget = getSmallestUnmergedFiles(batchMergeSize);
- File intermediateMergeFile = new File(mergeDir, String.format("intermediate-%s", count));
- Callable<File> mergeTask = new Task(mergeTarget, mergeTaskPhaser, intermediateMergeFile);
- markAsMerged(mergeTarget);
- results.add(executorService.submit(mergeTask));
- log.info("created merge task for {} with {}", intermediateMergeFile.getName(), mergeTarget);
- }
- } catch (InterruptedException e) {
- log.error("Failed while draining from sortedFiles", e);
- }
- }
- log.info("Waiting for batch sorting tasks completion");
-
- // Parent thread signals to stop immediately
- if (unmergedFiles.contains(MERGE_FORCE_STOP_POISON_PILL)) {
- log.info("Merger receives force stop signal, shutting down all merge tasks");
- this.mergeCancelled.set(true);
- mergeTaskPhaser.arriveAndAwaitAdvance();
- executorService.shutdown();
- mergeTaskPhaser.arrive();
- phaser.arriveAndDeregister();
- return;
- }
- mergeTaskPhaser.arriveAndAwaitAdvance();
- executorService.shutdown();
-
- // final merge in batch 64
- sortedFiles.drainTo(unmergedFiles);
- unmergedFiles.remove(MERGE_POISON_PILL);
- log.info("There are still {} sorted files not merged yet", unmergedFiles.size());
- try {
- boolean exceptionsCaught = false;
- for (Future<File> result : results) {
- try {
- unmergedFiles.add(result.get());
- } catch (Throwable e) {
- throwables.add(e);
- exceptionsCaught = true;
- }
- }
- log.debug("Completed merge result collection {}. Arriving at phaser now.", exceptionsCaught ? "partially" : "fully");
- } finally {
- mergeTaskPhaser.arrive();
- }
-
- finalMerge();
- log.info("Total batch sorted files length is {}", mergedFiles.size());
-
- phaser.arriveAndDeregister();
- }
-
- /**
- * Class responsible for merging sorted files
- */
- private class Task implements Callable<File> {
- private final Phaser mergeTaskPhaser;
- private final List<File> mergeTarget;
- private final File mergedFile;
-
- Task(List<File> mergeTarget, Phaser mergeTaskPhaser, File mergedFile) {
- this.mergeTarget = mergeTarget;
- this.mergeTaskPhaser = mergeTaskPhaser;
- this.mergedFile = mergedFile;
- mergeTaskPhaser.register();
- }
-
-
- @Override
- public File call() throws Exception {
- try {
- String mergedFileName = mergedFile.getName();
- if (mergeCancelled.get()) {
- log.debug("merge cancelled, skipping merge task");
- throw new EOFException("merge skipped for " + mergedFileName);
- } else if (merge(mergeTarget, mergedFile)) {
- log.info("merge complete for {}", mergedFileName);
- } else {
- log.error("merge failed for {}", mergedFileName);
- throw new RuntimeException("merge failed for " + mergedFileName);
- }
- } finally {
- mergeTaskPhaser.arriveAndDeregister();
- }
-
- return mergedFile;
- }
- }
-}
diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MultithreadedTraverseWithSortStrategy.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MultithreadedTraverseWithSortStrategy.java
deleted file mode 100644
index 9b84b0b3ee..0000000000
--- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MultithreadedTraverseWithSortStrategy.java
+++ /dev/null
@@ -1,562 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.jackrabbit.oak.index.indexer.document.flatfile;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.jackrabbit.guava.common.base.Stopwatch;
-import org.apache.jackrabbit.guava.common.collect.Lists;
-import org.apache.jackrabbit.oak.commons.Compression;
-import org.apache.jackrabbit.oak.index.indexer.document.CompositeException;
-import org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedStrategy;
-import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreSortStrategyBase;
-import org.apache.jackrabbit.oak.index.indexer.document.LastModifiedRange;
-import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntryTraverser;
-import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntryTraverserFactory;
-import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreUtils;
-import org.apache.jackrabbit.oak.plugins.document.mongo.TraversingRange;
-import org.apache.jackrabbit.oak.spi.blob.BlobStore;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.nio.file.StandardOpenOption;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.Phaser;
-import java.util.function.Predicate;
-import java.util.stream.Stream;
-
-import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileNodeStoreBuilder.DEFAULT_NUMBER_OF_DATA_DUMP_THREADS;
-import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileNodeStoreBuilder.DEFAULT_NUMBER_OF_FILES_PER_MERGE_TASK;
-import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileNodeStoreBuilder.DEFAULT_NUMBER_OF_MERGE_TASK_THREADS;
-import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileNodeStoreBuilder.PROP_MERGE_TASK_BATCH_SIZE;
-import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileNodeStoreBuilder.PROP_MERGE_THREAD_POOL_SIZE;
-import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileNodeStoreBuilder.PROP_THREAD_POOL_SIZE;
-
-/**
- * This class implements a sort strategy where node store is concurrently traversed for downloading node states by
- * multiple threads (number of threads is configurable via java system property {@link FlatFileNodeStoreBuilder#PROP_THREAD_POOL_SIZE}
- * and sorted node states are written to files while concurrently being merged by multiple threads (number of threads is
- * configurable via java system property {@link FlatFileNodeStoreBuilder#PROP_MERGE_THREAD_POOL_SIZE}.
- * The traverse/download and sort tasks are submitted to an executor service. Each of those tasks create some sorted files which
- * are merged while still downloading and in the end being merged (sorted) into one.
- *
- * <h3>Download task creation/splitting and result collection explanation -</h3>
- *
- * NEW_TASK_QUEUE=MultithreadedTraverseWithSortStrategy#taskQueue
- * FINISHED_QUEUE=TraverseAndSortTask#completedTasks
- * <ol>
- * <li>Have a BlockingQueue of tasks (NEW_TASK_QUEUE) that need to be executed. Each of this task has been assigned a _modified range.</li>
- * <li>Main thread (MultithreadedTraverseWithSortStrategy) adds an initial breakup of tasks (of type TraverseAndSortTask) to this queue.</li>
- * <li>Another monitoring thread (MultithreadedTraverseWithSortStrategy#TaskRunner) is consuming from this NEW_TASK_QUEUE and submitting those
- * tasks to executor service.</li>
- * <li>The executor service is a fixed thread pool executor so that number of threads remains unchanged overtime.</li>
- * <li>There is another queue for tasks that have finished execution - FINISHED_QUEUE. Each task adds itself or its ID to this queue when it
- * finishes execution. This is ConcurrentLinkedQueue because running tasks monitor this queue for finished tasks but don't need to block if no
- * finished task is there.
- * </li>
- * <li>Every running task monitors FINISHED_QUEUE as part of its execution cycle. If a running task has potential of splitting - identified by
- * what range of _modified values is left to be examined and downloaded - and if there is a task in the FINISHED_QUEUE, the running task tries
- * to dequeue the finished task.
- * </li>
- * <li>Only one of the running tasks would be able to dequeue a finished task. The running task which succeeds will reduce its upper limit of
- * _modified range and submit a new task with the _modified range after that (till its original upper limit).
- * </li>
- * <li>The monitoring thread is reading from NEW_TASK_QUEUE (blocking queue), so it needs an indication about when to stop looking for new tasks
- * from this queue. This is done by putting a poison pill entry in this queue. The main thread does it as explained below.
- * </li>
- * <li>
- * We use a phaser for coordination between main thread, the running tasks and the monitoring thread. This phaser has two phases -
- * <ol>
- * <li>Waiting for new tasks (from task splits)</li>
- * <li>Waiting for result collection</li>
- * </ol>
- * </li>
- * <li>
- * Each task registers itself (in phase 1 of phaser) inside its constructor itself. This ensures there is no created task which is not registered
- * to a phaser. Also when the task finishes (either normally or due to exception), it arrives and deregisters (since it is not needed in phase 2 of phaser).
- * </li>
- * <li>
- * Main thread after adding initial tasks to NEW_TASK_QUEUE, starts waiting for phase 1 to advance. Phase 1 would advance only if all the registered
- * (and hence created) tasks have arrived. When all tasks arrive, it means we won't have any more new tasks from task splitting.
- * </li>
- * <li>
- * So, once phase 1 advances, main thread adds the poison pill to NEW_TASK_QUEUE, reading which the monitoring thread advances to result collection.
- * Before starting result collection, it registers itself to phaser (now in phase 2).
- * </li>
- * <li>
- * Main thread after adding poison pill, starts waiting for phase 2 to advance. This advance happens when monitoring thread aggregates results from
- * all tasks and adds it to data structure which main thread could access.
- * </li>
- * <li>After the advance of phase 2, the main thread returns the result - the list of sorted files which are then merged by the next step of indexing process.</li>
- * </ol>
- *
- * <h3>Download task state save and resume explanation -</h3>
- * <ol>
- * <li>Each {@link TraverseAndSortTask} is assigned a {@link TraversingRange} which indicates the range of documents which this task should process.</li>
- * <li>Each task creates a directory sort-work-dir, where it stores the downloaded data and it also contains information about the traversing range this
- * task has to process.</li>
- * <li>Whenever task split happens, the upper limit of last modified range information stored in the sort-work-dir of split task is changed</li>
- * <li>Data is downloaded from document store in increasing order of (_modified, _id)</li>
- * <li>After the task dumps a batch of data into sort-work-dir, it also saves the (_modified, _id) state information of last dumped document.</li>
- * <li>If a task completes successfully without any exception, the completion event is also marked in its sort-work-dir</li>
- * <li>See {@link MultithreadedTraverseWithSortStrategy.DirectoryHelper} for sort-work-dir management APIs</li>
- * <li>If there is some exception and operation has to be retried, the retry happens in following way -
- * <ol>
- * <li>New tasks are created for all the non-completed sort-work-dirs of previous run, in the following way -
- * <ol>
- * <li>If no state information was saved in the sort-work-dir, that means, no data dump happened for that dir, so we create one task with same
- * traversing range information as this sort-work-dir</li>
- * <li>
- * If some state information is found, that means, some data was already dumped. So to avoid again downloading that data, we create two tasks-
- * Suppose the traversing range of this sort-work-dir is - _modified in [10, 100) and the state information is (50, doc_id)
- * <ol>
- * <li>First task will download docs with _modified = 50 and _id < doc_id</li>
- * <li>Second task downloads docs with _modified in [51, 100) </li>
- * </ol>
- * </li>
- * </ol>
- * </li>
- * <li>If multiple retries have happened, there would be a list of sort-work-dirs from each of the runs i.e. a {@code List<List<File>>}.</li>
- * <li>The list should be sorted in order of run i.e. first list would be the list of dirs from first run and last list would be the list of files from the most recent run
- * which failed.</li>
- * <li>The data dump files from each of sort-work-dirs of every run would be considered, but for creating further tasks for downloading remaining data
- * only the sort-work-dirs from most recent run would be considered.</li>
- * </ol>
- * </li>
- * </ol>
- * @deprecated Use {@link PipelinedStrategy} instead
- */
-@Deprecated
-public class MultithreadedTraverseWithSortStrategy extends IndexStoreSortStrategyBase {
-
- private static final Logger log = LoggerFactory.getLogger(MultithreadedTraverseWithSortStrategy.class);
- private final File mergeDir;
- private final String mergeDirName = "merge";
-
- /**
- * Comparator used for comparing node states for creating sorted files.
- */
- private final Comparator<NodeStateHolder> comparator;
- private final BlockingQueue<File> sortedFiles;
- private final ConcurrentLinkedQueue<Throwable> throwables;
- /**
- * Queue for traverse/download and sort tasks. After an initial creation of tasks, new tasks could be added to this queue
- * dynamically as already executing tasks decide to split up to provide more tasks for possibly idle threads.
- */
- private final BlockingQueue<Callable<List<File>>> taskQueue;
- /**
- * Phaser used for coordination with the traverse/download and sort tasks. All created tasks must register to this phaser
- * in the phase {@link Phases#WAITING_FOR_TASK_SPLITS}. If a task is created after this phase has advanced, its results
- * may get lost.
- */
- private final Phaser phaser;
-
- private final Phaser mergePhaser;
- /**
- * This poison pill is added to {@link #taskQueue} to indicate that no more new tasks would be submitted to this queue.
- */
- private static final Callable<List<File>> POISON_PILL = () -> null;
-
- private final MemoryManager memoryManager;
-
- private final long dumpThreshold;
-
- /**
- * Indicates the various phases of {@link #phaser}
- */
- private enum Phases {
- /**
- * This phase indicates we are in a state where new traverse/download and sort tasks could be created. After this
- * phase advances we don't expect any more new traverse/download and sort tasks to be created.
- */
- WAITING_FOR_TASK_SPLITS(0),
- /**
- * This phase indicates we are waiting for results from previously created tasks.
- */
- WAITING_FOR_RESULTS(1);
-
- private final int value;
-
- Phases(int value) {
- this.value = value;
- }
-
- public int getValue() {
- return value;
- }
- }
-
- /**
- * Constructor.
- * @param nodeStateEntryTraverserFactory factory class for creating {@link NodeStateEntryTraverser}s.
- * @param lastModifiedBreakPoints list of last modified values. We create initial {@link NodeStateEntryTraverser}s based
- * on entries in this list. For every pair of valid indices (i, i+1) of this list, we create
- * a traverser whose lower limit (inclusive) is the last modified value at index i and upper
- * limit (exclusive) is the last modified value at index i+1. For the last entry of this list,
- * we create a traverser with lower limit equal to that value and upper limit equal to that
- * value + 1. NOTE- This parameter is only read when {@code existingDataDumpDir} parameter is null.
- * @param pathComparator comparator used to help with sorting of node state entries.
- * @param blobStore blob store
- * @param storeDir Directory where sorted files will be created.
- * @param existingDataDumpDirs iterable over directories containing files from previous incomplete runs (which need to
- * be merged with the result from current run, if the current run has resumed from the point where
- * previous runs stopped). If this is not null and not empty, the {@code lastModifiedBreakPoints} parameter is ignored.
- * @param algorithm string representation of the compression algorithm, use "none" for disable compression.
- * @deprecated use {@link MultithreadedTraverseWithSortStrategy#MultithreadedTraverseWithSortStrategy(NodeStateEntryTraverserFactory, List, Set, BlobStore, File, List, Compression, MemoryManager, long, Predicate, String)} instead
- */
- @Deprecated
- MultithreadedTraverseWithSortStrategy(NodeStateEntryTraverserFactory nodeStateEntryTraverserFactory,
- List<Long> lastModifiedBreakPoints, PathElementComparator pathComparator,
- BlobStore blobStore, File storeDir, List<File> existingDataDumpDirs,
- Compression algorithm, MemoryManager memoryManager, long dumpThreshold,
- Predicate<String> pathPredicate) throws IOException {
- super(storeDir, algorithm, pathPredicate, null, null);
- this.mergeDir = new File(storeDir, mergeDirName);
- this.sortedFiles = new LinkedBlockingQueue<>();
- this.throwables = new ConcurrentLinkedQueue<>();
- this.comparator = (e1, e2) -> pathComparator.compare(e1.getPathElements(), e2.getPathElements());
- taskQueue = new LinkedBlockingQueue<>();
- phaser = new Phaser() {
- @Override
- protected boolean onAdvance(int phase, int registeredParties) {
- //terminate phaser only if it is advancing from last phase and there are no registered parties
- return phase == Phases.WAITING_FOR_RESULTS.value && registeredParties == 0;
- }
- };
- // arrives on final file sorted
- mergePhaser = new Phaser(1);
- this.memoryManager = memoryManager;
- this.dumpThreshold = dumpThreshold;
- createInitialTasks(nodeStateEntryTraverserFactory, lastModifiedBreakPoints, blobStore, existingDataDumpDirs);
- }
-
- public MultithreadedTraverseWithSortStrategy(NodeStateEntryTraverserFactory nodeStateEntryTraverserFactory,
- List<Long> lastModifiedBreakPoints, Set<String> preferredPaths,
- BlobStore blobStore, File storeDir, List<File> existingDataDumpDirs,
- Compression algorithm, MemoryManager memoryManager, long dumpThreshold,
- Predicate<String> pathPredicate, String checkpoint) throws IOException {
- super(storeDir, algorithm, pathPredicate, preferredPaths, checkpoint);
- this.mergeDir = new File(storeDir, mergeDirName);
- this.sortedFiles = new LinkedBlockingQueue<>();
- this.throwables = new ConcurrentLinkedQueue<>();
- this.comparator = (e1, e2) -> new PathElementComparator(preferredPaths).compare(e1.getPathElements(), e2.getPathElements());
- taskQueue = new LinkedBlockingQueue<>();
- phaser = new Phaser() {
- @Override
- protected boolean onAdvance(int phase, int registeredParties) {
- //terminate phaser only if it is advancing from last phase and there are no registered parties
- return phase == Phases.WAITING_FOR_RESULTS.value && registeredParties == 0;
- }
- };
- // arrives on final file sorted
- mergePhaser = new Phaser(1);
- this.memoryManager = memoryManager;
- this.dumpThreshold = dumpThreshold;
- createInitialTasks(nodeStateEntryTraverserFactory, lastModifiedBreakPoints, blobStore, existingDataDumpDirs);
- }
-
- void createInitialTasks(NodeStateEntryTraverserFactory nodeStateEntryTraverserFactory,
- List<Long> lastModifiedBreakPoints, BlobStore blobStore, List<File> existingDataDumpDirs)
- throws IOException {
- ConcurrentLinkedQueue<String> completedTasks = new ConcurrentLinkedQueue<>();
- if (existingDataDumpDirs != null && existingDataDumpDirs.size() > 0) {
- //include all sorted files from previous incomplete runs
- for (int i = 0; i < existingDataDumpDirs.size(); i++) {
- File existingDataDumpDir = existingDataDumpDirs.get(i);
- File [] existingWorkDirs = existingDataDumpDir.listFiles();
- if (existingWorkDirs == null) {
- throw new IllegalArgumentException("Could not obtain file from " + existingDataDumpDir.getPath());
- }
- for (File existingSortWorkDir : existingWorkDirs) {
- if (!existingSortWorkDir.isDirectory()) {
- log.info("Not a directory {}. Skipping it.", existingSortWorkDir.getAbsolutePath());
- continue;
- } else if (existingSortWorkDir.getName().equals(mergeDirName)) {
- log.info("Intermediate Merge Directory {}. Skipping it.", existingSortWorkDir.getAbsolutePath());
- DirectoryHelper.getDataFiles(existingSortWorkDir).forEach(file -> {
- log.debug("Including existing intermediate merged file {}", file.getPath());
- sortedFiles.add(file);
- });
- continue;
- }
- boolean downloadCompleted = DirectoryHelper.hasCompleted(existingSortWorkDir);
- if (!downloadCompleted && i == existingDataDumpDirs.size() - 1) {
- long start = DirectoryHelper.getLastModifiedLowerLimit(existingSortWorkDir);
- long end = DirectoryHelper.getLastModifiedUpperLimit(existingSortWorkDir);
- DirectoryHelper.SavedState savedState = DirectoryHelper.getIdOfLastDownloadedDocument(existingSortWorkDir);
- if (savedState == null) {
- addTask(new TraversingRange(new LastModifiedRange(start, end), null), nodeStateEntryTraverserFactory,
- blobStore, completedTasks);
- } else {
- start = savedState.lastModified;
- addTask(new TraversingRange(new LastModifiedRange(start, start + 1), savedState.id), nodeStateEntryTraverserFactory,
- blobStore, completedTasks);
- if (end > start + 1) {
- addTask(new TraversingRange(new LastModifiedRange(start + 1, end), null), nodeStateEntryTraverserFactory,
- blobStore, completedTasks);
- }
- }
- }
- log.info("Including existing sorted files from directory {} (hasCompleted={})",
- existingSortWorkDir.getAbsolutePath(), downloadCompleted);
- DirectoryHelper.getDataFiles(existingSortWorkDir).forEach(file -> {
- log.debug("Including existing sorted file {}", file.getName());
- sortedFiles.add(file);
- });
- }
- }
- } else {
- for (int i = 0; i < lastModifiedBreakPoints.size(); i++) {
- long start = lastModifiedBreakPoints.get(i);
- long end = i < lastModifiedBreakPoints.size() - 1 ? lastModifiedBreakPoints.get(i + 1) : lastModifiedBreakPoints.get(i) + 1;
- addTask(new TraversingRange(new LastModifiedRange(start, end), null),
- nodeStateEntryTraverserFactory, blobStore, completedTasks);
- }
- }
- }
-
- void addTask(TraversingRange range, NodeStateEntryTraverserFactory nodeStateEntryTraverserFactory, BlobStore blobStore,
- ConcurrentLinkedQueue<String> completedTasks) throws IOException {
- taskQueue.add(new TraverseAndSortTask(range, comparator, blobStore, this.getStoreDir(),
- this.getAlgorithm(), completedTasks, taskQueue, phaser, nodeStateEntryTraverserFactory,
- memoryManager, dumpThreshold, sortedFiles, this.getPathPredicate()));
- }
-
- @Override
- public File createSortedStoreFile() throws CompositeException {
- String watcherThreadName = "watcher";
- String mergerThreadName = "merger";
- Thread watcher = new Thread(new TaskRunner(), watcherThreadName);
- watcher.setDaemon(true);
- watcher.start();
- File sortedFile = new File(this.getStoreDir(), IndexStoreUtils.getSortedStoreFileName(this.getAlgorithm()));
- int threadPoolSize = Integer.getInteger(PROP_MERGE_THREAD_POOL_SIZE, DEFAULT_NUMBER_OF_MERGE_TASK_THREADS);
- int batchMergeSize = Integer.getInteger(PROP_MERGE_TASK_BATCH_SIZE, DEFAULT_NUMBER_OF_FILES_PER_MERGE_TASK);
- Runnable mergeRunner = new MergeRunner(sortedFile, sortedFiles, mergeDir, comparator, mergePhaser, batchMergeSize, threadPoolSize, this.getAlgorithm());
- Thread merger = new Thread(mergeRunner, mergerThreadName);
- merger.setDaemon(true);
- merger.start();
- phaser.awaitAdvance(Phases.WAITING_FOR_TASK_SPLITS.value);
- log.debug("All tasks completed. Signalling {} to proceed to result collection.", watcherThreadName);
- taskQueue.add(POISON_PILL);
- phaser.awaitAdvance(Phases.WAITING_FOR_RESULTS.value);
- if (!throwables.isEmpty()) {
- CompositeException exception = new CompositeException();
- for (Throwable throwable : throwables) {
- exception.addSuppressed(throwable);
- }
- sortedFiles.add(MergeRunner.MERGE_FORCE_STOP_POISON_PILL);
- mergePhaser.awaitAdvance(0);
- throw exception;
- }
- log.debug("Result collection complete. Proceeding to final merge.");
- Stopwatch w = Stopwatch.createStarted();
- sortedFiles.add(MergeRunner.MERGE_POISON_PILL);
- mergePhaser.awaitAdvance(0);
- log.info("Merging of sorted files completed in {}", w);
- return sortedFile;
- }
-
- @Override
- public long getEntryCount() {
- //todo - get actual entry count for correct progress estimation
- return 0;
- }
-
- /**
- * Class responsible for -
- * <ol>
- * <li>Watching {@link #taskQueue} for new tasks</li>
- * <li>Submitting those tasks to an {@link ExecutorService}</li>
- * <li>Collecting the results (sorted files) created by those tasks into one place</li>
- * </ol>
- */
- private class TaskRunner implements Runnable {
-
- private final ExecutorService executorService;
- private final int threadPoolSize = Integer.getInteger(PROP_THREAD_POOL_SIZE, DEFAULT_NUMBER_OF_DATA_DUMP_THREADS);
-
- public TaskRunner() {
- this.executorService = Executors.newFixedThreadPool(threadPoolSize);
- }
-
- @Override
- public void run() {
- try {
- log.info("Using a thread pool of size {}", threadPoolSize);
- List<Future<List<File>>> results = Lists.newArrayList();
- while (true) {
- Callable<List<File>> task = taskQueue.take();
- if (task == POISON_PILL) {
- log.debug("Won't wait for new tasks now.");
- break;
- }
- results.add(executorService.submit(task));
- }
- log.debug("Registering to phaser and waiting for results now.");
- phaser.register();
- try {
- boolean exceptionsCaught = false;
- for (Future<List<File>> result : results) {
- try {
- result.get();
- } catch (Throwable e) {
- throwables.add(e);
- exceptionsCaught = true;
- }
- }
- log.debug("Completed result collection {}. Arriving at phaser now.", !exceptionsCaught ? "fully" : "partially");
- } finally {
- phaser.arrive();
- }
- } catch (InterruptedException e) {
- log.error("Could not complete task submissions", e);
- }
- executorService.shutdown();
- }
- }
-
- static class DirectoryHelper {
-
- private static final String PREFIX = "sort-work-dir-";
- private static final String LAST_MODIFIED_TIME_DELIMITER = "-from-";
- /**
- * File name for file which indicates the _id and _modified of last processed document. Let's say we saved the following
- * values {_id=saved_id, _modified=saved_modified}, then while resuming download we need to download :
- * 1. All documents with _modified = saved_modified which have _id > saved_id
- * 2. All documents with _modified > saved_modified and _modified < LAST_MODIFIED_UPPER_LIMIT for this sort work dir
- */
- private static final String STATUS_FILE_NAME = "last-saved";
- /**
- * File name for file which indicates the upper limit (exclusive) of last modified time of the range of documents which this
- * sort work dir is meant to have data for.
- */
- private static final String LAST_MODIFIED_UPPER_LIMIT = "last-modified-upper-limit";
- /**
- * If this file is present, that means all the documents meant for this sort work dir have been processed.
- */
- private static final String COMPLETION_MARKER_FILE_NAME = "completed";
- private static final Logger log = LoggerFactory.getLogger(DirectoryHelper.class);
-
- static File createdSortWorkDir(File storeDir, String taskID, long lastModifiedLowerBound,
- long lastModifiedUpperBound) throws IOException {
- File sortedFileDir = new File(storeDir, PREFIX + taskID + LAST_MODIFIED_TIME_DELIMITER + lastModifiedLowerBound);
- FileUtils.forceMkdir(sortedFileDir);
- setLastModifiedUpperLimit(sortedFileDir, lastModifiedUpperBound);
- return sortedFileDir;
- }
-
- static long getLastModifiedLowerLimit(File dir) {
- if (!dir.isDirectory()) {
- throw new IllegalArgumentException(dir.getAbsolutePath() + " is not a directory");
- }
- return Long.parseLong(dir.getName().substring(dir.getName().lastIndexOf(LAST_MODIFIED_TIME_DELIMITER) +
- LAST_MODIFIED_TIME_DELIMITER.length()));
- }
-
- static void setLastModifiedUpperLimit(File sortWorkDir, long lastModifiedUpperLimit) throws IOException {
- Files.write(Paths.get(sortWorkDir.getAbsolutePath() + "/" + LAST_MODIFIED_UPPER_LIMIT), ("" + lastModifiedUpperLimit).getBytes(),
- StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING);
- }
-
- static long getLastModifiedUpperLimit(File sortWorkDir) throws IOException {
- File lastModifiedUpperLimitFile = new File(sortWorkDir.getAbsolutePath() + "/" + LAST_MODIFIED_UPPER_LIMIT);
- if (!lastModifiedUpperLimitFile.exists()) {
- throw new IOException("Could not find file containing last modified upper limit in " + sortWorkDir.getAbsolutePath());
- }
- return Long.parseLong(Files.readAllLines(lastModifiedUpperLimitFile.toPath()).get(0));
- }
-
- static void markCompleted(File sortWorkDir) {
- try {
- Files.write(Paths.get(sortWorkDir.getAbsolutePath() + "/" + COMPLETION_MARKER_FILE_NAME), ("completed").getBytes(),
- StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING);
- } catch (IOException e) {
- log.warn("Resuming download will not be accurate. Could not mark the directory "
- + sortWorkDir.getAbsolutePath() + " completed.", e);
- }
- }
-
- static boolean hasCompleted(File sortWorkDir) {
- return new File(sortWorkDir + "/" + COMPLETION_MARKER_FILE_NAME).exists();
- }
-
- static void markLastProcessedStatus(File sortWorkDir, long lastDownloadedDocLastModified, String lastDownloadedDocID) {
- try {
- Files.write(Paths.get(sortWorkDir.getAbsolutePath() + "/" + STATUS_FILE_NAME),
- new SavedState(lastDownloadedDocLastModified, lastDownloadedDocID).serialize().getBytes(),
- StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING);
- } catch (IOException e) {
- log.warn("Resuming download will not be accurate. Could not save last processed status = " + lastDownloadedDocID
- + " in " + sortWorkDir.getAbsolutePath(), e);
- }
- }
-
- static SavedState getIdOfLastDownloadedDocument(File sortWorkDir) throws IOException {
- File statusFile = new File(sortWorkDir.getAbsolutePath() + "/" + STATUS_FILE_NAME);
- if (!statusFile.exists()) {
- return null;
- }
- return SavedState.deserialize(Files.readAllLines(statusFile.toPath()).get(0));
- }
-
- static Stream<File> getDataFiles(File sortWorkDir) {
- return Arrays.stream(sortWorkDir.listFiles()).filter(f -> !STATUS_FILE_NAME.equals(f.getName()) &&
- !COMPLETION_MARKER_FILE_NAME.equals(f.getName()) && !LAST_MODIFIED_UPPER_LIMIT.equals(f.getName()));
- }
-
- static class SavedState {
- long lastModified;
- String id;
-
- public SavedState(long lastModified, String id) {
- this.lastModified = lastModified;
- this.id = id;
- }
-
- String serialize() {
- return lastModified + ":" + id;
- }
-
- static SavedState deserialize(String s) {
- int colonIndex = s.indexOf(":");
- if (colonIndex == -1) {
- throw new IllegalArgumentException("Invalid serialized string " + s);
- }
- long lastMod = Long.parseLong(s.substring(0, colonIndex));
- String id = s.substring(colonIndex + 1);
- return new SavedState(lastMod, id);
- }
- }
-
- }
-}
diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/TraverseAndSortTask.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/TraverseAndSortTask.java
deleted file mode 100644
index b11be622d1..0000000000
--- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/TraverseAndSortTask.java
+++ /dev/null
@@ -1,352 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.jackrabbit.oak.index.indexer.document.flatfile;
-
-import org.apache.jackrabbit.guava.common.base.Stopwatch;
-import org.apache.jackrabbit.oak.commons.Compression;
-import org.apache.jackrabbit.oak.index.indexer.document.LastModifiedRange;
-import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry;
-import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntryTraverser;
-import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntryTraverserFactory;
-import org.apache.jackrabbit.oak.plugins.document.mongo.TraversingRange;
-import org.apache.jackrabbit.oak.spi.blob.BlobStore;
-import org.apache.jackrabbit.oak.spi.state.NodeStateUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedWriter;
-import java.io.EOFException;
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Optional;
-import java.util.Queue;
-import java.util.Random;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.Callable;
-import java.util.concurrent.Phaser;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.Predicate;
-
-import static org.apache.jackrabbit.oak.commons.IOUtils.humanReadableByteCount;
-import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileStoreUtils.sizeOf;
-import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.MultithreadedTraverseWithSortStrategy.DirectoryHelper;
-
-/**
- * A callable representing a task for traversing/downloading the nodes states from the node store and creating sorted files
- * based on the downloaded data.
- */
-class TraverseAndSortTask implements Callable<List<File>>, MemoryManagerClient {
-
- /**
- * An prefix for ID of this task (value is acronym for TraverseAndSortTask).
- */
- private static final String ID_PREFIX = "TAST-";
- private final Logger log = LoggerFactory.getLogger(getClass());
- /**
- * Iterable over the nodeStates this task is supposed to traverse. Note that the iteration could stop without traversing
- * all the node states from this iterable, if this task decides to split up and offer some of its work to another
- * {@link TraverseAndSortTask}
- */
- private final NodeStateEntryTraverser nodeStates;
- private final NodeStateEntryWriter entryWriter;
- private final File storeDir;
- private final Compression algorithm;
- private final Comparator<NodeStateHolder> comparator;
- private long entryCount;
- private long memoryUsed;
- private final File sortWorkDir;
- private final List<File> sortedFiles = new ArrayList<>();
- private final BlockingQueue<File> parentSortedFiles;
- private final LinkedList<NodeStateHolder> entryBatch = new LinkedList<>();
- private NodeStateEntry lastSavedNodeStateEntry;
- private final String taskID;
- private final AtomicReference<Phaser> dataDumpNotifyingPhaserRef = new AtomicReference<>();
- /**
- * Queue to which the {@link #taskID} of completed tasks is added.
- */
- private final Queue<String> completedTasks;
- /**
- * Queue to which any newly created tasks should be added. New tasks are created when this task decides it needs to split up.
- */
- private final Queue<Callable<List<File>>> newTasksQueue;
- /**
- * {@link Phaser} used to signal the creation and completion events of this task. Task is registered to this phaser inside the
- * constructor itself, so that it is not possible to have any created task which is not registered to this phaser.
- */
- private final Phaser phaser;
- /**
- * The node states which have a last modified time less than this would not be considered by this task.
- */
- final long lastModifiedLowerBound;
- /**
- * The node states which have a last modified time greater than or equal to this would not be considered by this task.
- */
- private long lastModifiedUpperBound;
- private final BlobStore blobStore;
- private final NodeStateEntryTraverserFactory nodeStateEntryTraverserFactory;
- private final MemoryManager memoryManager;
- private String registrationID;
- private final long dumpThreshold;
- private Predicate<String> pathPredicate = path -> true;
-
- TraverseAndSortTask(TraversingRange range, Comparator<NodeStateHolder> comparator,
- BlobStore blobStore, File storeDir, Compression algorithm,
- Queue<String> completedTasks, Queue<Callable<List<File>>> newTasksQueue,
- Phaser phaser, NodeStateEntryTraverserFactory nodeStateEntryTraverserFactory,
- MemoryManager memoryManager, long dumpThreshold, BlockingQueue parentSortedFiles,
- Predicate<String> pathPredicate) throws IOException {
- this.nodeStates = nodeStateEntryTraverserFactory.create(range);
- this.taskID = ID_PREFIX + nodeStates.getId();
- this.lastModifiedLowerBound = nodeStates.getDocumentTraversalRange().getLastModifiedRange().getLastModifiedFrom();
- this.lastModifiedUpperBound = nodeStates.getDocumentTraversalRange().getLastModifiedRange().getLastModifiedTo();
- this.blobStore = blobStore;
- this.entryWriter = new NodeStateEntryWriter(blobStore);
- this.storeDir = storeDir;
- this.algorithm = algorithm;
- this.comparator = comparator;
- this.completedTasks = completedTasks;
- this.newTasksQueue = newTasksQueue;
- this.phaser = phaser;
- this.nodeStateEntryTraverserFactory = nodeStateEntryTraverserFactory;
- this.memoryManager = memoryManager;
- this.dumpThreshold = dumpThreshold;
- this.parentSortedFiles = parentSortedFiles;
- this.pathPredicate = pathPredicate;
- sortWorkDir = DirectoryHelper.createdSortWorkDir(storeDir, taskID, lastModifiedLowerBound, lastModifiedUpperBound);
- if (range.getStartAfterDocumentID() != null) {
- DirectoryHelper.markLastProcessedStatus(sortWorkDir, lastModifiedLowerBound, range.getStartAfterDocumentID());
- }
- phaser.register();
- log.debug("Task {} with traversing range {} registered to phaser", taskID, range);
- }
-
- @Override
- public void memoryLow(Phaser phaser) {
- if (dataDumpNotifyingPhaserRef.compareAndSet(null, phaser)) {
- log.info("{} registering to low memory notification phaser", taskID);
- phaser.register();
- } else {
- log.warn("{} already has a low memory notification phaser.", taskID);
- }
- }
-
- private boolean registerWithMemoryManager() {
- Optional<String> registrationIDOptional = memoryManager.registerClient(this);
- registrationIDOptional.ifPresent(s -> {
- registrationID = s;
- log.debug("{} Registered with memory manager with registration ID {}", taskID, registrationID);
- });
- return registrationIDOptional.isPresent();
- }
-
- @Override
- public List<File> call() throws EOFException {
- try {
- Random random = new Random();
- while (MemoryManager.Type.JMX_BASED.equals(memoryManager.getType()) && !registerWithMemoryManager()) {
- int sleepDuration = 1000 + random.nextInt(500);
- log.info("{} Could not register to memory manager. Sleeping {} ms before retrying", taskID, sleepDuration);
- try {
- Thread.sleep(sleepDuration);
- } catch (InterruptedException e) {
- log.warn(taskID + " Interrupted while sleeping while trying to register to memory manager", e);
- }
- }
- logFlags();
- writeToSortedFiles();
- log.info("Completed task {}", taskID);
- completedTasks.add(taskID);
- DirectoryHelper.markCompleted(sortWorkDir);
- parentSortedFiles.addAll(sortedFiles);
-
- return sortedFiles;
- } catch (RuntimeException e) {
- log.error(taskID + " dumping existing progress because it could not complete download with ", e);
- try {
- sortAndSaveBatch();
- reset();
- log.debug(taskID + " complete dumping existing progress");
- } catch (Exception dumpErr) {
- log.warn(taskID + " failed to dump existing progress with ", dumpErr);
- }
- throw e;
- } catch (IOException e) {
- log.error(taskID + " could not complete download with ", e);
- } finally {
- phaser.arriveAndDeregister();
- log.info("{} entered finally block.", taskID);
- Phaser dataDumpPhaser = dataDumpNotifyingPhaserRef.get();
- if (dataDumpPhaser != null) {
- log.info("{} Data dump phaser not null after task completion. Notifying memory listener.", taskID);
- dataDumpPhaser.arriveAndDeregister();
- }
- if (MemoryManager.Type.JMX_BASED.equals(memoryManager.getType())) {
- memoryManager.deregisterClient(registrationID);
- }
- try {
- nodeStates.close();
- } catch (IOException e) {
- log.error(taskID + " could not close NodeStateEntryTraverser", e);
- }
- }
- return Collections.emptyList();
- }
-
- private void writeToSortedFiles() throws IOException {
- Stopwatch w = Stopwatch.createStarted();
- for (NodeStateEntry e : nodeStates) {
- if (e.getLastModified() >= lastModifiedUpperBound) {
- break;
- }
- entryCount++;
- addEntry(e);
- }
-
- //Save the last batch
- sortAndSaveBatch();
- reset();
-
- log.info("{} Dumped {} nodestates in json format in {}", taskID, entryCount, w);
- log.info("{} Created {} sorted files of size {} to merge", taskID,
- sortedFiles.size(), humanReadableByteCount(sizeOf(sortedFiles)));
- }
-
- File getSortWorkDir() {
- return sortWorkDir;
- }
-
- long getLastModifiedLowerBound() {
- return lastModifiedLowerBound;
- }
-
- long getLastModifiedUpperBound() {
- return lastModifiedUpperBound;
- }
-
- void addEntry(NodeStateEntry e) throws IOException {
- if (memoryManager.isMemoryLow()) {
- if (memoryUsed >= dumpThreshold) {
- sortAndSaveBatch();
- reset();
- } else {
- log.trace("{} Memory manager reports low memory but there is not enough data ({}) to dump.", taskID, humanReadableByteCount(memoryUsed));
- }
- }
-
- long remainingNumberOfTimestamps = lastModifiedUpperBound - e.getLastModified();
- // check if this task can be split
- if (remainingNumberOfTimestamps > 1) {
- /*
- If there is a completed task, there is a chance of some worker thread being idle, so we create a new task from
- the current task. To split, we reduce the traversal upper bound for this task and pass on the node states from
- the new upper bound to the original upper bound to a new task.
- */
- if (completedTasks.poll() != null) {
- long splitPoint = e.getLastModified() + (long) Math.ceil((lastModifiedUpperBound - e.getLastModified()) / 2.0);
- log.info("Splitting task {}. New Upper limit for this task {}. New task range - {} to {}", taskID, splitPoint, splitPoint, this.lastModifiedUpperBound);
- newTasksQueue.add(new TraverseAndSortTask(new TraversingRange(
- new LastModifiedRange(splitPoint, this.lastModifiedUpperBound), null),
- comparator, blobStore, storeDir, algorithm, completedTasks,
- newTasksQueue, phaser, nodeStateEntryTraverserFactory, memoryManager,
- dumpThreshold, parentSortedFiles, pathPredicate));
- this.lastModifiedUpperBound = splitPoint;
- DirectoryHelper.setLastModifiedUpperLimit(sortWorkDir, lastModifiedUpperBound);
- }
- }
-
- String path = e.getPath();
- if (!NodeStateUtils.isHiddenPath(path) && pathPredicate.test(path)) {
- log.debug("Adding to entry, path={} hidden={} predicate={}", path, NodeStateUtils.isHiddenPath(path), pathPredicate.test(path));
- String jsonText = entryWriter.asJson(e.getNodeState());
- //Here logic differs from NodeStateEntrySorter in sense that
- //Holder line consist only of json and not 'path|json'
- NodeStateHolder h = new StateInBytesHolder(path, jsonText);
- entryBatch.add(h);
- updateMemoryUsed(h);
- }
-
- lastSavedNodeStateEntry = e;
-
- }
-
- /*
- * For explanation regarding synchronization see {@link #memoryLow(Phaser)}
- */
- private synchronized void reset() {
- log.trace("{} Reset called ", taskID);
- if (MemoryManager.Type.SELF_MANAGED.equals(memoryManager.getType())) {
- memoryManager.changeMemoryUsedBy(-1 * memoryUsed);
- }
- Phaser phaser = dataDumpNotifyingPhaserRef.get();
- if (phaser != null) {
- log.info("{} Finished saving data to disk. Notifying memory listener.", taskID);
- phaser.arriveAndDeregister();
- dataDumpNotifyingPhaserRef.compareAndSet(phaser, null);
- }
- memoryUsed = 0;
- }
-
- private void sortAndSaveBatch() throws IOException {
- if (entryBatch.isEmpty()) {
- return;
- }
- entryBatch.sort(comparator);
- Stopwatch w = Stopwatch.createStarted();
- File newtmpfile = File.createTempFile("sortInBatch", "flatfile", sortWorkDir);
- long textSize = 0;
- long size = entryBatch.size();
- try (BufferedWriter writer = FlatFileStoreUtils.createWriter(newtmpfile, algorithm)) {
- // no concurrency issue with this traversal because addition to this list is only done in #addEntry which, for
- // a given TraverseAndSortTask object will only be called from same thread
- while (!entryBatch.isEmpty()) {
- NodeStateHolder h = entryBatch.removeFirst();
- //Here holder line only contains nodeState json
- String text = entryWriter.toString(h.getPathElements(), h.getLine());
- writer.write(text);
- writer.newLine();
- textSize += text.length() + 1;
- }
- }
-
- log.info("{} Sorted and stored batch of size {} (uncompressed {}) with {} entries in {}. Last entry id = {}", taskID,
- humanReadableByteCount(newtmpfile.length()), humanReadableByteCount(textSize), size, w,
- lastSavedNodeStateEntry.getId());
- DirectoryHelper.markLastProcessedStatus(sortWorkDir, lastSavedNodeStateEntry.getLastModified(),
- lastSavedNodeStateEntry.getId());
- sortedFiles.add(newtmpfile);
- }
-
- private void logFlags() {
- log.info("Starting task {}", taskID);
- }
-
- private void updateMemoryUsed(NodeStateHolder h) {
- memoryUsed += h.getMemorySize();
- if (MemoryManager.Type.SELF_MANAGED.equals(memoryManager.getType())) {
- memoryManager.changeMemoryUsedBy(h.getMemorySize());
- }
- }
-
-}
diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/indexversion/PurgeOldIndexVersion.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/indexversion/PurgeOldIndexVersion.java
index 083e7ac576..73255e35a5 100644
--- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/indexversion/PurgeOldIndexVersion.java
+++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/indexversion/PurgeOldIndexVersion.java
@@ -39,7 +39,6 @@ import org.apache.jackrabbit.oak.plugins.index.IndexPathService;
import org.apache.jackrabbit.oak.plugins.index.IndexPathServiceImpl;
import org.apache.jackrabbit.oak.plugins.index.IndexUpdateProvider;
import org.apache.jackrabbit.oak.plugins.index.counter.NodeCounterEditorProvider;
-import org.apache.jackrabbit.oak.plugins.index.property.OrderedPropertyIndexEditorProvider;
import org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexEditorProvider;
import org.apache.jackrabbit.oak.plugins.index.reference.ReferenceEditorProvider;
import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/BlobStoreFixtureProvider.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/BlobStoreFixtureProvider.java
index d4dad3b6d9..be89ca6756 100644
--- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/BlobStoreFixtureProvider.java
+++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/BlobStoreFixtureProvider.java
@@ -28,7 +28,6 @@ import java.util.Dictionary;
import java.util.Enumeration;
import java.util.Map;
import java.util.Properties;
-import java.util.UUID;
import org.apache.jackrabbit.guava.common.collect.Maps;
import org.apache.jackrabbit.guava.common.io.Closer;
diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilderTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilderTest.java
index 0d1465e588..c15c29003e 100644
--- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilderTest.java
+++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilderTest.java
@@ -57,7 +57,6 @@ import org.mockito.Mockito;
import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileNodeStoreBuilder.OAK_INDEXER_SORTED_FILE_PATH;
import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileNodeStoreBuilder.OAK_INDEXER_SORT_STRATEGY_TYPE;
-import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileNodeStoreBuilder.OAK_INDEXER_TRAVERSE_WITH_SORT;
import static org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreUtils.OAK_INDEXER_USE_LZ4;
import static org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreUtils.OAK_INDEXER_USE_ZIP;
import static org.junit.Assert.assertEquals;
@@ -82,7 +81,6 @@ public class FlatFileNodeStoreBuilderTest {
MongoDocumentStore mongoDocumentStore = mock(MongoDocumentStore.class);
when(mongoDocumentStore.isReadOnly()).thenReturn(true);
FlatFileNodeStoreBuilder builder = new FlatFileNodeStoreBuilder(folder.getRoot())
- .withLastModifiedBreakPoints(List.of())
.withNodeStateEntryTraverserFactory(nodeStateEntryTraverserFactory)
.withIndexDefinitions(Set.of())
.withMongoDocumentStore(mongoDocumentStore);
@@ -99,25 +97,6 @@ public class FlatFileNodeStoreBuilderTest {
assertTrue(sortStrategy instanceof TraverseWithSortStrategy);
}
- @Test
- public void enableTraverseAndSortStrategyUsingSystemProperty() throws Exception {
- System.setProperty(OAK_INDEXER_TRAVERSE_WITH_SORT, "true");
- FlatFileNodeStoreBuilder builder = new FlatFileNodeStoreBuilder(folder.getRoot())
- .withNodeStateEntryTraverserFactory(nodeStateEntryTraverserFactory);
- SortStrategy sortStrategy = builder.createSortStrategy(builder.createStoreDir());
- assertTrue(sortStrategy instanceof TraverseWithSortStrategy);
- }
-
- @Test
- public void sortStrategySystemPropertyPrecedence() throws Exception {
- System.setProperty(OAK_INDEXER_TRAVERSE_WITH_SORT, "false");
- System.setProperty(OAK_INDEXER_SORT_STRATEGY_TYPE, FlatFileNodeStoreBuilder.SortStrategyType.TRAVERSE_WITH_SORT.toString());
- FlatFileNodeStoreBuilder builder = new FlatFileNodeStoreBuilder(folder.getRoot())
- .withNodeStateEntryTraverserFactory(nodeStateEntryTraverserFactory);
- SortStrategy sortStrategy = builder.createSortStrategy(builder.createStoreDir());
- assertTrue(sortStrategy instanceof TraverseWithSortStrategy);
- }
-
@Test
public void testBuild() throws CompositeException, IOException {
System.setProperty(OAK_INDEXER_USE_ZIP, "false");
diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreTest.java
index 2c54d6953b..fabe1273a8 100644
--- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreTest.java
+++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreTest.java
@@ -19,83 +19,49 @@
package org.apache.jackrabbit.oak.index.indexer.document.flatfile;
-import org.apache.commons.io.FileUtils;
-import org.apache.jackrabbit.guava.common.collect.Iterables;
-import org.apache.jackrabbit.oak.index.indexer.document.CompositeException;
import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry;
-import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry.NodeStateEntryBuilder;
import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntryTraverser;
-import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntryTraverserFactory;
-import org.apache.jackrabbit.oak.plugins.document.mongo.DocumentStoreSplitter;
-import org.apache.jackrabbit.oak.plugins.document.mongo.TraversingRange;
import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore;
-import org.apache.jackrabbit.oak.spi.state.NodeStateUtils;
import org.jetbrains.annotations.NotNull;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.mockito.Mockito;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import java.io.File;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Collections;
import java.util.Iterator;
import java.util.List;
-import java.util.Objects;
-import java.util.Optional;
import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Predicate;
-import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
import static java.util.Arrays.asList;
import static java.util.Collections.singleton;
import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileNodeStoreBuilder.OAK_INDEXER_SORT_STRATEGY_TYPE;
-import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileNodeStoreBuilder.PROP_MERGE_TASK_BATCH_SIZE;
-import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileNodeStoreBuilder.PROP_THREAD_POOL_SIZE;
-import static org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreUtils.OAK_INDEXER_USE_ZIP;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
@SuppressWarnings("StaticPseudoFunctionalStyleMethod")
public class FlatFileStoreTest {
- static Logger logger = LoggerFactory.getLogger(FlatFileStoreTest.class);
-
private static final String BUILD_TARGET_FOLDER = "target";
@Rule
public TemporaryFolder folder = new TemporaryFolder(new File(BUILD_TARGET_FOLDER));
- private Set<String> preferred = singleton("jcr:content");
- private Predicate<String> pathPredicate = path -> !path.equals("/remove");
-
- private static final String EXCEPTION_MESSAGE = "Framed exception.";
+ private final Set<String> preferred = singleton("jcr:content");
+ private final Predicate<String> pathPredicate = path -> !path.equals("/remove");
private void runBasicTest() throws Exception {
List<String> paths = createTestPaths();
FlatFileNodeStoreBuilder spyBuilder = Mockito.spy(new FlatFileNodeStoreBuilder(folder.getRoot()));
FlatFileStore flatStore = spyBuilder.withBlobStore(new MemoryBlobStore())
.withPreferredPathElements(preferred)
- .withLastModifiedBreakPoints(Collections.singletonList(0L))
.withPathPredicate(pathPredicate)
- .withNodeStateEntryTraverserFactory(new NodeStateEntryTraverserFactory() {
+ .withNodeStateEntryTraverserFactory(range -> new NodeStateEntryTraverser("NS-1", null, null,null, range) {
@Override
- public NodeStateEntryTraverser create(TraversingRange range) {
- return new NodeStateEntryTraverser("NS-1", null, null,
- null, range) {
- @Override
- public @NotNull Iterator<NodeStateEntry> iterator() {
- return TestUtils.createEntries(paths).iterator();
- }
- };
+ public @NotNull Iterator<NodeStateEntry> iterator() {
+ return TestUtils.createEntries(paths).iterator();
}
})
.build();
@@ -130,468 +96,8 @@ public class FlatFileStoreTest {
}
}
- @Test
- public void basicTestMultithreadedTraverseAndSortStrategy() throws Exception {
- try {
- System.setProperty(OAK_INDEXER_SORT_STRATEGY_TYPE, FlatFileNodeStoreBuilder.SortStrategyType.MULTITHREADED_TRAVERSE_WITH_SORT.toString());
- runBasicTest();
- } finally {
- System.clearProperty(OAK_INDEXER_SORT_STRATEGY_TYPE);
- }
- }
-
- @Test
- public void parallelDownload() throws Exception {
- try {
- System.setProperty(OAK_INDEXER_SORT_STRATEGY_TYPE, FlatFileNodeStoreBuilder.SortStrategyType.MULTITHREADED_TRAVERSE_WITH_SORT.toString());
- List<TestMongoDoc> mongoDocs = getTestData();
- List<Long> lmValues = mongoDocs.stream().map(md -> md.lastModified).distinct().sorted().collect(Collectors.toList());
- List<Long> lastModifiedBreakpoints = DocumentStoreSplitter.simpleSplit(lmValues.get(0), lmValues.get(lmValues.size() - 1), 10);
- FlatFileNodeStoreBuilder spyBuilder = Mockito.spy(new FlatFileNodeStoreBuilder(folder.getRoot()));
- FlatFileStore flatStore = spyBuilder.withBlobStore(new MemoryBlobStore())
- .withPreferredPathElements(preferred)
- .withLastModifiedBreakPoints(lastModifiedBreakpoints)
- .withNodeStateEntryTraverserFactory(new TestNodeStateEntryTraverserFactory(mongoDocs))
- .build();
-
- List<String> entryPaths = StreamSupport.stream(flatStore.spliterator(), false)
- .map(NodeStateEntry::getPath)
- .collect(Collectors.toList());
-
- List<String> sortedPaths = TestUtils.sortPaths(mongoDocs.stream().map(md -> md.path).collect(Collectors.toList()));
-
- assertEquals(sortedPaths, entryPaths);
- } finally {
- System.clearProperty(OAK_INDEXER_SORT_STRATEGY_TYPE);
- }
- }
-
- private FlatFileStore buildFlatFileStore(FlatFileNodeStoreBuilder spyBuilder, List<Long> lastModifiedBreakpoints,
- TestNodeStateEntryTraverserFactory nsetf, boolean expectException, Long dumpThreshold) throws Exception {
- boolean exceptionCaught = false;
- FlatFileStore flatFileStore = null;
- try {
- flatFileStore = spyBuilder.withBlobStore(new MemoryBlobStore())
- .withPreferredPathElements(preferred)
- .withLastModifiedBreakPoints(lastModifiedBreakpoints)
- .withNodeStateEntryTraverserFactory(nsetf)
- .withDumpThreshold(dumpThreshold)
- .build();
- } catch (CompositeException e) {
- exceptionCaught = true;
- e.logAllExceptions("Exceptions caught", logger);
- if (expectException) {
- assertEquals(EXCEPTION_MESSAGE, e.getSuppressed()[0].getCause().getMessage());
- }
- }
- assertEquals(expectException, exceptionCaught);
- return flatFileStore;
- }
-
- @Test
- public void resumePreviousUnfinishedDownload() throws Exception {
- Long dumpThreshold = 0L;
- try {
- System.setProperty(OAK_INDEXER_SORT_STRATEGY_TYPE, FlatFileNodeStoreBuilder.SortStrategyType.MULTITHREADED_TRAVERSE_WITH_SORT.toString());
- List<TestMongoDoc> mongoDocs = getTestData();
- List<Long> lmValues = mongoDocs.stream().map(md -> md.lastModified).distinct().sorted().collect(Collectors.toList());
- List<Long> lastModifiedBreakpoints = DocumentStoreSplitter.simpleSplit(lmValues.get(0), lmValues.get(lmValues.size() - 1), 10);
- TestMemoryManager memoryManager = new TestMemoryManager(true);
- FlatFileNodeStoreBuilder spyBuilder = Mockito.spy(new FlatFileNodeStoreBuilder(folder.getRoot(), memoryManager));
- TestNodeStateEntryTraverserFactory nsetf = new TestNodeStateEntryTraverserFactory(mongoDocs);
- nsetf.setDeliveryBreakPoint((int)(mongoDocs.size() * 0.25));
- FlatFileStore flatStore = buildFlatFileStore(spyBuilder, lastModifiedBreakpoints, nsetf, true, dumpThreshold);
- assertNull(flatStore);
- spyBuilder.addExistingDataDumpDir(spyBuilder.getFlatFileStoreDir());
- nsetf.setDeliveryBreakPoint((int)(mongoDocs.size() * 0.50));
- flatStore = buildFlatFileStore(spyBuilder, lastModifiedBreakpoints, nsetf, true, dumpThreshold);
- assertNull(flatStore);
- memoryManager.isMemoryLow = false;
- List<String> entryPaths;
- spyBuilder.addExistingDataDumpDir(spyBuilder.getFlatFileStoreDir());
- nsetf.setDeliveryBreakPoint(Integer.MAX_VALUE);
- flatStore = buildFlatFileStore(spyBuilder, lastModifiedBreakpoints, nsetf, false, dumpThreshold);
- entryPaths = StreamSupport.stream(flatStore.spliterator(), false)
- .map(NodeStateEntry::getPath)
- .collect(Collectors.toList());
-
- List<String> sortedPaths = TestUtils.sortPaths(mongoDocs.stream().map(md -> md.path).collect(Collectors.toList()));
- assertEquals(mongoDocs.size(), nsetf.getTotalProvidedDocCount());
- assertEquals(sortedPaths, entryPaths);
- } finally {
- System.clearProperty(OAK_INDEXER_SORT_STRATEGY_TYPE);
- }
- }
-
- private boolean flatFileStoreMatchCondition(File dir, String filenamePattern, String entry) {
- Pattern pattern = Pattern.compile(filenamePattern, Pattern.CASE_INSENSITIVE);
- for (File innerDir : Objects.requireNonNull(dir.listFiles())) {
- if (innerDir.isDirectory()) {
- for (File innerFile : Objects.requireNonNull(innerDir.listFiles())) {
- if (!pattern.matcher(innerFile.getName()).find()) {
- continue;
- }
- List<String> lines = null;
- try {
- lines = FileUtils.readLines(innerFile, StandardCharsets.UTF_8);
- } catch (Exception e) {
- fail("failed to read FlatFileStore");
- }
- for (String line : lines) {
- if (line.contains(entry)) {
- return true;
- }
- }
- }
- }
- }
- return false;
- }
-
- // with larger size of dump threshold (which result in almost never dump),
- // fail in the middle, check that hidden node progress are saved
- @Test
- public void resumePreviousUnfinishedDownloadWithHiddenNode() throws Exception {
- Long dumpThreshold = FileUtils.ONE_MB;
- try {
- System.setProperty(OAK_INDEXER_SORT_STRATEGY_TYPE, FlatFileNodeStoreBuilder.SortStrategyType.MULTITHREADED_TRAVERSE_WITH_SORT.toString());
- System.setProperty(PROP_THREAD_POOL_SIZE, "1");
- System.setProperty(OAK_INDEXER_USE_ZIP, "false");
- List<TestMongoDoc> mongoDocs = new ArrayList<TestMongoDoc>() {{
- add(new TestMongoDoc("/10-0", 10));
- add(new TestMongoDoc("/10-0/:hidden1", 10));
- add(new TestMongoDoc("/10-0/:hidden2", 10));
- add(new TestMongoDoc("/10-0/:hidden3", 10));
- add(new TestMongoDoc("/10-0/:hidden4", 10));
- add(new TestMongoDoc("/10-1/end", 10));
- }};
- List<Long> lmValues = mongoDocs.stream().map(md -> md.lastModified).distinct().sorted().collect(Collectors.toList());
- List<Long> lastModifiedBreakpoints = DocumentStoreSplitter.simpleSplit(lmValues.get(0), lmValues.get(lmValues.size() - 1), 1);
- TestMemoryManager memoryManager = new TestMemoryManager(true);
- FlatFileNodeStoreBuilder spyBuilder = Mockito.spy(new FlatFileNodeStoreBuilder(folder.getRoot(), memoryManager));
- TestNodeStateEntryTraverserFactory nsetf = new TestNodeStateEntryTraverserFactory(mongoDocs);
- List<String> entryPaths;
- nsetf.setDeliveryBreakPoint(4);
- FlatFileStore flatStore = buildFlatFileStore(spyBuilder, lastModifiedBreakpoints, nsetf, true, dumpThreshold);
- assertNull(flatStore);
- File existingFlatFileStoreDir1 = spyBuilder.getFlatFileStoreDir();
- assertTrue("flatFileStore should dump entry even if exception caught",
- flatFileStoreMatchCondition(existingFlatFileStoreDir1, "flatfile", "/10-0"));
- assertTrue("flatFileStore should save hidden node progress on exception caught",
- flatFileStoreMatchCondition(existingFlatFileStoreDir1, "last-saved","/10-0/:hidden3"));
- nsetf.setDeliveryBreakPoint(Integer.MAX_VALUE);
- spyBuilder.addExistingDataDumpDir(existingFlatFileStoreDir1);
- flatStore = buildFlatFileStore(spyBuilder, lastModifiedBreakpoints, nsetf, false, dumpThreshold);
- entryPaths = StreamSupport.stream(flatStore.spliterator(), false)
- .map(NodeStateEntry::getPath)
- .collect(Collectors.toList());
-
- List<String> sortedPaths = TestUtils.sortPaths(mongoDocs.stream()
- .map(md -> md.path)
- .filter(path -> !NodeStateUtils.isHiddenPath(path))
- .collect(Collectors.toList()));
- assertEquals(mongoDocs.size(), nsetf.getTotalProvidedDocCount());
- assertEquals(sortedPaths, entryPaths);
- } finally {
- System.clearProperty(OAK_INDEXER_SORT_STRATEGY_TYPE);
- System.clearProperty(PROP_THREAD_POOL_SIZE);
- System.clearProperty(OAK_INDEXER_USE_ZIP);
- }
- }
-
- // with larger size of dump threshold (which result in almost never dump),
- // fail in the middle, check that there are data being dumped
- @Test
- public void resumePreviousUnfinishedDownloadWithGracefulDump() throws Exception {
- Long dumpThreshold = FileUtils.ONE_MB;
- try {
- System.setProperty(OAK_INDEXER_SORT_STRATEGY_TYPE, FlatFileNodeStoreBuilder.SortStrategyType.MULTITHREADED_TRAVERSE_WITH_SORT.toString());
- System.setProperty(PROP_THREAD_POOL_SIZE, "1");
- System.setProperty(OAK_INDEXER_USE_ZIP, "false");
- List<TestMongoDoc> mongoDocs = new ArrayList<TestMongoDoc>() {{
- add(new TestMongoDoc("/10-0", 10));
- add(new TestMongoDoc("/10-1", 10));
- }};
- List<Long> lmValues = mongoDocs.stream().map(md -> md.lastModified).distinct().sorted().collect(Collectors.toList());
- List<Long> lastModifiedBreakpoints = DocumentStoreSplitter.simpleSplit(lmValues.get(0), lmValues.get(lmValues.size() - 1), 1);
- TestMemoryManager memoryManager = new TestMemoryManager(true);
- FlatFileNodeStoreBuilder spyBuilder = Mockito.spy(new FlatFileNodeStoreBuilder(folder.getRoot(), memoryManager));
- TestNodeStateEntryTraverserFactory nsetf = new TestNodeStateEntryTraverserFactory(mongoDocs);
- List<String> entryPaths;
- nsetf.setDeliveryBreakPoint(1);
- FlatFileStore flatStore = buildFlatFileStore(spyBuilder, lastModifiedBreakpoints, nsetf, true, dumpThreshold);
- assertNull(flatStore);
- File existingFlatFileStoreDir1 = spyBuilder.getFlatFileStoreDir();
- assertTrue("flatFileStore should dump entry even if exception caught",
- flatFileStoreMatchCondition(existingFlatFileStoreDir1, "flatfile", "/10-0"));
- nsetf.setDeliveryBreakPoint(Integer.MAX_VALUE);
- spyBuilder.addExistingDataDumpDir(existingFlatFileStoreDir1);
- flatStore = buildFlatFileStore(spyBuilder, lastModifiedBreakpoints, nsetf, false, dumpThreshold);
- entryPaths = StreamSupport.stream(flatStore.spliterator(), false)
- .map(NodeStateEntry::getPath)
- .collect(Collectors.toList());
-
- List<String> sortedPaths = TestUtils.sortPaths(mongoDocs.stream().map(md -> md.path).collect(Collectors.toList()));
- assertEquals(mongoDocs.size(), nsetf.getTotalProvidedDocCount());
- assertEquals(sortedPaths, entryPaths);
- } finally {
- System.clearProperty(OAK_INDEXER_SORT_STRATEGY_TYPE);
- System.clearProperty(PROP_THREAD_POOL_SIZE);
- System.clearProperty(OAK_INDEXER_USE_ZIP);
- }
- }
-
- private void assertContainsMergeFolder(File dir, Boolean mustBeEmpty) {
- Boolean mergeFolderExist = false;
- for (File workDir : dir.listFiles()) {
- if (workDir.getName().equals("merge") && workDir.isDirectory()) {
- mergeFolderExist = true;
- if (mustBeEmpty) {
- assertTrue("merge directory should not be empty", workDir.listFiles().length == 0);
- }
- break;
- }
- }
- assertTrue("merge directory should exist", mergeFolderExist);
- }
-
- @Test
- public void resumePreviousUnfinishedDownloadAndMerge() throws Exception {
- Long dumpThreshold = 0L;
- try {
- System.setProperty(OAK_INDEXER_SORT_STRATEGY_TYPE, FlatFileNodeStoreBuilder.SortStrategyType.MULTITHREADED_TRAVERSE_WITH_SORT.toString());
- System.setProperty(PROP_MERGE_TASK_BATCH_SIZE, "2");
- List<TestMongoDoc> mongoDocs = getTestData();
- List<Long> lmValues = mongoDocs.stream().map(md -> md.lastModified).distinct().sorted().collect(Collectors.toList());
- List<Long> lastModifiedBreakpoints = DocumentStoreSplitter.simpleSplit(lmValues.get(0), lmValues.get(lmValues.size() - 1), 10);
- TestMemoryManager memoryManager = new TestMemoryManager(true);
- FlatFileNodeStoreBuilder spyBuilder = Mockito.spy(new FlatFileNodeStoreBuilder(folder.getRoot(), memoryManager));
- TestNodeStateEntryTraverserFactory nsetf = new TestNodeStateEntryTraverserFactory(mongoDocs);
- nsetf.setDeliveryBreakPoint((int)(mongoDocs.size() * 0.50));
- FlatFileStore flatStore = buildFlatFileStore(spyBuilder, lastModifiedBreakpoints, nsetf, true, dumpThreshold);
- assertNull(flatStore);
- File existingFlatFileStoreDir1 = spyBuilder.getFlatFileStoreDir();
- assertContainsMergeFolder(existingFlatFileStoreDir1, false);
- spyBuilder.addExistingDataDumpDir(existingFlatFileStoreDir1);
- nsetf.setDeliveryBreakPoint((int)(mongoDocs.size() * 0.75));
- flatStore = buildFlatFileStore(spyBuilder, lastModifiedBreakpoints, nsetf, true, dumpThreshold);
- assertNull(flatStore);
- memoryManager.isMemoryLow = false;
- List<String> entryPaths;
- File existingFlatFileStoreDir2 = spyBuilder.getFlatFileStoreDir();
- assertContainsMergeFolder(existingFlatFileStoreDir2, false);
- spyBuilder.addExistingDataDumpDir(existingFlatFileStoreDir2);
- nsetf.setDeliveryBreakPoint(Integer.MAX_VALUE);
- flatStore = buildFlatFileStore(spyBuilder, lastModifiedBreakpoints, nsetf, false, dumpThreshold);
- entryPaths = StreamSupport.stream(flatStore.spliterator(), false)
- .map(NodeStateEntry::getPath)
- .collect(Collectors.toList());
-
- // Intermediate MergeFiles should be deleted after being merged
- assertContainsMergeFolder(existingFlatFileStoreDir1, true);
- assertContainsMergeFolder(existingFlatFileStoreDir2, true);
- assertContainsMergeFolder(spyBuilder.getFlatFileStoreDir(), true);
-
- List<String> sortedPaths = TestUtils.sortPaths(mongoDocs.stream().map(md -> md.path).collect(Collectors.toList()));
- // with multi-threaded download and multiple threads,
- // we can't expect that each entry is downloaded exactly once,
- // as there could be some overlap (the range check happends
- // _after_ retrieving the entry)
- assertTrue(mongoDocs.size() <= nsetf.getTotalProvidedDocCount());
- assertEquals(sortedPaths, entryPaths);
- } finally {
- System.clearProperty(OAK_INDEXER_SORT_STRATEGY_TYPE);
- System.clearProperty(PROP_MERGE_TASK_BATCH_SIZE);
- }
- }
-
- private static class TestMemoryManager implements MemoryManager {
-
- boolean isMemoryLow;
-
- public TestMemoryManager(boolean isMemoryLow) {
- this.isMemoryLow = isMemoryLow;
- }
-
- @Override
- public Type getType() {
- return Type.SELF_MANAGED;
- }
-
- @Override
- public boolean isMemoryLow() {
- return isMemoryLow;
- }
-
- @Override
- public void changeMemoryUsedBy(long memory) {
-
- }
-
- @Override
- public boolean deregisterClient(String registrationID) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Optional<String> registerClient(MemoryManagerClient client) {
- throw new UnsupportedOperationException();
- }
- }
-
- private static class TestNodeStateEntryTraverserFactory implements NodeStateEntryTraverserFactory {
-
- final List<TestMongoDoc> mongoDocs;
- /**
- * The traversers will throw exception after these many documents have been returned in total from all traversers
- * created till now from this factory
- */
- final AtomicInteger breakAfterDelivering;
- /**
- * Keeps count of all the node states that have been iterated using all the {@link NodeStateEntryTraverser}s this
- * factory has created till now.
- */
- final AtomicInteger providedDocuments;
-
- public TestNodeStateEntryTraverserFactory(List<TestMongoDoc> mongoDocs) {
- this.mongoDocs = mongoDocs;
- this.breakAfterDelivering = new AtomicInteger(Integer.MAX_VALUE);
- this.providedDocuments = new AtomicInteger(0);
- }
-
- void setDeliveryBreakPoint(int value) {
- breakAfterDelivering.set(value);
- }
-
- @Override
- public NodeStateEntryTraverser create(TraversingRange range) {
- return new NodeStateEntryTraverser("NS-" + range.getLastModifiedRange().getLastModifiedFrom(),
- null, null, null, range) {
- @Override
- public @NotNull Iterator<NodeStateEntry> iterator() {
- List<TestMongoDoc> resultDocs = mongoDocs.stream().filter(doc -> range.getLastModifiedRange().contains(doc.lastModified) &&
- (range.getStartAfterDocumentID() == null || range.getStartAfterDocumentID().compareTo(doc.id) < 0))
- .sorted().collect(Collectors.toList()); // should be sorted in increasing order of (lastModificationTime, id)
- if (resultDocs.isEmpty()) {
- return Collections.emptyIterator();
- }
- Iterator<NodeStateEntry> nodeStateEntryIterator = createEntriesFromMongoDocs(resultDocs).iterator();
- String traverserId = getId();
- return new Iterator<NodeStateEntry>() {
-
- // ensure we don't get a NullPointerException in logger.debug below
- NodeStateEntry lastReturnedDoc = new NodeStateEntryBuilder(null, "/").build();
-
- @Override
- public boolean hasNext() {
- return nodeStateEntryIterator.hasNext();
- }
-
- @Override
- public NodeStateEntry next() {
- // multiple threads could fail at the same time, that's fine
- if (providedDocuments.get() >= breakAfterDelivering.get()) {
- logger.debug("{} Breaking after getting docs with id {}", traverserId, lastReturnedDoc.getId());
- throw new IllegalStateException(EXCEPTION_MESSAGE);
- }
- providedDocuments.incrementAndGet();
- NodeStateEntry next = nodeStateEntryIterator.next();
- lastReturnedDoc = next;
- logger.debug("Returning {} to {} with LM={}", next.getPath(), traverserId, lastReturnedDoc.getLastModified());
- return next;
- }
- };
- }
- };
- }
-
- int getTotalProvidedDocCount() {
- return providedDocuments.get();
- }
- }
private List<String> createTestPaths() {
return asList("/a", "/b", "/c", "/a/b w", "/a/jcr:content", "/a/b", "/", "/b/l", "/remove");
}
-
- static Iterable<NodeStateEntry> createEntriesFromMongoDocs(List<TestMongoDoc> mongoDocs) {
- return Iterables.transform(mongoDocs, d -> new NodeStateEntry.NodeStateEntryBuilder(TestUtils.createNodeState(d.path),d.path)
- .withLastModified(d.lastModified).withID(d.id).build());
- }
-
- static class TestMongoDoc implements Comparable<TestMongoDoc> {
- final String id;
- final String path;
- final long lastModified;
-
- public TestMongoDoc(String path, long lastModified) {
- this.path = path;
- this.lastModified = lastModified;
- int slashCount = 0, fromIndex = 0;
- while ( (fromIndex = path.indexOf("/", fromIndex) + 1) != 0) {
- slashCount++;
- }
- id = slashCount + ":" + path;
- }
-
- @Override
- public int compareTo(@NotNull FlatFileStoreTest.TestMongoDoc o) {
- int mod_comparison = Long.compare(lastModified, o.lastModified);
- if (mod_comparison != 0) {
- return mod_comparison;
- }
- return id.compareTo(o.id);
- }
- }
-
- private List<TestMongoDoc> getTestData() {
- return new ArrayList<TestMongoDoc>() {{
- add(new TestMongoDoc("/content", 10));
- add(new TestMongoDoc("/content/mysite", 20));
- add(new TestMongoDoc("/content/mysite/page1", 30));
- add(new TestMongoDoc("/content/mysite/page2", 30));
- add(new TestMongoDoc("/content/mysite/page3", 30));
- add(new TestMongoDoc("/content/mysite/page4", 30));
- add(new TestMongoDoc("/content/mysite/page5", 30));
- add(new TestMongoDoc("/content/mysite/page6", 30));
- add(new TestMongoDoc("/content/mysite/page1/child1", 40));
- add(new TestMongoDoc("/content/mysite/page2/child1", 40));
- add(new TestMongoDoc("/content/mysite/page3/child1", 40));
- add(new TestMongoDoc("/content/mysite/page4/child1", 40));
- add(new TestMongoDoc("/content/mysite/page5/child1", 40));
- add(new TestMongoDoc("/content/mysite/page6/child1", 40));
- add(new TestMongoDoc("/content/mysite/page1/child2", 80));
- add(new TestMongoDoc("/content/mysite/page2/child2", 80));
- add(new TestMongoDoc("/content/mysite/page3/child2", 80));
- add(new TestMongoDoc("/content/mysite/page4/child2", 80));
- add(new TestMongoDoc("/content/mysite/page5/child2", 80));
- add(new TestMongoDoc("/content/mysite/page6/child2", 80));
- add(new TestMongoDoc("/content/mysite/page1/child3", 120));
- add(new TestMongoDoc("/content/mysite/page2/child3", 120));
- add(new TestMongoDoc("/content/mysite/page3/child3", 120));
- add(new TestMongoDoc("/content/mysite/page4/child3", 120));
- add(new TestMongoDoc("/content/mysite/page5/child3", 120));
- add(new TestMongoDoc("/content/mysite/page6/child3", 120));
- add(new TestMongoDoc("/content/myassets", 20));
- add(new TestMongoDoc("/content/myassets/asset1", 30));
- add(new TestMongoDoc("/content/myassets/asset2", 30));
- add(new TestMongoDoc("/content/myassets/asset3", 30));
- add(new TestMongoDoc("/content/myassets/asset4", 30));
- add(new TestMongoDoc("/content/myassets/asset5", 30));
- add(new TestMongoDoc("/content/myassets/asset6", 30));
- add(new TestMongoDoc("/content/myassets/asset1/jcr:content", 50));
- add(new TestMongoDoc("/content/myassets/asset2/jcr:content", 50));
- add(new TestMongoDoc("/content/myassets/asset3/jcr:content", 50));
- add(new TestMongoDoc("/content/myassets/asset4/jcr:content", 50));
- add(new TestMongoDoc("/content/myassets/asset5/jcr:content", 50));
- add(new TestMongoDoc("/content/myassets/asset6/jcr:content", 50));
- add(new TestMongoDoc("/content/myassets/asset1/jcr:content/metadata", 100));
- add(new TestMongoDoc("/content/myassets/asset2/jcr:content/metadata", 100));
- add(new TestMongoDoc("/content/myassets/asset3/jcr:content/metadata", 100));
- add(new TestMongoDoc("/content/myassets/asset4/jcr:content/metadata", 100));
- add(new TestMongoDoc("/content/myassets/asset5/jcr:content/metadata", 100));
- add(new TestMongoDoc("/content/myassets/asset6/jcr:content/metadata", 100));
-
- }};
- }
}
\ No newline at end of file
diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MergeRunnerTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MergeRunnerTest.java
deleted file mode 100644
index a8822c6405..0000000000
--- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MergeRunnerTest.java
+++ /dev/null
@@ -1,301 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.jackrabbit.oak.index.indexer.document.flatfile;
-
-import org.apache.jackrabbit.guava.common.collect.Lists;
-import org.apache.jackrabbit.oak.commons.Compression;
-import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.slf4j.event.Level;
-import org.apache.commons.io.FileUtils;
-import org.apache.jackrabbit.oak.commons.junit.LogCustomizer;
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.Phaser;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-
-import static org.apache.jackrabbit.guava.common.base.Charsets.UTF_8;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-public class MergeRunnerTest {
- private final LogCustomizer lc = LogCustomizer.forLogger(MergeRunner.class)
- .filter(Level.INFO)
- .enable(Level.INFO)
- .create();
- @Rule
- public TemporaryFolder temporaryFolder = new TemporaryFolder(new File("target"));
- private final String newline = System.lineSeparator();
- private final List<File> testFiles = Lists.newArrayList();
- private final int threadPoolSize = 1,
- batchMergeSize = 3;
- private final PathElementComparator pathComparator = new PathElementComparator();
- private final Comparator<NodeStateHolder> comparator = (e1, e2) -> pathComparator.compare(e1.getPathElements(), e2.getPathElements());
- private final NodeStateEntryWriter entryWriter = new NodeStateEntryWriter(new MemoryBlobStore());
-
- @Before
- public void setup(){
- lc.starting();
- }
-
- @After
- public void after() {
- lc.finished();
- testFiles.clear();
- }
-
- @Test
- public void test() throws Exception {
- lc.starting();
-
- File tmpDir = temporaryFolder.newFolder(),
- mergeDir = new File(tmpDir, "merge-reverse"),
- sortedFile = new File(tmpDir, "sorted-file.json");
- List<String> expectedLogOutput = Lists.newArrayList(),
- actualLogOutput = Lists.newArrayList();
-
- int testFileCount = 13;
- String expectedSortedFileStr = generateTestFiles(tmpDir, testFileCount);
- assertEquals("expected generated test files number does not match", testFileCount, testFiles.size());
-
- BlockingQueue<File> sortedFiles = new LinkedBlockingQueue<>();
- Phaser mergePhaser = new Phaser(1);
- Runnable mergeRunner = new MergeRunner(sortedFile, sortedFiles, mergeDir, comparator, mergePhaser, batchMergeSize, threadPoolSize, Compression.NONE);
- Thread merger = new Thread(mergeRunner, "test-merger");
- merger.setDaemon(true);
-
- sortedFiles.addAll(testFiles);
- merger.start();
- sortedFiles.add(MergeRunner.MERGE_POISON_PILL);
- mergePhaser.awaitAdvance(0);
-
- actualLogOutput = lc.getLogs();
- System.out.println(String.join(newline, actualLogOutput));
- String actualSortedFileStr = FileUtils.readFileToString(sortedFile, UTF_8);
- assertEquals("sorted-file content should be expected", expectedSortedFileStr, actualSortedFileStr);
-
- List<String> intermediateMergeLog = new ArrayList<String>(){
- {
- add("merge complete for intermediate-1");
- add("merge complete for intermediate-2");
- add("merge complete for intermediate-3");
- }
- };
- assertTrue("merge complete output should exist", actualLogOutput.containsAll(intermediateMergeLog));
- actualLogOutput.removeAll(intermediateMergeLog);
-
- expectedLogOutput.add("created merge task for intermediate-1 with " + (new ArrayList<File>(batchMergeSize){
- {
- add(testFiles.get(0));
- add(testFiles.get(1));
- add(testFiles.get(2));
- }
- }));
- expectedLogOutput.add("created merge task for intermediate-2 with " + (new ArrayList<File>(batchMergeSize){
- {
- add(testFiles.get(3));
- add(testFiles.get(4));
- add(testFiles.get(5));
- }
- }));
- expectedLogOutput.add("created merge task for intermediate-3 with " + (new ArrayList<File>(batchMergeSize){
- {
- add(testFiles.get(6));
- add(testFiles.get(7));
- add(testFiles.get(8));
- }
- }));
- expectedLogOutput.add("Waiting for batch sorting tasks completion");
- expectedLogOutput.add("There are still 4 sorted files not merged yet");
- expectedLogOutput.add("running final batch merge task for final-1 with " + (new ArrayList<File>(batchMergeSize){
- {
- add(new File(mergeDir, "intermediate-1"));
- add(testFiles.get(9));
- add(testFiles.get(10));
- }
- }));
- expectedLogOutput.add("running final batch merge task for final-2 with " + (new ArrayList<File>(batchMergeSize){
- {
- add(testFiles.get(11));
- add(testFiles.get(12));
- add(new File(mergeDir, "intermediate-2"));
- }
- }));
- expectedLogOutput.add("running final batch merge task for " + sortedFile.getName() + " with " + (new ArrayList<File>(batchMergeSize){
- {
- add(new File(mergeDir, "intermediate-3"));
- add(new File(mergeDir, "final-1"));
- add(new File(mergeDir, "final-2"));
- }
- }));
- expectedLogOutput.add("Total batch sorted files length is 18");
- assertEquals("final merge log output should be expected", String.join(newline, expectedLogOutput), String.join(newline, actualLogOutput));
- }
-
- @Test
- public void testReverse() throws Exception {
- lc.starting();
-
- File tmpDir = temporaryFolder.newFolder(),
- mergeDir = new File(tmpDir, "merge"),
- sortedFile = new File(tmpDir, "sorted-file.json");
- List<String> expectedLogOutput = Lists.newArrayList(),
- actualLogOutput = Lists.newArrayList();
-
- int testFileCount = 16;
- String expectedSortedFileStr = generateTestFiles(tmpDir, testFileCount);
- assertEquals("expected generated test files number does not match", testFileCount, testFiles.size());
-
- BlockingQueue<File> sortedFiles = new LinkedBlockingQueue<>();
- Phaser mergePhaser = new Phaser(1);
- Runnable mergeRunner = new MergeRunner(sortedFile, sortedFiles, mergeDir, comparator, mergePhaser, batchMergeSize, threadPoolSize, Compression.NONE);
- Thread merger = new Thread(mergeRunner, "test-merger");
- merger.setDaemon(true);
-
- Collections.reverse(testFiles);
- sortedFiles.addAll(testFiles);
- merger.start();
- sortedFiles.add(MergeRunner.MERGE_POISON_PILL);
- mergePhaser.awaitAdvance(0);
-
- actualLogOutput = lc.getLogs();
- System.out.println(String.join(newline, actualLogOutput));
- String actualSortedFileStr = FileUtils.readFileToString(sortedFile, UTF_8);
- assertEquals("sorted-file content should be expected", expectedSortedFileStr, actualSortedFileStr);
-
- Collections.reverse(testFiles);
- List<String> intermediateMergeLog = new ArrayList<String>(){
- {
- add("merge complete for intermediate-1");
- add("merge complete for intermediate-2");
- add("merge complete for intermediate-3");
- add("merge complete for intermediate-4");
- }
- };
- assertTrue("merge complete output should exist", actualLogOutput.containsAll(intermediateMergeLog));
- actualLogOutput.removeAll(intermediateMergeLog);
-
- expectedLogOutput.add("created merge task for intermediate-1 with " + (new ArrayList<File>(batchMergeSize){
- {
- add(testFiles.get(10));
- add(testFiles.get(11));
- add(testFiles.get(12));
- }
- }));
- expectedLogOutput.add("created merge task for intermediate-2 with " + (new ArrayList<File>(batchMergeSize){
- {
- add(testFiles.get(7));
- add(testFiles.get(8));
- add(testFiles.get(9));
- }
- }));
- expectedLogOutput.add("created merge task for intermediate-3 with " + (new ArrayList<File>(batchMergeSize){
- {
- add(testFiles.get(4));
- add(testFiles.get(5));
- add(testFiles.get(6));
- }
- }));
- expectedLogOutput.add("created merge task for intermediate-4 with " + (new ArrayList<File>(batchMergeSize){
- {
- add(testFiles.get(1));
- add(testFiles.get(2));
- add(testFiles.get(3));
- }
- }));
- expectedLogOutput.add("Waiting for batch sorting tasks completion");
- expectedLogOutput.add("There are still 4 sorted files not merged yet");
- expectedLogOutput.add("running final batch merge task for final-1 with " + (new ArrayList<File>(batchMergeSize){
- {
- add(testFiles.get(0));
- add(new File(mergeDir, "intermediate-4"));
- add(testFiles.get(13));
- }
- }));
- expectedLogOutput.add("running final batch merge task for final-2 with " + (new ArrayList<File>(batchMergeSize){
- {
- add(testFiles.get(14));
- add(testFiles.get(15));
- add(new File(mergeDir, "intermediate-3"));
- }
- }));
- expectedLogOutput.add("running final batch merge task for final-3 with " + (new ArrayList<File>(batchMergeSize){
- {
- add(new File(mergeDir, "final-1"));
- add(new File(mergeDir, "intermediate-2"));
- add(new File(mergeDir, "intermediate-1"));
- }
- }));
- expectedLogOutput.add("running final batch merge task for " + sortedFile.getName() + " with " + (new ArrayList<File>(batchMergeSize){
- {
- add(new File(mergeDir, "final-2"));
- add(new File(mergeDir, "final-3"));
- }
- }));
- expectedLogOutput.add("Total batch sorted files length is 23");
- assertEquals("final merge log output should be expected", String.join(newline, expectedLogOutput), String.join(newline, actualLogOutput));
- }
-
- // The function will generate <fileCount> files where the filename equals number of lines in the file.
- private String generateTestFiles(File tmpDir, int fileCount) throws IOException {
- LinkedList<NodeStateHolder> resultNodeState = new LinkedList<>();
- List<Integer> input = IntStream.rangeClosed(1, (1 + fileCount) * fileCount / 2).boxed().collect(Collectors.toList());
- Collections.shuffle(input);
- // cannot have duplicate because externalMerge does not take care of duplicate items
- for (int fname = 1; fname <= fileCount; fname++) {
- LinkedList<NodeStateHolder> tmpNodeState = new LinkedList<>();
- for (int line = 0; line < fname; line++) {
- NodeStateHolder nodeState = new StateInBytesHolder(String.format("/%08d", input.remove(0)), "{}");
- tmpNodeState.add(nodeState);
- }
- tmpNodeState.sort(comparator);
- resultNodeState.addAll(tmpNodeState);
- String testFileContent = "";
- while (!tmpNodeState.isEmpty()) {
- NodeStateHolder h = tmpNodeState.removeFirst();
- String text = entryWriter.toString(h.getPathElements(), h.getLine());
- testFileContent += text + newline;
- }
- File testFile = new File(tmpDir, Integer.toString(fname));
- FileUtils.writeStringToFile(testFile, testFileContent, UTF_8);
- testFiles.add(testFile);
- }
-
- String expectedSortedFileStr = "";
- resultNodeState.sort(comparator);
- while (!resultNodeState.isEmpty()) {
- NodeStateHolder h = resultNodeState.removeFirst();
- String text = entryWriter.toString(h.getPathElements(), h.getLine());
- expectedSortedFileStr += text + newline;
- }
- return expectedSortedFileStr;
- }
-}
\ No newline at end of file
diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MultithreadedTraverseWithSortStrategyTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MultithreadedTraverseWithSortStrategyTest.java
deleted file mode 100644
index c084de0d76..0000000000
--- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MultithreadedTraverseWithSortStrategyTest.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.jackrabbit.oak.index.indexer.document.flatfile;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.jackrabbit.oak.commons.Compression;
-import org.apache.jackrabbit.oak.index.indexer.document.LastModifiedRange;
-import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntryTraverserFactory;
-import org.apache.jackrabbit.oak.plugins.document.mongo.TraversingRange;
-import org.apache.jackrabbit.oak.spi.blob.BlobStore;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.List;
-import java.util.concurrent.ConcurrentLinkedQueue;
-
-import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.MultithreadedTraverseWithSortStrategy.DirectoryHelper;
-import static org.junit.Assert.assertEquals;
-
-public class MultithreadedTraverseWithSortStrategyTest {
-
- @Test
- public void initialRanges() throws IOException {
- List<Long> lastModifiedBreakpoints = Arrays.asList(10L, 20L, 30L, 40L);
- List<TraversingRange> ranges = new ArrayList<>();
- MultithreadedTraverseWithSortStrategy mtws = new MultithreadedTraverseWithSortStrategy(null,
- lastModifiedBreakpoints, null, null, null, null, Compression.GZIP, null,
- FlatFileNodeStoreBuilder.OAK_INDEXER_DUMP_THRESHOLD_IN_MB_DEFAULT * FileUtils.ONE_MB, path -> true) {
-
- @Override
- void addTask(TraversingRange range, NodeStateEntryTraverserFactory nodeStateEntryTraverserFactory, BlobStore blobStore, ConcurrentLinkedQueue<String> completedTasks) throws IOException {
- ranges.add(range);
- }
- };
- assertEquals(lastModifiedBreakpoints.size(), ranges.size());
- for (int i = 0; i < lastModifiedBreakpoints.size(); i++) {
- long lm = lastModifiedBreakpoints.get(i);
- LastModifiedRange lmRange = new LastModifiedRange(lm, i < lastModifiedBreakpoints.size() - 1 ? lastModifiedBreakpoints.get(i + 1) : lm + 1);
- assertEquals(ranges.get(i), new TraversingRange(lmRange, null));
- }
- }
-
- static class Execution {
- String taskID;
- long lastModLowerBound;
- long lastModUpperBound;
- long lastDownloadedLastMod;
- String lastDownloadedID;
- boolean completed;
-
- public Execution(String taskID, long lastModLowerBound, long lastModUpperBound, long lastDownloadedLastMod,
- String lastDownloadedID, boolean completed) {
- this.taskID = taskID;
- this.lastModLowerBound = lastModLowerBound;
- this.lastModUpperBound = lastModUpperBound;
- this.lastDownloadedLastMod = lastDownloadedLastMod;
- this.lastDownloadedID = lastDownloadedID;
- this.completed = completed;
- }
- }
-
- private void createSortWorkDir(File workDir, Execution execution) throws IOException {
- File dir = DirectoryHelper.createdSortWorkDir(workDir, execution.taskID, execution.lastModLowerBound,
- execution.lastModUpperBound);
- if (execution.completed) {
- DirectoryHelper.markCompleted(dir);
- } else if (execution.lastDownloadedID != null) {
- DirectoryHelper.markLastProcessedStatus(dir, execution.lastDownloadedLastMod, execution.lastDownloadedID);
- }
- }
-
- @Test
- public void rangesDuringResume() throws IOException {
- List<Execution> previousRun = new ArrayList<Execution>() {{
- add(new Execution("1", 10, 20, -1, null, true));
- add(new Execution("2", 20, 30, 22, "1:/content", false));
- add(new Execution("3", 30, 40, 34, "2:/sites/mypage", false));
- }};
- List<File> workDirs = new ArrayList<>();
- File workDir = new File("target/" + this.getClass().getSimpleName() + "-" + System.currentTimeMillis());
- for (Execution execution : previousRun) {
- createSortWorkDir(workDir, execution);
- }
- workDirs.add(workDir);
- List<TraversingRange> ranges = new ArrayList<>();
- MultithreadedTraverseWithSortStrategy mtws = new MultithreadedTraverseWithSortStrategy(null,
- null, null, null, null, workDirs, Compression.GZIP, null,
- FlatFileNodeStoreBuilder.OAK_INDEXER_DUMP_THRESHOLD_IN_MB_DEFAULT * FileUtils.ONE_MB, path -> true) {
- @Override
- void addTask(TraversingRange range, NodeStateEntryTraverserFactory nodeStateEntryTraverserFactory,
- BlobStore blobStore, ConcurrentLinkedQueue<String> completedTasks) throws IOException {
- ranges.add(range);
- }
- };
- ranges.sort(Comparator.comparing(tr -> tr.getLastModifiedRange().getLastModifiedFrom()));
- List<TraversingRange> expectedRanges = new ArrayList<TraversingRange>() {{
- add(new TraversingRange(new LastModifiedRange(22, 23), "1:/content"));
- add(new TraversingRange(new LastModifiedRange(23, 30), null));
- add(new TraversingRange(new LastModifiedRange(34, 35), "2:/sites/mypage"));
- add(new TraversingRange(new LastModifiedRange(35, 40), null));
- }};
- assertEquals(expectedRanges.size(), ranges.size());
- for (int i = 0; i < expectedRanges.size(); i++) {
- assertEquals(expectedRanges.get(i), ranges.get(i));
- }
- }
-
-}
diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/TraverseAndSortTaskTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/TraverseAndSortTaskTest.java
deleted file mode 100644
index afe1d9caf2..0000000000
--- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/TraverseAndSortTaskTest.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.jackrabbit.oak.index.indexer.document.flatfile;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.jackrabbit.oak.commons.Compression;
-import org.apache.jackrabbit.oak.index.indexer.document.LastModifiedRange;
-import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry;
-import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntryTraverser;
-import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntryTraverserFactory;
-import org.apache.jackrabbit.oak.plugins.document.mongo.TraversingRange;
-import org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState;
-import org.junit.Test;
-import org.mockito.Mockito;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Queue;
-import java.util.concurrent.Callable;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.Phaser;
-
-import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.MultithreadedTraverseWithSortStrategy.DirectoryHelper;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.when;
-
-public class TraverseAndSortTaskTest {
-
- private static class NodeStateEntryTraverserFactoryImpl implements NodeStateEntryTraverserFactory {
-
- @Override
- public NodeStateEntryTraverser create(TraversingRange range) {
- return new NodeStateEntryTraverser("Test-NSET", null, null, null, range);
- }
- }
-
- @Test
- public void taskSplit() throws IOException {
-
- LastModifiedRange lmRange = new LastModifiedRange(0, 10);
- TraversingRange traversingRange = new TraversingRange(lmRange, null);
-
- Phaser phaser = Mockito.mock(Phaser.class);
- when(phaser.register()).thenReturn(1);
-
- MemoryManager mockMemManager = Mockito.mock(MemoryManager.class);
- when(mockMemManager.isMemoryLow()).thenReturn(false);
-
- Queue<Callable<List<File>>> newTaskQueue = new LinkedList<>();
- File store = new File("target/" + this.getClass().getSimpleName() + "-" + System.currentTimeMillis());
- TraverseAndSortTask tst = new TraverseAndSortTask(traversingRange, null, null, store, Compression.GZIP,
- new LinkedList<>(Collections.singletonList("1")), newTaskQueue, phaser, new NodeStateEntryTraverserFactoryImpl(), mockMemManager,
- FlatFileNodeStoreBuilder.OAK_INDEXER_DUMP_THRESHOLD_IN_MB_DEFAULT * FileUtils.ONE_MB, new LinkedBlockingQueue<File>(), path -> true);
-
- NodeStateEntry mockEntry = Mockito.mock(NodeStateEntry.class);
- long lastModified = (lmRange.getLastModifiedFrom() + lmRange.getLastModifiedTo())/2;
- when(mockEntry.getLastModified()).thenReturn(lastModified);
- when(mockEntry.getPath()).thenReturn("/content");
- when(mockEntry.getNodeState()).thenReturn(EmptyNodeState.EMPTY_NODE);
- assertEquals(lmRange.getLastModifiedTo(), DirectoryHelper.getLastModifiedUpperLimit(tst.getSortWorkDir()));
- tst.addEntry(mockEntry);
- long newUpperLimit = DirectoryHelper.getLastModifiedUpperLimit(tst.getSortWorkDir());
- assertTrue(newUpperLimit > lastModified);
- assertTrue(newUpperLimit < lmRange.getLastModifiedTo());
- assertEquals(1, newTaskQueue.size());
- TraverseAndSortTask tst2 = (TraverseAndSortTask)newTaskQueue.remove();
- assertEquals(newUpperLimit, tst2.getLastModifiedLowerBound());
- assertEquals(lmRange.getLastModifiedTo(), tst2.getLastModifiedUpperBound());
- }
-
-}