You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by th...@apache.org on 2024/12/02 16:39:08 UTC
(jackrabbit-oak) 01/01: OAK-11291 Tree store: speed up merging, and concurrent indexing
This is an automated email from the ASF dual-hosted git repository.
thomasm pushed a commit to branch OAK-11291
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit 82039113bb4a12d4553e1d67da2ac4fd7eef077a
Author: Thomas Mueller <th...@apache.org>
AuthorDate: Mon Dec 2 17:38:51 2024 +0100
OAK-11291 Tree store: speed up merging, and concurrent indexing
---
.../oak/plugins/index/lucene/writer/DefaultIndexWriter.java | 13 +------------
.../document/flatfile/pipelined/PipelinedTreeStoreTask.java | 2 +-
2 files changed, 2 insertions(+), 13 deletions(-)
diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriter.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriter.java
index 5f6cff1ffb..b080d73140 100644
--- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriter.java
+++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriter.java
@@ -145,12 +145,7 @@ class DefaultIndexWriter implements LuceneIndexWriter {
PERF_LOGGER.end(start, -1, "Completed suggester for directory {}", definition);
}
- if (reindex && writerConfig.getThreadCount() > 1) {
- // merges could be disabled, so it could in theory wait forever
- writer.close(false);
- } else {
- writer.close();
- }
+ writer.close();
PERF_LOGGER.end(start, -1, "Closed writer for directory {}", definition);
if (!indexUpdated){
@@ -175,12 +170,6 @@ class DefaultIndexWriter implements LuceneIndexWriter {
final long start = PERF_LOGGER.start();
directory = directoryFactory.newInstance(definition, definitionBuilder, dirName, reindex);
boolean serialScheduler = directoryFactory.remoteDirectory();
- if (writerConfig.getThreadCount() > 1) {
- // for multi-threaded indexing (using oak-run), use the serial scheduler,
- // to avoid concurrency issues with the concurrent merge policy
- log.info("Using the serial scheduler for parallel indexing");
- serialScheduler = true;
- }
IndexWriterConfig config = getIndexWriterConfig(definition, serialScheduler, writerConfig);
config.setMergePolicy(definition.getMergePolicy());
writer = localRefWriter = new IndexWriter(directory, config);
diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java
index 257852aaf0..d8688eb4e2 100644
--- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java
+++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java
@@ -52,7 +52,7 @@ public class PipelinedTreeStoreTask implements Callable<PipelinedSortBatchTask.R
private static final Logger LOG = LoggerFactory.getLogger(PipelinedTreeStoreTask.class);
private static final String THREAD_NAME = "tree-store-task";
- private static final int MERGE_BATCH = 10;
+ private static final int MERGE_BATCH = 25;
private static final boolean SKIP_FINAL_MERGE = false;
private final TreeStore treeStore;