You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by th...@apache.org on 2022/03/24 16:30:04 UTC
[jackrabbit-oak] branch trunk updated: OAK-9738 Merge indexing files in parallel and during download
This is an automated email from the ASF dual-hosted git repository.
thomasm pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new 7fa7ad5 OAK-9738 Merge indexing files in parallel and during download
7fa7ad5 is described below
commit 7fa7ad542d4e115357c672d1e7a898dd434c9487
Author: Thomas Mueller <th...@apache.org>
AuthorDate: Thu Mar 24 17:29:54 2022 +0100
OAK-9738 Merge indexing files in parallel and during download
---
.../indexer/document/flatfile/MergeRunner.java | 26 +++++++++++++++++-----
.../indexer/document/flatfile/MergeRunnerTest.java | 20 +++++++++++++++--
2 files changed, 38 insertions(+), 8 deletions(-)
diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MergeRunner.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MergeRunner.java
index 043e9f4..5a6f2e7 100644
--- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MergeRunner.java
+++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MergeRunner.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
package org.apache.jackrabbit.oak.index.indexer.document.flatfile;
import com.google.common.collect.Lists;
@@ -24,10 +42,6 @@ import java.util.concurrent.Phaser;
import java.util.function.Function;
import static com.google.common.base.Charsets.UTF_8;
-import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileNodeStoreBuilder.DEFAULT_NUMBER_OF_MERGE_TASK_THREADS;
-import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileNodeStoreBuilder.PROP_MERGE_THREAD_POOL_SIZE;
-import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileNodeStoreBuilder.DEFAULT_NUMBER_OF_FILES_PER_MERGE_TASK;
-import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileNodeStoreBuilder.PROP_MERGE_TASK_BATCH_SIZE;
import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileStoreUtils.createWriter;
@@ -96,7 +110,7 @@ public class MergeRunner implements Runnable {
private ExecutorService executorService;
private final int threadPoolSize;
private final int batchMergeSize;
- private final Comparator fileSizeComparator = new SizeFileComparator();
+ private final Comparator<? super File> fileSizeComparator = new SizeFileComparator();
/**
* The end result file after merging all sorted files.
@@ -133,7 +147,7 @@ public class MergeRunner implements Runnable {
* @param mergeDir directory where sorted files will be created.
* @param compressionEnabled if true, the created files would be compressed
*/
- MergeRunner(File sortedFile, BlockingQueue<File> sortedFiles, File mergeDir, Comparator comparator,
+ MergeRunner(File sortedFile, BlockingQueue<File> sortedFiles, File mergeDir, Comparator<NodeStateHolder> comparator,
Phaser phaser, int batchMergeSize, int threadPoolSize, boolean compressionEnabled) {
this.mergeDir = mergeDir;
this.compressionEnabled = compressionEnabled;
diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MergeRunnerTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MergeRunnerTest.java
index 6ee5795..ff26223 100644
--- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MergeRunnerTest.java
+++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MergeRunnerTest.java
@@ -1,7 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
package org.apache.jackrabbit.oak.index.indexer.document.flatfile;
import com.google.common.collect.Lists;
-import org.apache.commons.io.comparator.SizeFileComparator;
import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore;
import org.junit.After;
import org.junit.Before;
@@ -37,7 +54,6 @@ public class MergeRunnerTest {
batchMergeSize = 3;
private final PathElementComparator pathComparator = new PathElementComparator();
private final Comparator<NodeStateHolder> comparator = (e1, e2) -> pathComparator.compare(e1.getPathElements(), e2.getPathElements());
- private final Comparator fileSizeComparator = new SizeFileComparator();
private final NodeStateEntryWriter entryWriter = new NodeStateEntryWriter(new MemoryBlobStore());
@Before