You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by am...@apache.org on 2023/01/23 06:09:20 UTC
[jackrabbit-oak] branch trunk updated: OAK-10048: DocumentStoreIndexerBase#buildFlatFileStore outputs the wrong path when FlatFileSplitter used (#826)
This is an automated email from the ASF dual-hosted git repository.
amitj pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new aada6bd4b0 OAK-10048: DocumentStoreIndexerBase#buildFlatFileStore outputs the wrong path when FlatFileSplitter used (#826)
aada6bd4b0 is described below
commit aada6bd4b0888677b850b5ffd151b079507a0851
Author: Amit Jain <am...@apache.org>
AuthorDate: Mon Jan 23 11:39:13 2023 +0530
OAK-10048: DocumentStoreIndexerBase#buildFlatFileStore outputs the wrong path when FlatFileSplitter used (#826)
- System property (oak.indexer.sortedFilePath) defines the folder of the flat file store rather than the current specific file path
- Enable buildFlatFiles to use the folder in the system property defined as the place to look for files
- In case the folder is split then it uses that to init the split flat file store
---
.../indexer/document/DocumentStoreIndexerBase.java | 3 +-
.../flatfile/FlatFileNodeStoreBuilder.java | 82 +++++---
.../document/flatfile/FlatFileSplitter.java | 5 +-
.../indexer/document/flatfile/FlatFileStore.java | 2 +-
.../flatfile/FlatFileNodeStoreBuilderTest.java | 213 +++++++++++++++++----
5 files changed, 238 insertions(+), 67 deletions(-)
diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java
index 90478a2db1..974b6524f8 100644
--- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java
+++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java
@@ -217,7 +217,8 @@ public abstract class DocumentStoreIndexerBase implements Closeable{
preferredPathElements.addAll(indexDf.getRelativeNodeNames());
}
Predicate<String> predicate = s -> indexDefinitions.stream().anyMatch(indexDef -> indexDef.getPathFilter().filter(s) != PathFilter.Result.EXCLUDE);
- FlatFileStore flatFileStore = buildFlatFileStoreList(checkpointedState, null, predicate, preferredPathElements, false, indexDefinitions).get(0);
+ FlatFileStore flatFileStore = buildFlatFileStoreList(checkpointedState, null, predicate,
+ preferredPathElements, IndexerConfiguration.parallelIndexEnabled(), indexDefinitions).get(0);
log.info("FlatFileStore built at {}. To use this flatFileStore in a reindex step, set System Property-{} with value {}",
flatFileStore.getFlatFileStorePath(), OAK_INDEXER_SORTED_FILE_PATH, flatFileStore.getFlatFileStorePath());
return flatFileStore;
diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java
index 8cae86fa1a..177bab3446 100644
--- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java
+++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java
@@ -21,6 +21,7 @@ package org.apache.jackrabbit.oak.index.indexer.document.flatfile;
import com.google.common.collect.Iterables;
import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.jackrabbit.oak.commons.Compression;
import org.apache.jackrabbit.oak.index.IndexHelper;
import org.apache.jackrabbit.oak.index.IndexerSupport;
@@ -31,6 +32,7 @@ import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore;
import org.apache.jackrabbit.oak.query.NodeStateNodeTypeInfoProvider;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -38,6 +40,7 @@ import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Set;
@@ -65,6 +68,9 @@ public class FlatFileNodeStoreBuilder {
* Allowed values are the values from enum {@link SortStrategyType}
*/
public static final String OAK_INDEXER_SORT_STRATEGY_TYPE = "oak.indexer.sortStrategyType";
+ /**
+ * System property to define the existing folder containing the flat file store files
+ */
public static final String OAK_INDEXER_SORTED_FILE_PATH = "oak.indexer.sortedFilePath";
/**
@@ -191,7 +197,8 @@ public class FlatFileNodeStoreBuilder {
logFlags();
comparator = new PathElementComparator(preferredPathElements);
entryWriter = new NodeStateEntryWriter(blobStore);
- FlatFileStore store = new FlatFileStore(blobStore, createdSortedStoreFile(), new NodeStateEntryReader(blobStore),
+ FlatFileStore store = new FlatFileStore(blobStore, createdSortedStoreFiles().get(0),
+ new NodeStateEntryReader(blobStore),
unmodifiableSet(preferredPathElements), algorithm);
if (entryCount > 0) {
store.setEntryCount(entryCount);
@@ -205,13 +212,18 @@ public class FlatFileNodeStoreBuilder {
comparator = new PathElementComparator(preferredPathElements);
entryWriter = new NodeStateEntryWriter(blobStore);
- File flatStoreFile = createdSortedStoreFile();
+ List<File> fileList = createdSortedStoreFiles();
+
long start = System.currentTimeMillis();
- NodeStore nodeStore = new MemoryNodeStore(indexerSupport.retrieveNodeStateForCheckpoint());
- FlatFileSplitter splitter = new FlatFileSplitter(flatStoreFile, indexHelper.getWorkDir(), new NodeStateNodeTypeInfoProvider(nodeStore.getRoot()), new NodeStateEntryReader(blobStore),
+ // If not already split, split otherwise skip splitting
+ if (!fileList.stream().allMatch(FlatFileSplitter.IS_SPLIT)) {
+ NodeStore nodeStore = new MemoryNodeStore(indexerSupport.retrieveNodeStateForCheckpoint());
+ FlatFileSplitter splitter = new FlatFileSplitter(fileList.get(0), indexHelper.getWorkDir(),
+ new NodeStateNodeTypeInfoProvider(nodeStore.getRoot()), new NodeStateEntryReader(blobStore),
indexDefinitions);
- List<File> fileList = splitter.split();
- log.info("Split flat file to result files '{}' is done, took {} ms", fileList, System.currentTimeMillis() - start);
+ fileList = splitter.split();
+ log.info("Split flat file to result files '{}' is done, took {} ms", fileList, System.currentTimeMillis() - start);
+ }
List<FlatFileStore> storeList = new ArrayList<>();
for (File flatFileItem : fileList) {
@@ -222,26 +234,52 @@ public class FlatFileNodeStoreBuilder {
return storeList;
}
- private File createdSortedStoreFile() throws IOException, CompositeException {
+ /**
+ * Returns the existing list of store files if it can read from system property OAK_INDEXER_SORTED_FILE_PATH which
+ * defines the existing folder where the flat file store files are present. Will throw an exception if it cannot
+ * read or the path in the system property is not a directory.
+ * If the system property OAK_INDEXER_SORTED_FILE_PATH in undefined, or it cannot read relevant files it
+ * initializes the flat file store.
+ *
+ * @return list of flat files
+ * @throws IOException
+ * @throws CompositeException
+ */
+ private List<File> createdSortedStoreFiles() throws IOException, CompositeException {
+ // Check system property defined path
String sortedFilePath = System.getProperty(OAK_INDEXER_SORTED_FILE_PATH);
- if (sortedFilePath != null) {
- File sortedFile = new File(sortedFilePath);
- if (sortedFile.exists() && sortedFile.isFile() && sortedFile.canRead()) {
- log.info("Reading from provided sorted file [{}] (via system property '{}')",
- sortedFile.getAbsolutePath(), OAK_INDEXER_SORTED_FILE_PATH);
- return sortedFile;
- } else {
- String msg = String.format("Cannot read sorted file at [%s] configured via system property '%s'",
- sortedFile.getAbsolutePath(), OAK_INDEXER_SORTED_FILE_PATH);
- throw new IllegalArgumentException(msg);
+ if (StringUtils.isNotBlank(sortedFilePath)) {
+ File sortedDir = new File(sortedFilePath);
+ log.info("Attempting to read from provided sorted files directory [{}] (via system property '{}')",
+ sortedDir.getAbsolutePath(), OAK_INDEXER_SORTED_FILE_PATH);
+ List<File> files = getFiles(sortedDir);
+ if (files != null) {
+ return files;
+ }
+ }
+
+ // Initialize the flat file store again
+
+ createStoreDir();
+ SortStrategy strategy = createSortStrategy(flatFileStoreDir);
+ File result = strategy.createSortedStoreFile();
+ entryCount = strategy.getEntryCount();
+ return Collections.singletonList(result);
+ }
+
+ @Nullable
+ private List<File> getFiles(File sortedDir) {
+ if (sortedDir.exists() && sortedDir.canRead() && sortedDir.isDirectory()) {
+ File[] files = sortedDir.listFiles(
+ (dir, name) -> name.endsWith(FlatFileStoreUtils.getSortedStoreFileName(algorithm)));
+ if (files != null && files.length != 0) {
+ return Arrays.asList(files);
}
} else {
- createStoreDir();
- org.apache.jackrabbit.oak.index.indexer.document.flatfile.SortStrategy strategy = createSortStrategy(flatFileStoreDir);
- File result = strategy.createSortedStoreFile();
- entryCount = strategy.getEntryCount();
- return result;
+ String msg = String.format("Cannot read sorted files directory at [%s]", sortedDir.getAbsolutePath());
+ throw new IllegalArgumentException(msg);
}
+ return null;
}
SortStrategy createSortStrategy(File dir) throws IOException {
diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileSplitter.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileSplitter.java
index c99db1fbd6..663fcd4de9 100644
--- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileSplitter.java
+++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileSplitter.java
@@ -44,6 +44,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.Stack;
+import java.util.function.Predicate;
import java.util.stream.Collectors;
import static org.apache.jackrabbit.JcrConstants.JCR_PRIMARYTYPE;
@@ -73,7 +74,9 @@ public class FlatFileSplitter {
private Set<String> splitNodeTypeNames;
private boolean useCompression = Boolean.parseBoolean(System.getProperty(OAK_INDEXER_USE_ZIP, "true"));
private boolean useLZ4 = Boolean.parseBoolean(System.getProperty(OAK_INDEXER_USE_LZ4, "false"));
-
+
+ static Predicate<File> IS_SPLIT = path -> path.getParent().endsWith(SPLIT_DIR_NAME);
+
public FlatFileSplitter(File flatFile, File workdir, NodeTypeInfoProvider infoProvider, NodeStateEntryReader entryReader,
Set<IndexDefinition> indexDefinitions) {
this.flatFile = flatFile;
diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java
index 5c4b675416..5c31d0323f 100644
--- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java
+++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java
@@ -57,7 +57,7 @@ public class FlatFileStore implements Iterable<NodeStateEntry>, Closeable {
}
public String getFlatFileStorePath() {
- return storeFile.getAbsolutePath();
+ return storeFile.getParentFile().getAbsolutePath();
}
public long getEntryCount() {
diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilderTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilderTest.java
index 8fcbb74252..acd655192a 100644
--- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilderTest.java
+++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilderTest.java
@@ -18,23 +18,47 @@
*/
package org.apache.jackrabbit.oak.index.indexer.document.flatfile;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
import java.io.File;
import java.io.IOException;
import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
+import org.apache.commons.io.IOUtils;
+import org.apache.jackrabbit.oak.InitialContent;
+import org.apache.jackrabbit.oak.OakInitializer;
+import org.apache.jackrabbit.oak.commons.Compression;
import org.apache.jackrabbit.oak.index.IndexHelper;
import org.apache.jackrabbit.oak.index.IndexerSupport;
import org.apache.jackrabbit.oak.index.indexer.document.CompositeException;
+import org.apache.jackrabbit.oak.index.indexer.document.IndexerConfiguration;
import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntryTraverserFactory;
+import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition;
+import org.apache.jackrabbit.oak.plugins.index.search.util.IndexDefinitionBuilder;
+import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore;
+import org.apache.jackrabbit.oak.plugins.name.NamespaceEditorProvider;
+import org.apache.jackrabbit.oak.plugins.nodetype.TypeEditorProvider;
+import org.apache.jackrabbit.oak.query.ast.NodeTypeInfo;
+import org.apache.jackrabbit.oak.query.ast.NodeTypeInfoProvider;
+import org.apache.jackrabbit.oak.spi.commit.CompositeEditorProvider;
+import org.apache.jackrabbit.oak.spi.commit.EditorHook;
import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
import org.junit.Rule;
import org.junit.Test;
+import org.junit.contrib.java.lang.system.RestoreSystemProperties;
import org.junit.rules.TemporaryFolder;
+import org.junit.rules.TestRule;
+import org.mockito.Mockito;
import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileNodeStoreBuilder.OAK_INDEXER_SORTED_FILE_PATH;
import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileNodeStoreBuilder.OAK_INDEXER_SORT_STRATEGY_TYPE;
import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileNodeStoreBuilder.OAK_INDEXER_TRAVERSE_WITH_SORT;
+import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileNodeStoreBuilder.OAK_INDEXER_USE_LZ4;
+import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileNodeStoreBuilder.OAK_INDEXER_USE_ZIP;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
@@ -47,6 +71,9 @@ public class FlatFileNodeStoreBuilderTest {
@Rule
public TemporaryFolder folder = new TemporaryFolder(new File(BUILD_TARGET_FOLDER));
+ @Rule
+ public final TestRule restoreSystemProperties = new RestoreSystemProperties();
+
private final NodeStateEntryTraverserFactory nodeStateEntryTraverserFactory = range -> null;
@Test
@@ -60,60 +87,162 @@ public class FlatFileNodeStoreBuilderTest {
@Test
public void sortStrategyBasedOnSystemProperty() throws Exception {
- try {
- System.setProperty(OAK_INDEXER_SORT_STRATEGY_TYPE, FlatFileNodeStoreBuilder.SortStrategyType.TRAVERSE_WITH_SORT.toString());
- FlatFileNodeStoreBuilder builder = new FlatFileNodeStoreBuilder(folder.getRoot())
- .withNodeStateEntryTraverserFactory(nodeStateEntryTraverserFactory);
- SortStrategy sortStrategy = builder.createSortStrategy(builder.createStoreDir());
- assertTrue(sortStrategy instanceof TraverseWithSortStrategy);
- } finally {
- System.clearProperty(OAK_INDEXER_SORT_STRATEGY_TYPE);
- }
+ System.setProperty(OAK_INDEXER_SORT_STRATEGY_TYPE, FlatFileNodeStoreBuilder.SortStrategyType.TRAVERSE_WITH_SORT.toString());
+ FlatFileNodeStoreBuilder builder = new FlatFileNodeStoreBuilder(folder.getRoot())
+ .withNodeStateEntryTraverserFactory(nodeStateEntryTraverserFactory);
+ SortStrategy sortStrategy = builder.createSortStrategy(builder.createStoreDir());
+ assertTrue(sortStrategy instanceof TraverseWithSortStrategy);
}
@Test
public void disableTraverseAndSortStrategyUsingSystemProperty() throws Exception {
- try {
- System.setProperty(OAK_INDEXER_TRAVERSE_WITH_SORT, "false");
- FlatFileNodeStoreBuilder builder = new FlatFileNodeStoreBuilder(folder.getRoot())
- .withNodeStateEntryTraverserFactory(nodeStateEntryTraverserFactory);
- SortStrategy sortStrategy = builder.createSortStrategy(builder.createStoreDir());
- assertTrue(sortStrategy instanceof StoreAndSortStrategy);
- } finally {
- System.clearProperty(OAK_INDEXER_TRAVERSE_WITH_SORT);
- }
+ System.setProperty(OAK_INDEXER_TRAVERSE_WITH_SORT, "false");
+ FlatFileNodeStoreBuilder builder = new FlatFileNodeStoreBuilder(folder.getRoot())
+ .withNodeStateEntryTraverserFactory(nodeStateEntryTraverserFactory);
+ SortStrategy sortStrategy = builder.createSortStrategy(builder.createStoreDir());
+ assertTrue(sortStrategy instanceof StoreAndSortStrategy);
}
@Test
public void sortStrategySystemPropertyPrecedence() throws Exception {
- try {
- System.setProperty(OAK_INDEXER_TRAVERSE_WITH_SORT, "false");
- System.setProperty(OAK_INDEXER_SORT_STRATEGY_TYPE, FlatFileNodeStoreBuilder.SortStrategyType.TRAVERSE_WITH_SORT.toString());
- FlatFileNodeStoreBuilder builder = new FlatFileNodeStoreBuilder(folder.getRoot())
- .withNodeStateEntryTraverserFactory(nodeStateEntryTraverserFactory);
- SortStrategy sortStrategy = builder.createSortStrategy(builder.createStoreDir());
- assertTrue(sortStrategy instanceof TraverseWithSortStrategy);
- } finally {
- System.clearProperty(OAK_INDEXER_SORT_STRATEGY_TYPE);
- System.clearProperty(OAK_INDEXER_TRAVERSE_WITH_SORT);
- }
+ System.setProperty(OAK_INDEXER_TRAVERSE_WITH_SORT, "false");
+ System.setProperty(OAK_INDEXER_SORT_STRATEGY_TYPE, FlatFileNodeStoreBuilder.SortStrategyType.TRAVERSE_WITH_SORT.toString());
+ FlatFileNodeStoreBuilder builder = new FlatFileNodeStoreBuilder(folder.getRoot())
+ .withNodeStateEntryTraverserFactory(nodeStateEntryTraverserFactory);
+ SortStrategy sortStrategy = builder.createSortStrategy(builder.createStoreDir());
+ assertTrue(sortStrategy instanceof TraverseWithSortStrategy);
+ }
+
+ @Test
+ public void testBuild() throws CompositeException, IOException {
+ System.setProperty(OAK_INDEXER_USE_ZIP, "false");
+ File newFlatFile = getFile("simple-split.json", Compression.NONE);
+
+ System.setProperty(OAK_INDEXER_SORTED_FILE_PATH, newFlatFile.getParentFile().getAbsolutePath());
+ assertBuild(newFlatFile.getParentFile().getAbsolutePath());
+ }
+
+ @Test
+ public void testBuildGZIP() throws CompositeException, IOException {
+ System.setProperty(OAK_INDEXER_USE_ZIP, "true");
+ File newFlatFile = getFile("simple-split.json", Compression.GZIP);
+ System.setProperty(OAK_INDEXER_SORTED_FILE_PATH, newFlatFile.getParentFile().getAbsolutePath());
+
+ assertBuild(newFlatFile.getParentFile().getAbsolutePath());
+ }
+
+ @Test
+ public void testBuildLZ4() throws CompositeException, IOException {
+ System.setProperty(OAK_INDEXER_USE_ZIP, "true");
+ System.setProperty(OAK_INDEXER_USE_LZ4, "true");
+ LZ4Compression compression = new LZ4Compression();
+
+ File newFlatFile = getFile("simple-split.json", compression);
+ System.setProperty(OAK_INDEXER_SORTED_FILE_PATH, newFlatFile.getParentFile().getAbsolutePath());
+
+ assertBuild(newFlatFile.getParentFile().getAbsolutePath());
+ }
+
+ @Test
+ public void testBuildListNoSplit() throws CompositeException, IOException {
+ System.setProperty(OAK_INDEXER_USE_ZIP, "false");
+
+ File newFlatFile = getFile("complex-split.json", Compression.NONE);
+ System.setProperty(OAK_INDEXER_SORTED_FILE_PATH, newFlatFile.getParentFile().getAbsolutePath());
+
+ assertBuildList(newFlatFile.getParentFile().getAbsolutePath(), false);
+ }
+
+ @Test
+ public void testBuildListSplit() throws CompositeException, IOException {
+ System.setProperty(OAK_INDEXER_USE_ZIP, "false");
+ System.setProperty(IndexerConfiguration.PROP_OAK_INDEXER_MIN_SPLIT_THRESHOLD, "0");
+
+ File newFlatFile = getFile("complex-split.json", Compression.NONE);
+ System.setProperty(OAK_INDEXER_SORTED_FILE_PATH, newFlatFile.getParentFile().getAbsolutePath());
+
+ assertBuildList(newFlatFile.getParentFile().getAbsolutePath(), true);
+ }
+
+ @Test
+ public void testBuildListSplitGZIP() throws CompositeException, IOException {
+ System.setProperty(OAK_INDEXER_USE_ZIP, "true");
+ System.setProperty(IndexerConfiguration.PROP_OAK_INDEXER_MIN_SPLIT_THRESHOLD, "0");
+
+ File newFlatFile = getFile("complex-split.json", Compression.GZIP);
+ System.setProperty(OAK_INDEXER_SORTED_FILE_PATH, newFlatFile.getParentFile().getAbsolutePath());
+
+ assertBuildList(newFlatFile.getParentFile().getAbsolutePath(), true);
}
@Test
- public void testBuildList() throws CompositeException, IOException {
- try {
- File flatFile = new File(getClass().getClassLoader().getResource("simple-split.json").getFile());
- System.setProperty(OAK_INDEXER_SORTED_FILE_PATH, flatFile.getAbsolutePath());
- FlatFileNodeStoreBuilder builder = new FlatFileNodeStoreBuilder(folder.getRoot()).withNodeStateEntryTraverserFactory(
- nodeStateEntryTraverserFactory);
- IndexHelper indexHelper = mock(IndexHelper.class);
- IndexerSupport indexerSupport = mock(IndexerSupport.class);
- NodeState rootState = mock(NodeState.class);
- when(indexerSupport.retrieveNodeStateForCheckpoint()).thenReturn(rootState);
- List<FlatFileStore> storeList = builder.buildList(indexHelper, indexerSupport, null);
+ public void testBuildListSplitLZ4() throws CompositeException, IOException {
+ System.setProperty(OAK_INDEXER_USE_ZIP, "true");
+ System.setProperty(OAK_INDEXER_USE_LZ4, "true");
+ System.setProperty(IndexerConfiguration.PROP_OAK_INDEXER_MIN_SPLIT_THRESHOLD, "0");
+ LZ4Compression compression = new LZ4Compression();
+
+ File newFlatFile = getFile("complex-split.json", compression);
+ System.setProperty(OAK_INDEXER_SORTED_FILE_PATH, newFlatFile.getParentFile().getAbsolutePath());
+
+ assertBuildList(newFlatFile.getParentFile().getAbsolutePath(), true);
+ }
+
+ public void assertBuild(String dir) throws CompositeException, IOException {
+ FlatFileNodeStoreBuilder builder = new FlatFileNodeStoreBuilder(folder.getRoot()).withNodeStateEntryTraverserFactory(
+ nodeStateEntryTraverserFactory);
+ FlatFileStore store = builder.build();
+ assertEquals(dir, store.getFlatFileStorePath());
+ }
+
+ private File getFile(String dataFile, Compression compression) throws IOException {
+ File flatFile = new File(getClass().getClassLoader().getResource(dataFile).getFile());
+ File newFlatFile = new File(folder.getRoot(), FlatFileStoreUtils.getSortedStoreFileName(compression));
+ try (BufferedReader reader = FlatFileStoreUtils.createReader(flatFile, false);
+ BufferedWriter writer = FlatFileStoreUtils.createWriter(newFlatFile, compression)) {
+ IOUtils.copy(reader, writer);
+ }
+ return newFlatFile;
+ }
+
+ public void assertBuildList(String dir, boolean split) throws CompositeException, IOException {
+ FlatFileNodeStoreBuilder builder = new FlatFileNodeStoreBuilder(folder.getRoot()).withNodeStateEntryTraverserFactory(
+ nodeStateEntryTraverserFactory);
+ IndexHelper indexHelper = mock(IndexHelper.class);
+ when(indexHelper.getWorkDir()).thenReturn(new File(dir));
+ IndexerSupport indexerSupport = mock(IndexerSupport.class);
+ NodeState rootState = mock(NodeState.class);
+ when(indexerSupport.retrieveNodeStateForCheckpoint()).thenReturn(rootState);
+
+ List<FlatFileStore> storeList = builder.buildList(indexHelper, indexerSupport, mockIndexDefns());
+
+ if (split) {
+ assertEquals(new File(dir, "split").getAbsolutePath(), storeList.get(0).getFlatFileStorePath());
+ assertTrue(storeList.size() > 1);
+ } else {
+ assertEquals(dir, storeList.get(0).getFlatFileStorePath());
assertEquals(1, storeList.size());
- } finally {
- System.clearProperty(OAK_INDEXER_SORTED_FILE_PATH);
}
}
+
+ private static Set<IndexDefinition> mockIndexDefns() {
+ NodeStore store = new MemoryNodeStore();
+ EditorHook hook = new EditorHook(
+ new CompositeEditorProvider(new NamespaceEditorProvider(), new TypeEditorProvider()));
+ OakInitializer.initialize(store, new InitialContent(), hook);
+
+ Set<IndexDefinition> defns = new HashSet<>();
+ IndexDefinitionBuilder defnBuilder = new IndexDefinitionBuilder();
+ defnBuilder.indexRule("dam:Asset");
+ defnBuilder.aggregateRule("dam:Asset");
+ IndexDefinition defn = IndexDefinition.newBuilder(store.getRoot(), defnBuilder.build(), "/foo").build();
+ defns.add(defn);
+
+ NodeTypeInfoProvider mockNodeTypeInfoProvider = Mockito.mock(NodeTypeInfoProvider.class);
+ NodeTypeInfo mockNodeTypeInfo = Mockito.mock(NodeTypeInfo.class, "dam:Asset");
+ Mockito.when(mockNodeTypeInfo.getNodeTypeName()).thenReturn("dam:Asset");
+ Mockito.when(mockNodeTypeInfoProvider.getNodeTypeInfo("dam:Asset")).thenReturn(mockNodeTypeInfo);
+
+ return defns;
+ }
}