You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by nf...@apache.org on 2024/03/05 13:54:43 UTC
(jackrabbit-oak) branch trunk updated: OAK-10681 - [indexing job] Support custom filters of paths on Mongo (#1340)
This is an automated email from the ASF dual-hosted git repository.
nfsantos pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new d2531b0b6c OAK-10681 - [indexing job] Support custom filters of paths on Mongo (#1340)
d2531b0b6c is described below
commit d2531b0b6c5e8bf033bc846685e0f45dc5a6d25c
Author: Nuno Santos <ns...@adobe.com>
AuthorDate: Tue Mar 5 14:54:37 2024 +0100
OAK-10681 - [indexing job] Support custom filters of paths on Mongo (#1340)
- Add property to specify list of paths to be excluded from Mongo download: oak.indexer.pipelined.mongoCustomExcludedPaths.
- Simplify the regex expressions to filter on paths, remove the trailing .*$ at the end.
---
.../pipelined/MongoRegexPathFilterFactory.java | 56 +++++++++--
.../pipelined/PipelinedMongoDownloadTask.java | 37 +++++++-
.../pipelined/MongoRegexPathFilterFactoryTest.java | 102 +++++++++++++++++++++
.../document/flatfile/pipelined/PipelinedIT.java | 73 +++++++++++++++
.../pipelined/PipelinedMongoDownloadTaskTest.java | 38 ++++++--
5 files changed, 286 insertions(+), 20 deletions(-)
diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/MongoRegexPathFilterFactory.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/MongoRegexPathFilterFactory.java
index 299c7b19c8..c289c286c2 100644
--- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/MongoRegexPathFilterFactory.java
+++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/MongoRegexPathFilterFactory.java
@@ -24,6 +24,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.TreeSet;
@@ -76,11 +78,21 @@ public class MongoRegexPathFilterFactory {
}
public MongoFilterPaths buildMongoFilter(List<PathFilter> pathFilters) {
+ return buildMongoFilter(pathFilters, List.of());
+
+ }
+
+ /**
+ * @param pathFilters Filters representing the included/excludedPaths settings of the indexes
+ * @param customExcludedPaths Additional excluded paths set in the Oak configuration.
+ * @return Paths to include and exclude in the Mongo query
+ */
+ public MongoFilterPaths buildMongoFilter(List<PathFilter> pathFilters, List<String> customExcludedPaths) {
if (pathFilters == null || pathFilters.isEmpty()) {
return MongoFilterPaths.DOWNLOAD_ALL;
}
// Merge and deduplicate the included and excluded paths of all filters.
- // The included paths will also be further de-deuplicated, by removing paths that are children of other path in
+ // The included paths will also be further de-deuplicated, by removing paths that are children of other paths in
// the list, that is, by keeping only the top level paths. To make it easy to de-duplicate, we sort the paths
// alphabetically.
// The list of excluded paths does not need to be sorted, but we do it anyway for consistency when logging the
@@ -123,6 +135,14 @@ public class MongoRegexPathFilterFactory {
return MongoFilterPaths.DOWNLOAD_ALL;
}
+ if (customExcludedPaths.stream().anyMatch(PathUtils::denotesRoot)) {
+ LOG.warn("Ignoring custom excluded paths setting, root cannot be excluded: {}", customExcludedPaths);
+ } else if (!isRootPath(finalIncludedPathsRoots)) {
+ LOG.info("Ignoring custom excluded paths because included paths did not resolve to root. Mongo filters: {}", finalIncludedPathsRoots);
+ } else {
+ finalExcludedPaths = mergeIndexAndCustomExcludePaths(finalExcludedPaths, customExcludedPaths);
+ }
+
// Also check the number of excluded paths and disable filtering by them if there are too many.
// We do not need to revert to download the full repository in the case where there is a reasonable number of
// included paths but too many excluded paths. The excluded paths are always children of the included paths, so
@@ -134,16 +154,32 @@ public class MongoRegexPathFilterFactory {
finalExcludedPaths = List.of();
}
- if (finalExcludedPaths.isEmpty()) {
- // Only included paths
- if (finalIncludedPathsRoots.contains("/")) {
- return MongoFilterPaths.DOWNLOAD_ALL;
- } else {
- return new MongoFilterPaths(finalIncludedPathsRoots, List.of());
+ return new MongoFilterPaths(finalIncludedPathsRoots, finalExcludedPaths);
+ }
+
+ private boolean isRootPath(List<String> includedPaths) {
+ return includedPaths.isEmpty() || (includedPaths.size() == 1 && includedPaths.get(0).equals("/"));
+ }
+
+ /*
+ * Merge the excluded paths defined by the indexes with the custom excluded paths. This is done by taking the union
+ * of the two lists and then removing any path that is a subpath of any other path in the list.
+ */
+ static List<String> mergeIndexAndCustomExcludePaths(List<String> indexExcludedPaths, List<String> customExcludedPaths) {
+ if (customExcludedPaths.isEmpty()) {
+ return indexExcludedPaths;
+ }
+
+ var excludedUnion = new HashSet<>(indexExcludedPaths);
+ excludedUnion.addAll(customExcludedPaths);
+ var mergedExcludes = new ArrayList<String>();
+ for (String testPath : excludedUnion) {
+ // Add a path only if it is not a subpath of any other path in the list
+ if (excludedUnion.stream().noneMatch(p -> PathUtils.isAncestor(p, testPath))) {
+ mergedExcludes.add(testPath);
}
- } else {
- return new MongoFilterPaths(finalIncludedPathsRoots, finalExcludedPaths);
}
+ Collections.sort(mergedExcludes);
+ return mergedExcludes;
}
-
}
diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMongoDownloadTask.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMongoDownloadTask.java
index d9276d9a69..af81b21782 100644
--- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMongoDownloadTask.java
+++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMongoDownloadTask.java
@@ -126,6 +126,13 @@ public class PipelinedMongoDownloadTask implements Callable<PipelinedMongoDownlo
public static final String OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING_MAX_PATHS = "oak.indexer.pipelined.mongoRegexPathFilteringMaxPaths";
public static final int DEFAULT_OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING_MAX_PATHS = 20;
+ /**
+ * Additional Oak paths to exclude from downloading from Mongo. This is a comma-separated list of paths.
+ * These paths are only filtered if the included paths computed from the indexes resolve to the root tree (/),
+ * otherwise the value of this property is ignored.
+ */
+ public static final String OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDED_PATHS = "oak.indexer.pipelined.mongoCustomExcludedPaths";
+ public static final String DEFAULT_OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDED_PATHS = "";
// Use a short initial retry interval. In most cases if the connection to a replica fails, there will be other
// replicas available so a reconnection attempt will succeed immediately.
@@ -137,7 +144,7 @@ public class PipelinedMongoDownloadTask implements Callable<PipelinedMongoDownlo
private static final int MIN_INTERVAL_BETWEEN_DELAYED_ENQUEUING_MESSAGES = 10;
private final static BsonDocument NATURAL_HINT = BsonDocument.parse("{ $natural: 1 }");
private final static BsonDocument ID_INDEX_HINT = BsonDocument.parse("{ _id: 1 }");
- final static Pattern LONG_PATH_ID_PATTERN = Pattern.compile("^[0-9]{1,3}:h.*$");
+ final static Pattern LONG_PATH_ID_PATTERN = Pattern.compile("^[0-9]{1,3}:h");
private static final String THREAD_NAME = "mongo-dump";
@@ -209,7 +216,7 @@ public class PipelinedMongoDownloadTask implements Callable<PipelinedMongoDownlo
path = path + "/";
}
String quotedPath = Pattern.quote(path);
- patterns.add(Pattern.compile("^[0-9]{1,3}:" + quotedPath + ".*$"));
+ patterns.add(Pattern.compile("^[0-9]{1,3}:" + quotedPath));
}
return patterns;
}
@@ -267,6 +274,7 @@ public class PipelinedMongoDownloadTask implements Callable<PipelinedMongoDownlo
private final IndexingReporter reporter;
private final MongoRegexPathFilterFactory regexPathFilterFactory;
private final String customExcludeEntriesRegex;
+ private final List<String> customExcludedPaths;
private long totalEnqueueWaitTimeMillis = 0;
private Instant lastDelayedEnqueueWarningMessageLoggedTimestamp = Instant.now();
@@ -328,6 +336,29 @@ public class PipelinedMongoDownloadTask implements Callable<PipelinedMongoDownlo
);
this.reporter.addConfig(OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDE_ENTRIES_REGEX, customExcludeEntriesRegex);
+ var excludePathsString = ConfigHelper.getSystemPropertyAsString(
+ OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDED_PATHS,
+ DEFAULT_OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDED_PATHS
+ ).trim();
+ this.reporter.addConfig(OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDED_PATHS, excludePathsString);
+ if (excludePathsString.isEmpty()) {
+ this.customExcludedPaths = List.of();
+ } else if (!regexPathFiltering) {
+ LOG.info("Ignoring custom excluded paths because regex path filtering is disabled");
+ this.customExcludedPaths = List.of();
+ } else {
+ this.customExcludedPaths = Arrays.stream(excludePathsString.split(","))
+ .map(String::trim)
+ .collect(Collectors.toList());
+ }
+ var invalidPaths = customExcludedPaths.stream()
+ .filter(p -> !PathUtils.isValid(p) || !PathUtils.isAbsolute(p) || PathUtils.denotesRoot(p))
+ .collect(Collectors.toList());
+ if (!invalidPaths.isEmpty()) {
+ throw new IllegalArgumentException("Invalid paths in " + OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDED_PATHS + " " +
+ " system property: " + invalidPaths + ". Paths must be valid, must be absolute and must not be the root.");
+ }
+
//TODO This may lead to reads being routed to secondary depending on MongoURI
//So caller must ensure that its safe to read from secondary
// this.readPreference = MongoDocumentStoreHelper.getConfiguredReadPreference(mongoStore, collection);
@@ -525,7 +556,7 @@ public class PipelinedMongoDownloadTask implements Callable<PipelinedMongoDownlo
.map(pf -> "PF{includedPaths=" + pf.getIncludedPaths() + ", excludedPaths=" + pf.getExcludedPaths() + "}")
.collect(Collectors.joining(", "))
);
- MongoFilterPaths mongoFilterPaths = this.regexPathFilterFactory.buildMongoFilter(pathFilters);
+ MongoFilterPaths mongoFilterPaths = this.regexPathFilterFactory.buildMongoFilter(pathFilters, customExcludedPaths);
LOG.info("Paths used for regex filtering on Mongo: {}", mongoFilterPaths);
return mongoFilterPaths;
}
diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/MongoRegexPathFilterFactoryTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/MongoRegexPathFilterFactoryTest.java
new file mode 100644
index 0000000000..8075f90722
--- /dev/null
+++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/MongoRegexPathFilterFactoryTest.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined;
+
+import org.apache.jackrabbit.oak.spi.filter.PathFilter;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import static org.junit.Assert.*;
+
+public class MongoRegexPathFilterFactoryTest {
+
+ private static MongoRegexPathFilterFactory filter;
+ PathFilter fullTree = new PathFilter(Set.of("/"), Set.of());
+ PathFilter multipleExcludes = new PathFilter(Set.of("/"), Set.of("/excluded/sub1/sub2", "/excluded2", "/tmp"));
+ PathFilter singleInclude = new PathFilter(Set.of("/included/sub"), Set.of());
+
+ @BeforeClass
+ public static void init() {
+ filter = new MongoRegexPathFilterFactory(PipelinedMongoDownloadTask.DEFAULT_OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING_MAX_PATHS);
+ }
+
+ @Test
+ public void includeRootSingleExclude() {
+ var f = filter.buildMongoFilter(List.of(fullTree), List.of("/excluded"));
+ assertPaths(List.of("/"), List.of("/excluded"), f);
+ }
+
+ @Test
+ public void includeRootMultipleExclude() {
+ var f = filter.buildMongoFilter(List.of(fullTree), List.of("/excluded1", "/excluded2"));
+ assertPaths(List.of("/"), List.of("/excluded1", "/excluded2"), f);
+ }
+
+
+ @Test
+ public void includeRootMultipleExcludeSubtree() {
+ var f = filter.buildMongoFilter(List.of(fullTree), List.of("/excluded", "/excluded/sub"));
+ assertPaths(List.of("/"), List.of("/excluded"), f);
+ }
+
+ @Test
+ public void customExcludeRootFails() {
+ var f = filter.buildMongoFilter(List.of(fullTree), List.of("/"));
+ assertPaths(List.of("/"), List.of(), f);
+ }
+
+ @Test
+ public void customExcludeOutsideIncludeTree() {
+ var f = filter.buildMongoFilter(List.of(singleInclude), List.of("/excluded"));
+ assertPaths(List.of("/included/sub"), List.of(), f);
+ }
+
+ @Test
+ public void customExcludeEqualsToIncludeTree() {
+ var f = filter.buildMongoFilter(List.of(singleInclude), List.of("/included/sub"));
+ assertPaths(List.of("/included/sub"), List.of(), f);
+ }
+
+ @Test
+ public void customExcludeInsideIncludeTree() {
+ var f = filter.buildMongoFilter(List.of(singleInclude), List.of("/included/sub/sub2"));
+ assertPaths(List.of("/included/sub"), List.of(), f);
+ }
+
+ @Test
+ public void customExcludeChildrenOfIndexExcludes() {
+ var f = filter.buildMongoFilter(List.of(multipleExcludes), List.of("/excluded2", "/tmp/bin"));
+ assertPaths(List.of("/"), multipleExcludes.getExcludedPaths(), f);
+ }
+
+ @Test
+ public void customExcludeParentOfIndexExcludes() {
+ var f = filter.buildMongoFilter(List.of(multipleExcludes), List.of("/excluded"));
+ assertPaths(List.of("/"), List.of("/excluded", "/excluded2", "/tmp"), f);
+ }
+
+ private static void assertPaths(List<String> expectedIncluded, List<String> expectedExcluded, MongoRegexPathFilterFactory.MongoFilterPaths actual) {
+ assertEquals(new HashSet<>(expectedIncluded), new HashSet<>(actual.included));
+ assertEquals(new HashSet<>(expectedExcluded), new HashSet<>(actual.excluded));
+ }
+}
diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedIT.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedIT.java
index 30ab0a3d8a..0c1ec406db 100644
--- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedIT.java
+++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedIT.java
@@ -71,6 +71,7 @@ import java.util.function.Predicate;
import java.util.stream.Collectors;
import static java.lang.management.ManagementFactory.getPlatformMBeanServer;
+import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMongoDownloadTask.OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDED_PATHS;
import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMongoDownloadTask.OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDE_ENTRIES_REGEX;
import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMongoDownloadTask.OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING;
import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMongoDownloadTask.OAK_INDEXER_PIPELINED_RETRY_ON_CONNECTION_ERRORS;
@@ -619,6 +620,78 @@ public class PipelinedIT {
}
+ @Test
+ public void createFFS_mongoFiltering_custom_excluded_paths_1() throws Exception {
+ System.setProperty(OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING, "true");
+ System.setProperty(OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDED_PATHS, "/etc,/home");
+
+ Predicate<String> pathPredicate = s -> true;
+ List<PathFilter> pathFilters = List.of(new PathFilter(List.of("/"), List.of("/content/dam", "/etc", "/home", "/jcr:system")));
+
+ testSuccessfulDownload(pathPredicate, pathFilters, List.of(
+ "/|{}",
+ "/content|{}",
+ "/content/dam|{}",
+ "/etc|{}",
+ "/home|{}",
+ "/jcr:system|{}"
+ ), true);
+ }
+
+ @Test
+ public void createFFS_mongoFiltering_custom_excluded_paths_2() throws Exception {
+ System.setProperty(OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING, "true");
+ System.setProperty(OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDED_PATHS, "/etc,/home");
+
+ Predicate<String> pathPredicate = s -> true;
+ List<PathFilter> pathFilters = List.of(new PathFilter(List.of("/"), List.of("/content/dam", "/jcr:system")));
+
+ testSuccessfulDownload(pathPredicate, pathFilters, List.of(
+ "/|{}",
+ "/content|{}",
+ "/content/dam|{}",
+ "/etc|{}",
+ "/home|{}",
+ "/jcr:system|{}"
+ ), true);
+ }
+
+ @Test
+ public void createFFS_mongoFiltering_custom_excluded_paths_3() throws Exception {
+ System.setProperty(OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING, "true");
+ System.setProperty(OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDED_PATHS, "/etc,/home,/content/dam,/jcr:system");
+
+ Predicate<String> pathPredicate = s -> true;
+ List<PathFilter> pathFilters = List.of(new PathFilter(List.of("/"), List.of()));
+
+ testSuccessfulDownload(pathPredicate, pathFilters, List.of(
+ "/|{}",
+ "/content|{}",
+ "/content/dam|{}",
+ "/etc|{}",
+ "/home|{}",
+ "/jcr:system|{}"
+ ), true);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void createFFS_mongoFiltering_custom_excluded_paths_cannot_exclude_root() throws Exception {
+ System.setProperty(OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING, "true");
+ System.setProperty(OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDED_PATHS, "/etc,/");
+ Predicate<String> pathPredicate = s -> true;
+ List<PathFilter> pathFilters = List.of(new PathFilter(List.of("/"), List.of()));
+
+ testSuccessfulDownload(pathPredicate, pathFilters, List.of(
+ "/|{}",
+ "/content|{}",
+ "/content/dam|{}",
+ "/etc|{}",
+ "/home|{}",
+ "/jcr:system|{}"
+ ), true);
+ }
+
+
@Ignore("This test is for manual execution only. It allocates two byte buffers of 2GB each, which might exceed the memory available in the CI")
public void createFFSWithPipelinedStrategy_veryLargeWorkingMemorySetting() throws Exception {
System.setProperty(PipelinedStrategy.OAK_INDEXER_PIPELINED_TRANSFORM_THREADS, "1");
diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMongoDownloadTaskTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMongoDownloadTaskTest.java
index 3e87830560..6515b5ab0c 100644
--- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMongoDownloadTaskTest.java
+++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMongoDownloadTaskTest.java
@@ -330,8 +330,8 @@ public class PipelinedMongoDownloadTaskTest {
// The generated filter should not include any condition to include the descendants of /
var expected =
Filters.nin(NodeDocument.ID,
- Pattern.compile("^[0-9]{1,3}:" + Pattern.quote("/excluded1/") + ".*$"),
- Pattern.compile("^[0-9]{1,3}:" + Pattern.quote("/content/excluded2/") + ".*$")
+ Pattern.compile("^[0-9]{1,3}:" + Pattern.quote("/excluded1/")),
+ Pattern.compile("^[0-9]{1,3}:" + Pattern.quote("/content/excluded2/"))
);
assertBsonEquals(expected, actual);
}
@@ -345,7 +345,7 @@ public class PipelinedMongoDownloadTaskTest {
null
);
var expected = Filters.in(NodeDocument.ID,
- Pattern.compile("^[0-9]{1,3}:" + Pattern.quote("/parent/") + ".*$"),
+ Pattern.compile("^[0-9]{1,3}:" + Pattern.quote("/parent/")),
LONG_PATH_ID_PATTERN
);
assertBsonEquals(expected, actual);
@@ -371,7 +371,7 @@ public class PipelinedMongoDownloadTaskTest {
Pattern excludesPattern = Pattern.compile("^[0-9]{1,3}:/a/b.*$");
var expected =
Filters.and(
- Filters.in(NodeDocument.ID, Pattern.compile("^[0-9]{1,3}:" + Pattern.quote("/parent/") + ".*$"), LONG_PATH_ID_PATTERN),
+ Filters.in(NodeDocument.ID, Pattern.compile("^[0-9]{1,3}:" + Pattern.quote("/parent/")), LONG_PATH_ID_PATTERN),
Filters.nin(NodeDocument.ID, excludesPattern)
);
assertBsonEquals(expected, actual);
@@ -388,7 +388,7 @@ public class PipelinedMongoDownloadTaskTest {
var expected =
Filters.and(
Filters.in(NodeDocument.ID,
- Pattern.compile("^[0-9]{1,3}:" + Pattern.quote("/parent/") + ".*$"),
+ Pattern.compile("^[0-9]{1,3}:" + Pattern.quote("/parent/")),
LONG_PATH_ID_PATTERN
),
Filters.nin(NodeDocument.ID, excludePattern)
@@ -405,7 +405,7 @@ public class PipelinedMongoDownloadTaskTest {
Pattern excludePattern = Pattern.compile("^[0-9]{1,3}:/a/b.*$");
var expected =
- Filters.nin(NodeDocument.ID, Pattern.compile("^[0-9]{1,3}:" + Pattern.quote("/excluded/") + ".*$"), excludePattern);
+ Filters.nin(NodeDocument.ID, Pattern.compile("^[0-9]{1,3}:" + Pattern.quote("/excluded/")), excludePattern);
assertBsonEquals(expected, actual);
}
@@ -418,7 +418,7 @@ public class PipelinedMongoDownloadTaskTest {
Pattern excludePattern = Pattern.compile("^[0-9]{1,3}:/a/b.*$");
var expected = Filters.nin(NodeDocument.ID,
- Pattern.compile("^[0-9]{1,3}:" + Pattern.quote("/excluded/") + ".*$"),
+ Pattern.compile("^[0-9]{1,3}:" + Pattern.quote("/excluded/")),
excludePattern
);
assertBsonEquals(expected, actual);
@@ -435,5 +435,29 @@ public class PipelinedMongoDownloadTaskTest {
expected.toBsonDocument(BsonDocument.class, MongoClient.getDefaultCodecRegistry())
);
}
+
+ @Test
+ public void mergeExcludedPaths() {
+ // Indexes do not define any exclude, the custom index adds one exclude
+ testMergeExcludedPaths(List.of(), List.of("/b"), List.of("/b"));
+ // Indexes define one exclude, the custom index adds one exclude
+ testMergeExcludedPaths(List.of("/a"), List.of("/b"), List.of("/a", "/b"));
+ // Adds an exclude already in the list
+ testMergeExcludedPaths(List.of("/a"), List.of("/a"), List.of("/a"));
+ // Adds an exclude that is a child of an existing exclude
+ testMergeExcludedPaths(List.of("/a"), List.of("/a/s"), List.of("/a"));
+ // Adds an exclude that is a parent of an existing exclude
+ testMergeExcludedPaths(List.of("/a/s"), List.of("/a"), List.of("/a"));
+ testMergeExcludedPaths(List.of("/a/s"), List.of(), List.of("/a/s"));
+ testMergeExcludedPaths(List.of("/a", "/b/b1", "/c/c1/c2"), List.of("/d/d1/d2", "/b", "/c/c1/c2/c3"),
+ List.of("/a", "/b", "/c/c1/c2", "/d/d1/d2"));
+ testMergeExcludedPaths(List.of("/a/s"), List.of("/"), List.of("/"));
+
+ }
+
+ private void testMergeExcludedPaths(List<String> indexExcludedPaths, List<String> customExcludedPaths, List<String> expected) {
+ var mergedExcludedPaths = MongoRegexPathFilterFactory.mergeIndexAndCustomExcludePaths(indexExcludedPaths, customExcludedPaths);
+ assertEquals(expected, mergedExcludedPaths);
+ }
}