You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by th...@apache.org on 2026/10/09 07:26:01 UTC
(jackrabbit-oak) branch trunk updated: OAK-12441 Simplified index mgmt: support multiple json and external files (#3198)
This is an automated email from the ASF dual-hosted git repository.
thomasmueller pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new f150236972 OAK-12441 Simplified index mgmt: support multiple json and external files (#3198)
f150236972 is described below
commit f150236972abc8c792d7a42ba48c51a28bf3ad3a
Author: Thomas Mueller <th...@apache.org>
AuthorDate: Fri Oct 9 09:25:55 2026 +0200
OAK-12441 Simplified index mgmt: support multiple json and external files (#3198)
* OAK-12441 Simplified index mgmt: support multiple json and external files
* OAK-12441 Simplified index mgmt: support multiple json and external files
* OAK-12441 Simplified index mgmt: support multiple json and external files
* OAK-12441 Simplified index mgmt: support multiple json and external files
* OAK-12441 Simplified index mgmt: support multiple json and external files
* OAK-12441 Simplified index mgmt: support multiple json and external files
* OAK-12441 Simplified index mgmt: support multiple json and external files
---
.../oak/plugins/index/diff/DiffIndex.java | 152 ++++++-
.../oak/plugins/index/diff/DiffIndexMerger.java | 174 ++++++--
.../plugins/index/diff/DiffIndexLegacyTest.java | 233 ++++++++++
.../plugins/index/diff/DiffIndexMergerTest.java | 478 ++++++++++++++++++++-
.../oak/plugins/index/diff/DiffIndexTest.java | 387 ++++++++++++++++-
5 files changed, 1391 insertions(+), 33 deletions(-)
diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java
index abf83d6f98..19d7326a03 100644
--- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java
+++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java
@@ -21,19 +21,30 @@ import java.io.InputStream;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
+import java.util.Base64;
import java.util.Comparator;
+import java.util.HashMap;
import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
import org.apache.jackrabbit.oak.api.PropertyState;
import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.commons.PathUtils;
import org.apache.jackrabbit.oak.commons.json.JsonObject;
+import org.apache.jackrabbit.oak.commons.json.JsopBuilder;
+import org.apache.jackrabbit.oak.osgi.OsgiWhiteboard;
import org.apache.jackrabbit.oak.plugins.index.IndexConstants;
import org.apache.jackrabbit.oak.plugins.index.IndexName;
import org.apache.jackrabbit.oak.plugins.tree.TreeConstants;
import org.apache.jackrabbit.oak.spi.nodetype.NodeTypeConstants;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.apache.jackrabbit.oak.spi.toggle.Feature;
+import org.osgi.framework.BundleContext;
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Deactivate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -44,12 +55,46 @@ import org.slf4j.LoggerFactory;
* (applied) to the index definitions. This allows to simplify index management,
* because it allows to modify (add, update) indexes in a simple way.
*/
+@Component(service = {})
public class DiffIndex {
private static final Logger LOG = LoggerFactory.getLogger(DiffIndex.class);
private final static DiffIndexMerger MERGER = new DiffIndexMerger();
+ public static final String LEGACY_DIFF_INDEX_TOGGLE = "FT_LEGACY_DIFF_INDEX_OAK-12441";
+
+ private static final boolean LEGACY_DIFF_INDEX = Boolean.getBoolean("oak.diffIndex.legacy");
+
+ private static volatile Feature legacyDiffIndexFeature;
+
+ @Activate
+ private void activate(BundleContext context) {
+ legacyDiffIndexFeature = Feature.newFeature(LEGACY_DIFF_INDEX_TOGGLE, new OsgiWhiteboard(context));
+ }
+
+ @Deactivate
+ private void deactivate() {
+ Feature feature = legacyDiffIndexFeature;
+ legacyDiffIndexFeature = null;
+ if (feature != null) {
+ feature.close();
+ }
+ }
+
+ static boolean isLegacyMode() {
+ Feature feature = legacyDiffIndexFeature;
+ return LEGACY_DIFF_INDEX || (feature != null && feature.isEnabled());
+ }
+
+ private static void applyDiffIndexChangesLegacy(NodeStore store, NodeBuilder indexDefinitions) {
+ JsonObject diffs = collectDiffsLegacy(indexDefinitions, MERGER);
+ if (diffs != null) {
+ processDiffs(store, indexDefinitions, diffs, MERGER);
+ }
+ storeOrRemoveWarnings(indexDefinitions, MERGER);
+ }
+
/**
* Apply changes to the index definitions. That means merge the index diff with
* the existing indexes, creating new index versions. It might also mean to
@@ -59,21 +104,26 @@ public class DiffIndex {
* @param indexDefinitions the /oak:index node
*/
public static void applyDiffIndexChanges(NodeStore store, NodeBuilder indexDefinitions) {
+ if (isLegacyMode()) {
+ applyDiffIndexChangesLegacy(store, indexDefinitions);
+ return;
+ }
JsonObject diffs = collectDiffs(indexDefinitions, MERGER);
if (diffs != null) {
processDiffs(store, indexDefinitions, diffs, MERGER);
+ // only store or remove warnings if there was a change
+ storeOrRemoveWarnings(indexDefinitions, MERGER);
}
- storeOrRemoveWarnings(indexDefinitions, MERGER);
}
/**
- * Collect the diffs from the diff.index and diff.index.optimizer.
+ * Collect only changed diff.json files using the legacy behavior.
*
* @param indexDefinitions the node builder for /oak:index
* @param merger the merger instance to use for collecting warnings
* @return the diffs, or null if none
*/
- public static JsonObject collectDiffs(NodeBuilder indexDefinitions, DiffIndexMerger merger) {
+ public static JsonObject collectDiffsLegacy(NodeBuilder indexDefinitions, DiffIndexMerger merger) {
JsonObject diffs = null;
for (String diffIndex : new String[] {
DiffIndexMerger.DIFF_INDEX,
@@ -123,6 +173,93 @@ public class DiffIndex {
return diffs;
}
+ /**
+ * Collect the diffs from the diff.index and diff.index.optimizer.
+ *
+ * @param indexDefinitions the node builder for /oak:index
+ * @param merger the merger instance to use for collecting warnings
+ * @return the complete snapshot, or null if absent, unchanged, or invalid
+ */
+ public static JsonObject collectDiffs(NodeBuilder indexDefinitions, DiffIndexMerger merger) {
+ JsonObject result = new JsonObject(true);
+ List<String> readErrors = new ArrayList<>();
+ boolean wasModified = false;
+ for (String diffIndex : new String[] {
+ DiffIndexMerger.DIFF_INDEX,
+ DiffIndexMerger.DIFF_INDEX_OPTIMIZER }) {
+ String path = "/oak:index/" + diffIndex;
+ JsonObject files = new JsonObject(true);
+ result.getChildren().put(path, files);
+ if (!indexDefinitions.hasChildNode(diffIndex)) {
+ // Override the old repository snapshot when a diff index was removed.
+ files.getChildren().put("diff", new JsonObject(true));
+ continue;
+ }
+ NodeBuilder diffIndexDefinition = indexDefinitions.child(diffIndex);
+ List<String> sortedChildren = StreamSupport.stream(
+ diffIndexDefinition.getChildNodeNames().spliterator(), false)
+ .sorted().collect(Collectors.toList());
+ if (!diffIndexDefinition.hasProperty("info")) {
+ diffIndexDefinition.setProperty("info", "This diff is automatically merged with other indexes. See https://oak-indexing.github.io/oakTools/simplified.html");
+ }
+ for (String child : sortedChildren) {
+ String filePath = path + "/" + child;
+ NodeBuilder diffContent = diffIndexDefinition.getChildNode(child).getChildNode("jcr:content");
+ PropertyState lastMod = diffContent.getProperty(NodeTypeConstants.JCR_LASTMODIFIED);
+ if (lastMod != null) {
+ String modified = lastMod.getValue(Type.DATE);
+ if (!modified.equals(diffContent.getString(DiffIndexMerger.LAST_PROCESSED))) {
+ wasModified = true;
+ // Record attempts, including failures, so unchanged files are not processed again.
+ diffContent.setProperty(DiffIndexMerger.LAST_PROCESSED, modified);
+ }
+ }
+ if (!diffContent.exists()) {
+ if (child.endsWith("diff.json")) {
+ readErrors.add("jcr:content is missing in " + filePath);
+ }
+ continue;
+ }
+ PropertyState jcrData = diffContent.getProperty("jcr:data");
+ if (jcrData == null) {
+ readErrors.add("jcr:data is missing in " + filePath + "/jcr:content");
+ continue;
+ }
+ JsonObject content = new JsonObject(true);
+ try (InputStream in = jcrData.getValue(Type.BINARY).getNewStream()) {
+ content.getProperties().put("jcr:data", JsopBuilder.encode(
+ ":blobId:" + Base64.getEncoder().encodeToString(in.readAllBytes())));
+ } catch (IOException | IllegalArgumentException | IllegalStateException e) {
+ readErrors.add("Cannot read " + filePath + "/jcr:content/jcr:data: " + e.getMessage());
+ continue;
+ }
+ JsonObject file = new JsonObject(true);
+ file.getChildren().put("jcr:content", content);
+ files.getChildren().put(child, file);
+ }
+ if (files.getChildren().keySet().stream().noneMatch(name -> name.endsWith("diff.json"))) {
+ files.getChildren().put("diff", new JsonObject(true));
+ }
+ }
+ if (!wasModified) {
+ return null;
+ }
+ for (String error : readErrors) {
+ merger.logAndCollectWarn("{}", error);
+ }
+ if (!readErrors.isEmpty()) {
+ storeOrRemoveWarnings(indexDefinitions, merger);
+ return null;
+ }
+ for (String path : result.getChildren().keySet()) {
+ if (merger.tryExtractDiffIndex(result, path, new HashMap<>()) != null) {
+ storeOrRemoveWarnings(indexDefinitions, merger);
+ return null;
+ }
+ }
+ return result;
+ }
+
/**
* Process the diffs.
*
@@ -150,6 +287,9 @@ public class DiffIndex {
}
removeDisabledMergedIndexes(indexDefinitions);
sortIndexes(indexDefinitions);
+ if (!isLegacyMode() && indexDefinitions.hasChildNode(DiffIndexMerger.DIFF_INDEX)) {
+ indexDefinitions.getChildNode(DiffIndexMerger.DIFF_INDEX).removeProperty("error");
+ }
} catch (Exception e) {
LOG.warn("Error merging diffs: {}", e.getMessage(), e);
NodeBuilder diffIndexDefinition = indexDefinitions.child(DiffIndexMerger.DIFF_INDEX);
@@ -169,7 +309,7 @@ public class DiffIndex {
if (jcrData == null) {
return null;
}
- try (InputStream in = jcrData.getValue(Type.BINARY).getNewStream()) {
+ try (InputStream in = jcrData.getValue(Type.BINARY).getNewStream()) {
return new String(in.readAllBytes(), StandardCharsets.UTF_8);
} catch (IOException e) {
LOG.warn("Can not read jcr:data", e);
@@ -178,7 +318,7 @@ public class DiffIndex {
}
/**
- * Store warnings collected during diff index processing in the diff.index node.
+ * Store warnings in diff.index, if that node exists.
* Warnings are stored in separate properties named "warn.01", "warn.02", etc.
* Any existing "warn." properties are removed first.
*
@@ -189,7 +329,7 @@ public class DiffIndex {
if (!indexDefinitions.hasChildNode(DiffIndexMerger.DIFF_INDEX)) {
return;
}
- NodeBuilder diffIndexDefinition = indexDefinitions.child(DiffIndexMerger.DIFF_INDEX);
+ NodeBuilder diffIndexDefinition = indexDefinitions.getChildNode(DiffIndexMerger.DIFF_INDEX);
// remove existing warn.* properties
for (PropertyState ps : diffIndexDefinition.getNodeState().getProperties()) {
if (ps.getName().startsWith("warn.")) {
diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMerger.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMerger.java
index 3808e81b64..57a36ca2e9 100644
--- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMerger.java
+++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMerger.java
@@ -57,6 +57,12 @@ public class DiffIndexMerger {
public static final String LAST_PROCESSED = ":lastProcessed";
public static final String MERGE_CHECKSUM = "mergeChecksum";
+ /**
+ * Prefix of a property value that references another file of the "diff.index"
+ * node, instead of inlining the binary value as ":blobId:<base64>".
+ */
+ public static final String FILE_PREFIX = ":file:";
+
private static final String MERGE_INFO = "This index was auto-merged. See also https://oak-indexing.github.io/oakTools/simplified.html";
// the list of unsupported included paths, e.g. "/apps,/libs"
@@ -142,9 +148,9 @@ public class DiffIndexMerger {
// index definitions in the repository
combined.getChildren().putAll(repositoryDefinitions.getChildren());
- // read the diff.index.optimizer explicitly,
- // because it's a not a regular index definition,
- // and so it is not in the repositoryDefinitions
+ // read the diff indexes explicitly,
+ // because they are not regular index definitions,
+ // and so they are not in the repositoryDefinitions
if (repositoryNodeStore != null) {
Map<String, JsonObject> diffInRepo = readDiffIndex(repositoryNodeStore, DIFF_INDEX_OPTIMIZER);
combined.getChildren().putAll(diffInRepo);
@@ -169,7 +175,7 @@ public class DiffIndexMerger {
* Remove "diff.index" and/or "diff.index.optimizer" from
* newImageLuceneDefinitions, if their content is unchanged compared to what
* is already stored in the writable repository. This avoids writing (and
- * later re-committing, see ReindexCmd) an index definition that didn't
+ * later re-committing) an index definition that didn't
* actually change.
*
* @param newImageLuceneDefinitions
@@ -242,8 +248,14 @@ public class DiffIndexMerger {
// collect the diff index(es)
HashMap<String, JsonObject> toProcess = new HashMap<>();
- tryExtractDiffIndex(combined, "/oak:index/" + DIFF_INDEX, toProcess);
- tryExtractDiffIndex(combined, "/oak:index/" + DIFF_INDEX_OPTIMIZER, toProcess);
+ if (DiffIndex.isLegacyMode()) {
+ tryExtractDiffIndexLegacy(combined, "/oak:index/" + DIFF_INDEX, toProcess);
+ tryExtractDiffIndexLegacy(combined, "/oak:index/" + DIFF_INDEX_OPTIMIZER, toProcess);
+ } else if (tryExtractDiffIndex(combined, "/oak:index/" + DIFF_INDEX, toProcess) != null
+ || tryExtractDiffIndex(combined, "/oak:index/" + DIFF_INDEX_OPTIMIZER, toProcess) != null) {
+ // Abort on extraction errors to avoid treating missing entries as removed customizations.
+ return false;
+ }
// if the diff index exists, but doesn't contain some of the previous indexes
// (indexes with mergeInfo), then we need to disable those (using /dummy includedPath)
extractExistingMergedIndexes(combined, toProcess);
@@ -265,23 +277,7 @@ public class DiffIndexMerger {
return hasChanges;
}
- /**
- * Extract a "diff.index" from the set of index definitions (if found), and if
- * found, store the nested entries in the target map, merging them with previous
- * entries if found.
- *
- * The diff.index may either have a file (a "jcr:content" child node with a
- * "jcr:data" property), or a "diff" JSON object. For customers (in the git
- * repository), the file is much easier to construct, but when running the
- * indexing job, the nested JSON is much easier.
- *
- * @param indexDefs the set of index definitions (may be empty)
- * @param name the name of the diff.index (either diff.index or
- * diff.index.optimizer)
- * @param target the target map of diff.index definitions
- * @return the error message trying to parse the JSON file, or null
- */
- public String tryExtractDiffIndex(JsonObject indexDefs, String name, HashMap<String, JsonObject> target) {
+ public String tryExtractDiffIndexLegacy(JsonObject indexDefs, String name, HashMap<String, JsonObject> target) {
JsonObject diffIndex = indexDefs.getChildren().get(name);
if (diffIndex == null) {
return null;
@@ -319,6 +315,130 @@ public class DiffIndexMerger {
return null;
}
+ /**
+ * Extract a "diff.index" from the set of index definitions (if found), and if
+ * found, store the nested entries in the target map, merging them with previous
+ * entries if found.
+ *
+ * The diff.index may either have a file (a "jcr:content" child node with a
+ * "jcr:data" property), or a "diff" JSON object. For customers (in the git
+ * repository), the file is much easier to construct, but when running the
+ * indexing job, the nested JSON is much easier.
+ *
+ * @param indexDefs the set of index definitions (may be empty)
+ * @param name the name of the diff.index (either diff.index or
+ * diff.index.optimizer)
+ * @param target the target map of diff.index definitions
+ * @return the error message trying to parse the JSON file, or null
+ */
+ public String tryExtractDiffIndex(JsonObject indexDefs, String name, HashMap<String, JsonObject> target) {
+ if (DiffIndex.isLegacyMode()) {
+ return tryExtractDiffIndexLegacy(indexDefs, name, target);
+ }
+ JsonObject diffIndex = indexDefs.getChildren().get(name);
+ if (diffIndex == null) {
+ return null;
+ }
+ // extract either the files, or the nested json
+ // the number of files is small, so we can read them all in memory;
+ // a TreeMap is used to process them in alphabetical order
+ TreeMap<String, JsonObject> files = new TreeMap<>();
+ for (Entry<String, JsonObject> e : diffIndex.getChildren().entrySet()) {
+ if (e.getKey().endsWith("diff.json")) {
+ files.put(e.getKey(), e.getValue());
+ }
+ }
+ JsonObject diff;
+ if (!files.isEmpty()) {
+ // files: merge the top-level entries,
+ // where the alphabetically higher file overwrites the earlier ones
+ diff = new JsonObject();
+ for (Entry<String, JsonObject> fileEntry : files.entrySet()) {
+ String fileName = fileEntry.getKey();
+ JsonObject jcrContent = fileEntry.getValue().getChildren().get("jcr:content");
+ if (jcrContent == null) {
+ String message = "jcr:content child node is missing in " + fileName;
+ logAndCollectWarn(message);
+ return message;
+ }
+ JsonObject fileDiff;
+ try {
+ fileDiff = JsonObject.fromJson(JsonNodeUpdater.oakStringValue(jcrContent, "jcr:data"), true);
+ } catch (Exception e) {
+ String message = "Illegal Json, ignoring: " + e.getMessage();
+ logAndCollectWarn("Illegal Json in {}, ignoring: {}", fileName, e.getMessage());
+ return message;
+ }
+ diff.getProperties().putAll(fileDiff.getProperties());
+ diff.getChildren().putAll(fileDiff.getChildren());
+ }
+ } else {
+ // nested json
+ diff = diffIndex.getChildren().get("diff");
+ if (diff != null) {
+ // copy, so that resolving file references doesn't modify the input
+ diff = JsonObject.fromJson(diff.toString(), true);
+ }
+ }
+ // store, if not empty
+ if (diff != null) {
+ String message = resolveFileReferences(diffIndex, diff);
+ if (message != null) {
+ return message;
+ }
+ for (Entry<String, JsonObject> e : diff.getChildren().entrySet()) {
+ String key = e.getKey();
+ target.put(key, mergeDiffs(target.get(key), e.getValue()));
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Replace all ":file:<fileName>" property values with the binary
+ * value (":blobId:<base64>") of the referenced file. The file is a
+ * child node of the "diff.index" node, with a "jcr:content" child node that
+ * has a "jcr:data" property.
+ *
+ * @param diffIndex the "diff.index" node, which contains the referenced files
+ * @param node the node to process (recursively; input + output)
+ * @return the error message if a reference could not be resolved, or null
+ */
+ private String resolveFileReferences(JsonObject diffIndex, JsonObject node) {
+ for (Entry<String, String> e : node.getProperties().entrySet()) {
+ if (e.getValue().startsWith("\"")
+ && JsopTokenizer.decodeQuoted(e.getValue()).startsWith(":blobId:")) {
+ continue;
+ }
+ String value = JsonNodeUpdater.oakStringValue(e.getValue());
+ if (value == null || !value.startsWith(FILE_PREFIX)) {
+ continue;
+ }
+ String fileName = value.substring(FILE_PREFIX.length());
+ JsonObject file = diffIndex.getChildren().get(fileName);
+ if (file == null) {
+ String message = "Referenced file is missing: " + fileName;
+ logAndCollectWarn(message);
+ return message;
+ }
+ JsonObject jcrContent = file.getChildren().get("jcr:content");
+ String data = jcrContent == null ? null : jcrContent.getProperties().get("jcr:data");
+ if (data == null) {
+ String message = "jcr:content/jcr:data is missing in the referenced file: " + fileName;
+ logAndCollectWarn(message);
+ return message;
+ }
+ e.setValue(data);
+ }
+ for (JsonObject child : node.getChildren().values()) {
+ String message = resolveFileReferences(diffIndex, child);
+ if (message != null) {
+ return message;
+ }
+ }
+ return null;
+ }
+
/**
* Extract the indexes with a "mergeInfo" property and store them in the target
* object. This is needed so that indexes that were removed from the index.diff
@@ -537,7 +657,13 @@ public class DiffIndexMerger {
key = prefix + indexName + "-" + productVersion + "-custom-";
}
if (latestCustomized != null) {
- key += (latestCustomized.getCustomerVersion() + 1);
+ int nextCustomerVersion;
+ if (!DiffIndex.isLegacyMode() && latestCustomized.getProductVersion() < productVersion) {
+ nextCustomerVersion = 1;
+ } else {
+ nextCustomerVersion = latestCustomized.getCustomerVersion() + 1;
+ }
+ key += nextCustomerVersion;
} else {
key += "1";
}
diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexLegacyTest.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexLegacyTest.java
new file mode 100644
index 0000000000..6b2c084f6e
--- /dev/null
+++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexLegacyTest.java
@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.index.diff;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.commons.json.JsonObject;
+import org.apache.jackrabbit.oak.commons.json.JsopBuilder;
+import org.apache.jackrabbit.oak.plugins.memory.BinaryPropertyState;
+import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.toggle.FeatureToggle;
+import org.apache.sling.testing.mock.osgi.MockOsgi;
+import org.apache.sling.testing.mock.osgi.junit.OsgiContext;
+import org.junit.Rule;
+import org.junit.Test;
+
+public class DiffIndexLegacyTest {
+
+ @Rule
+ public final OsgiContext context = new OsgiContext();
+
+ @Test
+ public void featureToggleSelectsLegacyAndNewCollectors() {
+ DiffIndex component = new DiffIndex();
+ MockOsgi.activate(component, context.bundleContext(), Map.of());
+ try {
+ FeatureToggle toggle = context.getService(FeatureToggle.class);
+ assertNotNull(toggle);
+ assertEquals(DiffIndex.LEGACY_DIFF_INDEX_TOGGLE, toggle.getName());
+ assertFalse(DiffIndex.isLegacyMode());
+ assertProcessedFiles(false);
+
+ toggle.setEnabled(true);
+ assertTrue(DiffIndex.isLegacyMode());
+ assertProcessedFiles(true);
+
+ toggle.setEnabled(false);
+ assertFalse(DiffIndex.isLegacyMode());
+ assertProcessedFiles(false);
+ } finally {
+ MockOsgi.deactivate(component, context.bundleContext());
+ }
+ assertNull(context.getService(FeatureToggle.class));
+ assertFalse(DiffIndex.isLegacyMode());
+ }
+
+ @Test
+ public void legacyExtractionIgnoresAdditionalJsonFilesAndFileReferences() {
+ DiffIndex component = new DiffIndex();
+ MockOsgi.activate(component, context.bundleContext(), Map.of());
+ try {
+ context.getService(FeatureToggle.class).setEnabled(true);
+ JsonObject definitions = JsonObject.fromJson("""
+ {
+ "/oak:index/diff.index": {
+ "additional-diff.json": {},
+ "diff": {
+ "acme.test": { "jcr:data": ":file:missing.txt" }
+ }
+ }
+ }
+ """, true);
+ HashMap<String, JsonObject> target = new HashMap<>();
+ assertNull(new DiffIndexMerger().tryExtractDiffIndex(definitions, "/oak:index/diff.index", target));
+ assertEquals("\":file:missing.txt\"", target.get("acme.test").getProperties().get("jcr:data"));
+ context.getService(FeatureToggle.class).setEnabled(false);
+ assertNotNull(new DiffIndexMerger().tryExtractDiffIndex(
+ definitions, "/oak:index/diff.index", new HashMap<>()));
+ } finally {
+ MockOsgi.deactivate(component, context.bundleContext());
+ }
+ }
+
+ @Test
+ public void legacyExtractorReadsOnlyDiffJsonAndReportsMalformedFiles() {
+ String path = "/oak:index/diff.index";
+ JsonObject definitions = new JsonObject(true);
+ JsonObject index = new JsonObject(true);
+ JsonObject file = new JsonObject(true);
+ JsonObject content = new JsonObject(true);
+ definitions.getChildren().put(path, index);
+ index.getChildren().put("diff.json", file);
+ file.getChildren().put("jcr:content", content);
+ content.getProperties().put("jcr:data", JsopBuilder.encode("{\"acme.test\":{\"type\":\"lucene\"}}"));
+ DiffIndexMerger merger = new DiffIndexMerger();
+ HashMap<String, JsonObject> target = new HashMap<>();
+ assertNull(merger.tryExtractDiffIndexLegacy(definitions, path, target));
+ assertEquals("\"lucene\"", target.get("acme.test").getProperties().get("type"));
+
+ file.getChildren().clear();
+ assertEquals("jcr:content child node is missing in diff.json",
+ merger.tryExtractDiffIndexLegacy(definitions, path, new HashMap<>()));
+
+ file.getChildren().put("jcr:content", content);
+ content.getProperties().put("jcr:data", JsopBuilder.encode("{broken"));
+ assertTrue(merger.tryExtractDiffIndexLegacy(definitions, path, new HashMap<>()).startsWith("Illegal Json"));
+ assertEquals(2, merger.getAndClearWarnings().size());
+
+ index.getChildren().clear();
+ assertNull(merger.tryExtractDiffIndexLegacy(definitions, path, new HashMap<>()));
+ }
+
+ @Test
+ public void legacyModePreservesCustomerVersionNumbering() {
+ DiffIndex component = new DiffIndex();
+ MockOsgi.activate(component, context.bundleContext(), Map.of());
+ try {
+ JsonObject repository = JsonObject.fromJson("""
+ {
+ "/oak:index/product-2": { "type": "lucene" },
+ "/oak:index/product-1-custom-3": {
+ "type": "lucene", "mergeInfo": "previous customization"
+ }
+ }
+ """, true);
+ for (boolean legacy : new boolean[] {false, true}) {
+ context.getService(FeatureToggle.class).setEnabled(legacy);
+ JsonObject definitions = JsonObject.fromJson("""
+ {
+ "/oak:index/diff.index": {
+ "diff": { "product": { "additional": true } }
+ }
+ }
+ """, true);
+ new DiffIndexMerger().merge(definitions, repository, null);
+ String expected = "/oak:index/product-2-custom-" + (legacy ? "4" : "1");
+ assertNotNull(definitions.getChildren().get(expected));
+ }
+ } finally {
+ MockOsgi.deactivate(component, context.bundleContext());
+ }
+ }
+
+ @Test
+ public void legacyCollectorSkipsMissingAndUnchangedData() {
+ MemoryNodeStore store = new MemoryNodeStore();
+ NodeBuilder definitions = store.getRoot().builder().child("oak:index");
+ DiffIndexMerger merger = new DiffIndexMerger();
+ assertNull(DiffIndex.collectDiffsLegacy(definitions, merger));
+
+ definitions.child("diff.index").child("diff.json");
+ assertNull(DiffIndex.collectDiffsLegacy(definitions, merger));
+ NodeBuilder content = definitions.child("diff.index").child("diff.json").child("jcr:content");
+ assertNull(DiffIndex.collectDiffsLegacy(definitions, merger));
+
+ content.setProperty("jcr:lastModified", "2026-01-01T00:00:00.000Z", Type.DATE);
+ assertNull(DiffIndex.collectDiffsLegacy(definitions, merger));
+ file(definitions, "diff.json", "{broken");
+ content.setProperty("jcr:lastModified", "2026-01-01T00:00:00.001Z", Type.DATE);
+
+ assertNull(DiffIndex.collectDiffsLegacy(definitions, merger));
+ assertEquals(1, merger.getAndClearWarnings().size());
+
+ file(definitions, "diff.json", "{\"acme.test\":{\"type\":\"lucene\"}}");
+ content.setProperty("jcr:lastModified", "2026-01-01T00:00:00.002Z", Type.DATE);
+ assertNotNull(DiffIndex.collectDiffsLegacy(definitions, merger));
+ assertNull(DiffIndex.collectDiffsLegacy(definitions, merger));
+ }
+
+ @Test
+ public void legacyWarningsAreClearedOnUnchangedCommits() {
+ DiffIndex component = new DiffIndex();
+ MockOsgi.activate(component, context.bundleContext(), Map.of());
+ try {
+ context.getService(FeatureToggle.class).setEnabled(true);
+ MemoryNodeStore store = new MemoryNodeStore();
+ NodeBuilder definitions = store.getRoot().builder().child("oak:index");
+ file(definitions, "diff.json", "{broken");
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ assertTrue(definitions.child("diff.index").hasProperty("warn.01"));
+
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ assertFalse(definitions.child("diff.index").hasProperty("warn.01"));
+
+ definitions.child("diff.index").remove();
+ DiffIndex.storeOrRemoveWarnings(definitions, new DiffIndexMerger());
+ } finally {
+ MockOsgi.deactivate(component, context.bundleContext());
+ }
+ }
+
+ @Test
+ public void deactivateWithoutActivationIsSafe() {
+ MockOsgi.deactivate(new DiffIndex(), context.bundleContext());
+ }
+
+ private static void assertProcessedFiles(boolean legacy) {
+ MemoryNodeStore store = new MemoryNodeStore();
+ NodeBuilder definitions = definitions(store);
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ assertTrue(definitions.hasChildNode("acme.main-1-custom-1"));
+ assertEquals(!legacy, definitions.hasChildNode("acme.additional-1-custom-1"));
+ NodeBuilder additional = definitions.child("diff.index").child("additional-diff.json").child("jcr:content");
+ assertEquals(!legacy, additional.hasProperty(DiffIndexMerger.LAST_PROCESSED));
+ }
+
+ private static NodeBuilder definitions(MemoryNodeStore store) {
+ NodeBuilder definitions = store.getRoot().builder().child("oak:index");
+ file(definitions, "diff.json", "{\"acme.main\":{\"type\":\"lucene\"}}");
+ file(definitions, "additional-diff.json", "{\"acme.additional\":{\"type\":\"lucene\"}}");
+ return definitions;
+ }
+
+ private static void file(NodeBuilder definitions, String name, String data) {
+ NodeBuilder content = definitions.child("diff.index").child(name).child("jcr:content");
+ content.setProperty("jcr:lastModified", "2026-01-01T00:00:00.000Z", Type.DATE);
+ content.setProperty(BinaryPropertyState.binaryProperty("jcr:data", data.getBytes(StandardCharsets.UTF_8)));
+ }
+}
diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMergerTest.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMergerTest.java
index 9d0d559f33..ce82f5a706 100644
--- a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMergerTest.java
+++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMergerTest.java
@@ -17,10 +17,13 @@
package org.apache.jackrabbit.oak.plugins.index.diff;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
+import java.util.HashMap;
+import java.util.TreeSet;
import org.apache.jackrabbit.oak.api.CommitFailedException;
import org.apache.jackrabbit.oak.commons.json.JsonObject;
@@ -222,7 +225,6 @@ public class DiffIndexMergerTest {
}
}
""", true);
-
getMerger().merge(newImageLuceneDefinitions, repositoryDefinitions, null);
assertEquals("""
{
@@ -258,8 +260,480 @@ public class DiffIndexMergerTest {
}""", newImageLuceneDefinitions.toString());
}
+ private static String jsonFileNode(String json) {
+ return """
+ {
+ "jcr:primaryType": "nam:nt:file",
+ "jcr:content": {
+ "jcr:primaryType": "nam:nt:resource",
+ "jcr:mimeType": "application/json",
+ "jcr:data": \"""" + ":blobId:"
+ + Base64.getEncoder().encodeToString(json.getBytes(StandardCharsets.UTF_8)) + """
+ \"
+ }
+ }
+ """;
+ }
+
+ // multiple *diff.json files are read, sorted by name, and their top-level entries
+ // are merged; the alphabetically higher file overwrites the earlier entries
+ @Test
+ public void multipleDiffJsonFiles() {
+ JsonObject repositoryDefinitions = JsonObject.fromJson("""
+ {
+ "/oak:index/diff.index": {
+ "jcr:primaryType": "nt:unstructured",
+ "type": "lucene", "includedPaths": "/same", "queryPaths": "/same",
+ "sites.diff.json": """ + jsonFileNode("{\"a\": {\"y\": \"2\"}}") + """
+ , "diff.json": """ + jsonFileNode("{\"a\": {\"x\": \"1\"}}") + """
+ , "asset-diff.json": """ + jsonFileNode("{\"b\": {\"z\": \"3\"}}") + """
+ , "readme.txt": { "jcr:primaryType": "nam:nt:unstructured" }
+ }
+ }
+ """, true);
+
+ HashMap<String, JsonObject> target = new HashMap<>();
+ assertNull(getMerger().tryExtractDiffIndex(repositoryDefinitions, "/oak:index/diff.index", target));
+ assertEquals("[a, b]", new TreeSet<>(target.keySet()).toString());
+ // "a" is not deep-merged: sites.diff.json replaces the entry from diff.json
+ assertEquals("{\n \"y\": \"2\"\n}", target.get("a").toString());
+ assertEquals("{\n \"z\": \"3\"\n}", target.get("b").toString());
+ }
+
+ @Test
+ public void nonMatchingJsonFilenamesAreNotParsedAsDiffs() {
+ for (String index : new String[] {"/oak:index/diff.index", "/oak:index/diff.index.optimizer"}) {
+ JsonObject definitions = new JsonObject(true);
+ JsonObject files = new JsonObject(true);
+ definitions.getChildren().put(index, files);
+ files.getChildren().put("sites.diff.json",
+ JsonObject.fromJson(jsonFileNode("{\"a\":{\"x\":1}}"), true));
+ for (String filename : new String[] {"settings.json", "diff.json.bak", "diff.JSON", "asset-diff.json.txt"}) {
+ files.getChildren().put(filename, JsonObject.fromJson(jsonFileNode("{broken"), true));
+ }
+ DiffIndexMerger merger = getMerger();
+ HashMap<String, JsonObject> target = new HashMap<>();
+ assertNull(merger.tryExtractDiffIndex(definitions, index, target));
+ assertEquals("[a]", target.keySet().toString());
+ assertEquals("1", target.get("a").getProperties().get("x"));
+ assertEquals(0, merger.getAndClearWarnings().size());
+ }
+ }
+
+ // a binary within the diff JSON may reference another file of the diff.index,
+ // instead of inlining the base64 encoded data
+ @Test
+ public void diffJsonWithFileReference() {
+ String helloWorld = ":blobId:" + Base64.getEncoder().encodeToString(
+ "Hello World".getBytes(StandardCharsets.UTF_8));
+ String diffWithFileRef = """
+ {
+ "damAssetLucene": {
+ "test.txt": {
+ "jcr:primaryType": "nam:nt:file",
+ "jcr:content": {
+ "jcr:primaryType": "nam:nt:resource",
+ "jcr:mimeType": "text/plain",
+ "jcr:data": ":file:test.txt"
+ }
+ }
+ }
+ }
+ """;
+ JsonObject repositoryDefinitions = JsonObject.fromJson("""
+ {
+ "/oak:index/diff.index": {
+ "jcr:primaryType": "nt:unstructured",
+ "type": "lucene", "includedPaths": "/same", "queryPaths": "/same",
+ "test.txt": {
+ "jcr:primaryType": "nam:nt:file",
+ "jcr:content": {
+ "jcr:primaryType": "nam:nt:resource",
+ "jcr:mimeType": "text/plain",
+ "jcr:data": \"""" + helloWorld + """
+ \"
+ }
+ },
+ "diff.json": """ + jsonFileNode(diffWithFileRef) + """
+ }
+ }
+ """, true);
+
+ HashMap<String, JsonObject> target = new HashMap<>();
+ assertNull(getMerger().tryExtractDiffIndex(repositoryDefinitions, "/oak:index/diff.index", target));
+ JsonObject jcrContent = target.get("damAssetLucene").
+ getChildren().get("test.txt").getChildren().get("jcr:content");
+ // the reference is replaced with the binary value of the referenced file
+ assertEquals("Hello World", JsonNodeUpdater.oakStringValue(jcrContent, "jcr:data"));
+ }
+
+ // the same content, but with the base64 data inlined in the diff JSON,
+ // results in exactly the same diff
+ @Test
+ public void diffJsonWithInlinedBinary() {
+ String helloWorld = ":blobId:" + Base64.getEncoder().encodeToString(
+ "Hello World".getBytes(StandardCharsets.UTF_8));
+ String diffWithInlinedBinary = """
+ {
+ "damAssetLucene": {
+ "test.txt": {
+ "jcr:primaryType": "nam:nt:file",
+ "jcr:content": {
+ "jcr:primaryType": "nam:nt:resource",
+ "jcr:mimeType": "text/plain",
+ "jcr:data": \"""" + helloWorld + """
+ \"
+ }
+ }
+ }
+ }
+ """;
+ JsonObject repositoryDefinitions = JsonObject.fromJson("""
+ {
+ "/oak:index/diff.index": {
+ "jcr:primaryType": "nt:unstructured",
+ "type": "lucene", "includedPaths": "/same", "queryPaths": "/same",
+ "diff.json": """ + jsonFileNode(diffWithInlinedBinary) + """
+ }
+ }
+ """, true);
+
+ HashMap<String, JsonObject> target = new HashMap<>();
+ assertNull(getMerger().tryExtractDiffIndex(repositoryDefinitions, "/oak:index/diff.index", target));
+ JsonObject jcrContent = target.get("damAssetLucene").
+ getChildren().get("test.txt").getChildren().get("jcr:content");
+ assertEquals("Hello World", JsonNodeUpdater.oakStringValue(jcrContent, "jcr:data"));
+ }
+
+ // a file reference within the nested "diff" JSON object is resolved as well
+ @Test
+ public void nestedDiffWithFileReference() {
+ String helloWorld = ":blobId:" + Base64.getEncoder().encodeToString(
+ "Hello World".getBytes(StandardCharsets.UTF_8));
+ JsonObject repositoryDefinitions = JsonObject.fromJson("""
+ {
+ "/oak:index/diff.index": {
+ "jcr:primaryType": "nt:unstructured",
+ "type": "lucene", "includedPaths": "/same", "queryPaths": "/same",
+ "test.txt": {
+ "jcr:primaryType": "nam:nt:file",
+ "jcr:content": {
+ "jcr:primaryType": "nam:nt:resource",
+ "jcr:mimeType": "text/plain",
+ "jcr:data": \"""" + helloWorld + """
+ \"
+ }
+ },
+ "diff": {
+ "damAssetLucene": {
+ "test.txt": {
+ "jcr:primaryType": "nam:nt:file",
+ "jcr:content": {
+ "jcr:primaryType": "nam:nt:resource",
+ "jcr:mimeType": "text/plain",
+ "jcr:data": ":file:test.txt"
+ }
+ }
+ }
+ }
+ }
+ }
+ """, true);
+
+ HashMap<String, JsonObject> target = new HashMap<>();
+ assertNull(getMerger().tryExtractDiffIndex(repositoryDefinitions, "/oak:index/diff.index", target));
+ JsonObject jcrContent = target.get("damAssetLucene").
+ getChildren().get("test.txt").getChildren().get("jcr:content");
+ assertEquals("Hello World", JsonNodeUpdater.oakStringValue(jcrContent, "jcr:data"));
+ // the input is not modified
+ assertEquals("\":file:test.txt\"", repositoryDefinitions.
+ getChildren().get("/oak:index/diff.index").
+ getChildren().get("diff").
+ getChildren().get("damAssetLucene").
+ getChildren().get("test.txt").
+ getChildren().get("jcr:content").
+ getProperties().get("jcr:data"));
+ }
+
+ // the diff.index that is stored in the result keeps the file references;
+ // only the generated index definition contains the resolved binary
+ @Test
+ public void fileReferenceIsKeptInStoredDiffIndex() {
+ String helloWorld = ":blobId:" + Base64.getEncoder().encodeToString(
+ "Hello World".getBytes(StandardCharsets.UTF_8));
+ String diffWithFileRef = """
+ {
+ "damAssetLucene": {
+ "test.txt": {
+ "jcr:primaryType": "nam:nt:file",
+ "jcr:content": {
+ "jcr:primaryType": "nam:nt:resource",
+ "jcr:mimeType": "text/plain",
+ "jcr:data": ":file:test.txt"
+ }
+ }
+ }
+ }
+ """;
+ JsonObject newImageLuceneDefinitions = JsonObject.fromJson("""
+ {
+ "/oak:index/diff.index": {
+ "jcr:primaryType": "nt:unstructured",
+ "type": "lucene", "includedPaths": "/same", "queryPaths": "/same",
+ "test.txt": {
+ "jcr:primaryType": "nam:nt:file",
+ "jcr:content": {
+ "jcr:primaryType": "nam:nt:resource",
+ "jcr:mimeType": "text/plain",
+ "jcr:data": \"""" + helloWorld + """
+ \"
+ }
+ },
+ "diff.json": """ + jsonFileNode(diffWithFileRef) + """
+ }
+ }
+ """, true);
+ JsonObject repositoryDefinitions = JsonObject.fromJson("""
+ {
+ "/oak:index/damAssetLucene-12": {
+ "jcr:primaryType": "oak:IndexDefinition",
+ "type": "lucene",
+ "async": ["async", "nrt"],
+ "includedPaths": "/content/dam"
+ }
+ }
+ """, true);
+
+ getMerger().merge(newImageLuceneDefinitions, repositoryDefinitions, null);
+
+ // the generated index definition contains the resolved binary
+ JsonObject merged = newImageLuceneDefinitions.getChildren().get("/oak:index/damAssetLucene-12-custom-1");
+ JsonObject mergedContent = merged.getChildren().get("test.txt").getChildren().get("jcr:content");
+ assertEquals("Hello World", JsonNodeUpdater.oakStringValue(mergedContent, "jcr:data"));
+
+ // the diff.index that is stored in the result keeps the unresolved reference
+ JsonObject storedDiffIndex = newImageLuceneDefinitions.getChildren().get("/oak:index/diff.index");
+ String storedDiff = JsonNodeUpdater.oakStringValue(
+ storedDiffIndex.getChildren().get("diff.json").getChildren().get("jcr:content"), "jcr:data");
+ assertEquals(diffWithFileRef, storedDiff);
+ }
+
+ // All *diff.json children are sorted alphabetically, and merged.
+ // For duplicate indexes, the (alphabetically) last one wins.
+ @Test
+ public void openListOfDiffs() {
+ JsonObject newImageLuceneDefinitions = JsonObject.fromJson("""
+ {
+ "/oak:index/diff.index": {
+ "jcr:primaryType": "nt:unstructured",
+ "type": "lucene", "includedPaths": "/same", "queryPaths": "/same",
+ "diff.json": """ + jsonFileNode(indexDiff("ntFileFolder", "a", "1")) + """
+ , "asset1-diff.json": """ + jsonFileNode(indexDiff("damAssetLucene", "b", "2")) + """
+ , "asset2-diff.json": """ + jsonFileNode(indexDiff("damAssetLucene", "b", "3")) + """
+ }
+ }
+ """, true);
+ JsonObject repositoryDefinitions = JsonObject.fromJson("""
+ {
+ "/oak:index/damAssetLucene-12": {
+ "jcr:primaryType": "oak:IndexDefinition",
+ "type": "lucene",
+ "async": ["async", "nrt"],
+ "includedPaths": "/content/dam"
+ },
+ "/oak:index/ntFileFolder-3": {
+ "jcr:primaryType": "oak:IndexDefinition",
+ "type": "lucene",
+ "async": ["async", "nrt"],
+ "includedPaths": "/content/dam"
+ }
+ }
+ """, true);
+
+ getMerger().merge(newImageLuceneDefinitions, repositoryDefinitions, null);
+ assertEquals("""
+ {
+ "/oak:index/diff.index": {
+ "jcr:primaryType": "nt:unstructured",
+ "type": "lucene",
+ "includedPaths": "/same",
+ "queryPaths": "/same",
+ "diff.json": {
+ "jcr:primaryType": "nam:nt:file",
+ "jcr:content": {
+ "jcr:primaryType": "nam:nt:resource",
+ "jcr:mimeType": "application/json",
+ "jcr:data": ":blobId:ewogICJudEZpbGVGb2xkZXIiOiB7ICJpbmRleFJ1bGVzIjogewogICAgImRhbTpBc3NldCI6IHsKICAgICAgInByb3BlcnRpZXMiOiB7CiAgICAgICAgImEiOiB7CiAgICAgICAgICAgICJuYW1lIjogIjEiLAogICAgICAgICAgICAicHJvcGVydHlJbmRleCI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIH0KICAgIH0KICB9Cn0K"
+ }
+ },
+ "asset1-diff.json": {
+ "jcr:primaryType": "nam:nt:file",
+ "jcr:content": {
+ "jcr:primaryType": "nam:nt:resource",
+ "jcr:mimeType": "application/json",
+ "jcr:data": ":blobId:ewogICJkYW1Bc3NldEx1Y2VuZSI6IHsgImluZGV4UnVsZXMiOiB7CiAgICAiZGFtOkFzc2V0IjogewogICAgICAicHJvcGVydGllcyI6IHsKICAgICAgICAiYiI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiMiIsCiAgICAgICAgICAgICJwcm9wZXJ0eUluZGV4IjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgfQogICAgfQogIH0KfQo="
+ }
+ },
+ "asset2-diff.json": {
+ "jcr:primaryType": "nam:nt:file",
+ "jcr:content": {
+ "jcr:primaryType": "nam:nt:resource",
+ "jcr:mimeType": "application/json",
+ "jcr:data": ":blobId:ewogICJkYW1Bc3NldEx1Y2VuZSI6IHsgImluZGV4UnVsZXMiOiB7CiAgICAiZGFtOkFzc2V0IjogewogICAgICAicHJvcGVydGllcyI6IHsKICAgICAgICAiYiI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiMyIsCiAgICAgICAgICAgICJwcm9wZXJ0eUluZGV4IjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgfQogICAgfQogIH0KfQo="
+ }
+ }
+ },
+ "/oak:index/ntFileFolder-3-custom-1": {
+ "jcr:primaryType": "oak:IndexDefinition",
+ "type": "lucene",
+ "async": ["async", "nrt"],
+ "includedPaths": "/content/dam",
+ "mergeInfo": "This index was auto-merged. See also https://oak-indexing.github.io/oakTools/simplified.html",
+ "mergeChecksum": "a7ac2c59503b5bfea38c78a3299a536965a79f744eeabbd060381caa6e47037c",
+ "merges": ["/oak:index/ntFileFolder"],
+ "indexRules": {
+ "jcr:primaryType": "nam:nt:unstructured",
+ "dam:Asset": {
+ "jcr:primaryType": "nam:nt:unstructured",
+ "properties": {
+ "jcr:primaryType": "nam:nt:unstructured",
+ "a": {
+ "name": "1",
+ "propertyIndex": true,
+ "jcr:primaryType": "nam:nt:unstructured"
+ }
+ }
+ }
+ }
+ },
+ "/oak:index/damAssetLucene-12-custom-1": {
+ "jcr:primaryType": "oak:IndexDefinition",
+ "type": "lucene",
+ "async": ["async", "nrt"],
+ "includedPaths": "/content/dam",
+ "mergeInfo": "This index was auto-merged. See also https://oak-indexing.github.io/oakTools/simplified.html",
+ "mergeChecksum": "0cf99580627a63f488212a5f54f8f762f7e7a51af41181ee019ef9a9f58b0d9a",
+ "merges": ["/oak:index/damAssetLucene"],
+ "indexRules": {
+ "jcr:primaryType": "nam:nt:unstructured",
+ "dam:Asset": {
+ "jcr:primaryType": "nam:nt:unstructured",
+ "properties": {
+ "jcr:primaryType": "nam:nt:unstructured",
+ "b": {
+ "name": "3",
+ "propertyIndex": true,
+ "jcr:primaryType": "nam:nt:unstructured"
+ }
+ }
+ }
+ }
+ }
+ }""", newImageLuceneDefinitions.toString());
+ }
+
+ private static String indexDiff(String indexName, String propertyName, String value) {
+ return """
+ {
+ \"""" + indexName + """
+ \": { "indexRules": {
+ "dam:Asset": {
+ "properties": {
+ \"""" + propertyName + """
+ \": {
+ "name": \"""" + value + """
+ \",
+ "propertyIndex": true
+ }
+ }
+ }
+ }
+ }
+ }
+ """;
+ }
+
+ // referencing a file that doesn't exist results in an error message
+ @Test
+ public void diffJsonWithMissingFileReference() {
+ String diffWithFileRef = """
+ {
+ "damAssetLucene": {
+ "test.txt": {
+ "jcr:primaryType": "nam:nt:file",
+ "jcr:content": {
+ "jcr:primaryType": "nam:nt:resource",
+ "jcr:data": ":file:test.txt"
+ }
+ }
+ }
+ }
+ """;
+ JsonObject repositoryDefinitions = JsonObject.fromJson("""
+ {
+ "/oak:index/diff.index": {
+ "jcr:primaryType": "nt:unstructured",
+ "type": "lucene", "includedPaths": "/same", "queryPaths": "/same",
+ "diff.json": """ + jsonFileNode(diffWithFileRef) + """
+ }
+ }
+ """, true);
+
+ HashMap<String, JsonObject> target = new HashMap<>();
+ assertEquals("Referenced file is missing: test.txt",
+ getMerger().tryExtractDiffIndex(repositoryDefinitions, "/oak:index/diff.index", target));
+ assertEquals("[]", target.keySet().toString());
+ }
+
+ @Test
+ public void inlinedBinaryIsNotInterpretedAsFileReference() {
+ String binary = ":blobId:" + Base64.getEncoder().encodeToString(
+ ":file:missing.txt".getBytes(StandardCharsets.UTF_8));
+ JsonObject definitions = JsonObject.fromJson(
+ "{\"/oak:index/diff.index\":{\"diff\":{\"acme.test\":{\"jcr:data\":\"" + binary + "\"}}}}", true);
+ HashMap<String, JsonObject> target = new HashMap<>();
+ assertNull(getMerger().tryExtractDiffIndex(definitions, "/oak:index/diff.index", target));
+ assertEquals("\"" + binary + "\"", target.get("acme.test").getProperties().get("jcr:data"));
+ }
+
+ @Test
+ public void invalidDiffDoesNotDisablePreviouslyMergedIndexes() {
+ JsonObject repository = JsonObject.fromJson("""
+ {
+ "/oak:index/acme.test-1-custom-1": {
+ "type": "lucene",
+ "mergeInfo": "previously merged",
+ "mergeChecksum": "old"
+ }
+ }
+ """, true);
+ for (String diff : new String[] {
+ "{broken",
+ "{\"acme.test\":{\"file\":{\"jcr:data\":\":file:missing.txt\"}}}" }) {
+ JsonObject definitions = JsonObject.fromJson(
+ "{\"/oak:index/diff.index\":{\"diff.json\":" + jsonFileNode(diff) + "}}", true);
+ String original = definitions.toString();
+ DiffIndexMerger merger = getMerger();
+ merger.merge(definitions, repository, null);
+ assertEquals(original, definitions.toString());
+ assertEquals(1, merger.getAndClearWarnings().size());
+ }
+ }
+
+ @Test
+ public void invalidOptimizerDoesNotApplyCustomerChanges() {
+ JsonObject definitions = JsonObject.fromJson(
+ "{\"/oak:index/diff.index\":{\"diff.json\":"
+ + jsonFileNode("{\"acme.test\":{\"type\":\"lucene\"}}")
+ + "},\"/oak:index/diff.index.optimizer\":{\"diff.json\":"
+ + jsonFileNode("{broken") + "}}", true);
+ String original = definitions.toString();
+ getMerger().merge(definitions, new JsonObject(true), null);
+ assertEquals(original, definitions.toString());
+ }
+
@Test
- public void firstOptimizerCustomization() {
+ public void firstCustomizationAsJson() {
JsonObject newImageLuceneDefinitions = JsonObject.fromJson("{}", true);
// notice the the diff.index.optimizer is already stored in the repo,
// and so does not need to appear in the newImageLuceneDefinitions
diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexTest.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexTest.java
index edefd828f1..a33431e86a 100644
--- a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexTest.java
+++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexTest.java
@@ -20,17 +20,21 @@ import static org.apache.jackrabbit.oak.InitialContentHelper.INITIAL_CONTENT;
import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NAME;
import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_DISABLED;
import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_PROPERTY_NAME;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.when;
+import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
+import java.time.Instant;
import java.util.List;
import org.apache.jackrabbit.JcrConstants;
@@ -51,6 +55,7 @@ import org.apache.jackrabbit.oak.plugins.memory.BinaryPropertyState;
import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore;
import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
import org.apache.jackrabbit.oak.spi.commit.EditorHook;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
import org.junit.Test;
@@ -116,6 +121,387 @@ public class DiffIndexTest {
assertNull(DiffIndex.tryReadString(prop));
}
+ @Test
+ public void completeSnapshotIsProcessedOnlyWhenChanged() {
+ NodeStore store = new MemoryNodeStore();
+ NodeBuilder definitions = store.getRoot().builder().child(INDEX_DEFINITIONS_NAME);
+ diffFile(definitions, "diff.index", "a-diff.json", "{\"acme.a\":{\"type\":\"lucene\"}}");
+ diffFile(definitions, "diff.index", "b-diff.json", "{\"acme.b\":{\"type\":\"lucene\"}}");
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ assertTrue(definitions.hasChildNode("acme.a-1-custom-1"));
+ assertTrue(definitions.hasChildNode("acme.b-1-custom-1"));
+ assertNull(DiffIndex.collectDiffs(definitions, new DiffIndexMerger()));
+
+ diffFile(definitions, "diff.index", "a-diff.json", "{\"acme.a\":{\"type\":\"lucene\",\"tags\":[\"new\"]}}");
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ assertTrue(definitions.hasChildNode("acme.a-1-custom-2"));
+ assertTrue(definitions.hasChildNode("acme.b-1-custom-1"));
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ assertTrue(definitions.hasChildNode("acme.a-1-custom-2"));
+ assertTrue(definitions.hasChildNode("acme.b-1-custom-1"));
+ assertFalse(definitions.hasChildNode("acme.a-1-custom-3"));
+ }
+
+ @Test
+ public void deletingDiffFilesDoesNotTriggerProcessing() {
+ NodeStore store = new MemoryNodeStore();
+ NodeBuilder definitions = store.getRoot().builder().child(INDEX_DEFINITIONS_NAME);
+ diffFile(definitions, "diff.index", "a-diff.json", "{\"acme.a\":{\"type\":\"lucene\"}}");
+ diffFile(definitions, "diff.index", "b-diff.json", "{\"acme.b\":{\"type\":\"lucene\"}}");
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ definitions.child("diff.index").child("a-diff.json").remove();
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ assertTrue(definitions.hasChildNode("acme.a-1-custom-1"));
+ assertTrue(definitions.hasChildNode("acme.b-1-custom-1"));
+ assertNull(DiffIndex.collectDiffs(definitions, new DiffIndexMerger()));
+
+ touchFile(definitions.child("diff.index").child("b-diff.json").child("jcr:content"));
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ assertFalse(definitions.hasChildNode("acme.a-1-custom-1"));
+ assertTrue(definitions.hasChildNode("acme.b-1-custom-1"));
+
+ definitions.child("diff.index").child("b-diff.json").remove();
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ assertTrue(definitions.hasChildNode("acme.b-1-custom-1"));
+ assertNull(DiffIndex.collectDiffs(definitions, new DiffIndexMerger()));
+ diffFile(definitions, "diff.index", "diff.json", "{}");
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ assertFalse(definitions.hasChildNode("acme.b-1-custom-1"));
+ }
+
+ @Test
+ public void filesAreSortedAndReplaceWholeIndexEntries() {
+ NodeStore store = new MemoryNodeStore();
+ NodeBuilder definitions = store.getRoot().builder().child(INDEX_DEFINITIONS_NAME);
+ diffFile(definitions, "diff.index", "z-diff.json", "{\"acme.a\":{\"type\":\"lucene\",\"tags\":[\"last\"]}}");
+ diffFile(definitions, "diff.index", "a-diff.json", "{\"acme.a\":{\"type\":\"lucene\",\"earlier\":true}}");
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ NodeBuilder merged = definitions.child("acme.a-1-custom-1");
+ assertEquals(List.of("last"), merged.getProperty("tags").getValue(Type.STRINGS));
+ assertFalse(merged.hasProperty("earlier"));
+ }
+
+ @Test
+ public void customerAndOptimizerMergeWithSeparateBinaryFiles() throws Exception {
+ NodeStore store = new MemoryNodeStore();
+ NodeBuilder definitions = store.getRoot().builder().child(INDEX_DEFINITIONS_NAME);
+ byte[] customerData = {0, (byte) 0xff, (byte) 0x80, 42};
+ byte[] optimizerData = {3, (byte) 0xfe, 9};
+ diffFile(definitions, "diff.index", "a-diff.json",
+ "{\"acme.a\":{\"type\":\"lucene\",\"files\":{\"customer\":{\"jcr:data\":\":file:same.txt\"}}}}");
+ diffFile(definitions, "diff.index", "same.txt", customerData);
+ diffFile(definitions, "diff.index.optimizer", "b-diff.json",
+ "{\"acme.a\":{\"optimizer\":true,\"files\":{\"optimizer\":{\"jcr:data\":\":file:same.txt\"}}}}");
+ diffFile(definitions, "diff.index.optimizer", "same.txt", optimizerData);
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ NodeBuilder merged = definitions.child("acme.a-1-custom-1");
+ assertEquals("lucene", merged.getString("type"));
+ assertTrue(merged.getBoolean("optimizer"));
+ assertBinaryEquals(customerData, merged.child("files").child("customer").getProperty("jcr:data"));
+ assertBinaryEquals(optimizerData, merged.child("files").child("optimizer").getProperty("jcr:data"));
+ assertTrue(DiffIndex.tryReadString(definitions.child("diff.index").child("a-diff.json")
+ .child("jcr:content").getProperty("jcr:data")).contains(":file:same.txt"));
+
+ byte[] changedData = {1, (byte) 0xfd};
+ diffFile(definitions, "diff.index", "same.txt", changedData);
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ assertBinaryEquals(changedData, definitions.child("acme.a-1-custom-2")
+ .child("files").child("customer").getProperty("jcr:data"));
+ assertBinaryEquals(optimizerData, definitions.child("acme.a-1-custom-2")
+ .child("files").child("optimizer").getProperty("jcr:data"));
+ }
+
+ @Test
+ public void removedOptimizerDoesNotTriggerProcessing() throws Exception {
+ NodeStore store = new MemoryNodeStore();
+ NodeBuilder root = store.getRoot().builder();
+ NodeBuilder definitions = root.child(INDEX_DEFINITIONS_NAME);
+ diffFile(definitions, "diff.index", "a-diff.json", "{\"acme.a\":{\"type\":\"lucene\"}}");
+ diffFile(definitions, "diff.index.optimizer", "b-diff.json", "{\"acme.a\":{\"optimizer\":true}}");
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ store.merge(root, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ root = store.getRoot().builder();
+ definitions = root.child(INDEX_DEFINITIONS_NAME);
+ definitions.child("diff.index.optimizer").remove();
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ assertTrue(definitions.hasChildNode("acme.a-1-custom-1"));
+ assertTrue(definitions.child("acme.a-1-custom-1").getBoolean("optimizer"));
+ assertFalse(definitions.hasChildNode("acme.a-1-custom-2"));
+ assertNull(DiffIndex.collectDiffs(definitions, new DiffIndexMerger()));
+
+ touchFile(definitions.child("diff.index").child("a-diff.json").child("jcr:content"));
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ assertTrue(definitions.hasChildNode("acme.a-1-custom-2"));
+ assertFalse(definitions.child("acme.a-1-custom-2").hasProperty("optimizer"));
+ }
+
+ @Test
+ public void removedCustomerDoesNotTriggerProcessing() {
+ NodeStore store = new MemoryNodeStore();
+ NodeBuilder definitions = store.getRoot().builder().child(INDEX_DEFINITIONS_NAME);
+ diffFile(definitions, "diff.index", "a-diff.json", "{\"acme.a\":{\"type\":\"lucene\"}}");
+ diffFile(definitions, "diff.index.optimizer", "b-diff.json", "{\"acme.a\":{\"optimizer\":true}}");
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ definitions.child("diff.index").remove();
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ assertTrue(definitions.hasChildNode("acme.a-1-custom-1"));
+ assertEquals("lucene", definitions.child("acme.a-1-custom-1").getString("type"));
+ assertTrue(definitions.child("acme.a-1-custom-1").getBoolean("optimizer"));
+ assertFalse(definitions.hasChildNode("acme.a-1-custom-2"));
+ assertNull(DiffIndex.collectDiffs(definitions, new DiffIndexMerger()));
+ }
+
+ @Test
+ public void malformedJsonPreservesIndexesUntilFilesChange() {
+ NodeStore store = new MemoryNodeStore();
+ NodeBuilder definitions = store.getRoot().builder().child(INDEX_DEFINITIONS_NAME);
+ diffFile(definitions, "diff.index", "a-diff.json", "{\"acme.a\":{\"type\":\"lucene\"}}");
+ diffFile(definitions, "diff.index", "b-diff.json", "{\"acme.b\":{\"type\":\"lucene\"}}");
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+
+ diffFile(definitions, "diff.index", "a-diff.json", "{broken");
+ diffFile(definitions, "diff.index", "b-diff.json", "{\"acme.b\":{\"type\":\"lucene\",\"new\":true}}");
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ assertTrue(definitions.hasChildNode("acme.a-1-custom-1"));
+ assertTrue(definitions.hasChildNode("acme.b-1-custom-1"));
+ assertFalse(definitions.hasChildNode("acme.b-1-custom-2"));
+ assertTrue(definitions.child("diff.index").hasProperty("warn.01"));
+ String warning = definitions.child("diff.index").getString("warn.01");
+ assertNull(DiffIndex.collectDiffs(definitions, new DiffIndexMerger()));
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ assertEquals(warning, definitions.child("diff.index").getString("warn.01"));
+ assertTrue(definitions.hasChildNode("acme.b-1-custom-1"));
+
+ diffFile(definitions, "diff.index", "a-diff.json", "{\"acme.a\":{\"type\":\"lucene\"}}");
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ assertTrue(definitions.hasChildNode("acme.a-1-custom-1"));
+ assertTrue(definitions.hasChildNode("acme.b-1-custom-2"));
+ assertFalse(definitions.child("diff.index").hasProperty("warn.01"));
+ }
+
+ @Test
+ public void missingOrUnreadableDataPreservesIndexes() throws Exception {
+ NodeStore store = new MemoryNodeStore();
+ NodeBuilder definitions = store.getRoot().builder().child(INDEX_DEFINITIONS_NAME);
+ diffFile(definitions, "diff.index", "a-diff.json", "{\"acme.a\":{\"type\":\"lucene\"}}");
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ NodeBuilder content = definitions.child("diff.index").child("a-diff.json").child("jcr:content");
+ content.removeProperty("jcr:data");
+ touchFile(content);
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ assertTrue(definitions.hasChildNode("acme.a-1-custom-1"));
+ assertTrue(definitions.child("diff.index").getString("warn.01").contains("jcr:data is missing"));
+
+ Blob failingBlob = mock(Blob.class);
+ when(failingBlob.getNewStream()).thenAnswer(invocation -> new InputStream() {
+ @Override
+ public int read() throws IOException {
+ throw new IOException("Blob unavailable");
+ }
+ });
+ content.setProperty("jcr:data", failingBlob);
+ touchFile(content);
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ assertTrue(definitions.hasChildNode("acme.a-1-custom-1"));
+ assertTrue(definitions.child("diff.index").getString("warn.01").contains("Blob unavailable"));
+ when(failingBlob.getNewStream()).thenThrow(new IllegalStateException("Blob store unavailable"));
+ touchFile(content);
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ assertTrue(definitions.hasChildNode("acme.a-1-custom-1"));
+ assertTrue(definitions.child("diff.index").getString("warn.01").contains("Blob store unavailable"));
+ content.remove();
+ diffFile(definitions, "diff.index", "b-diff.json", "{\"acme.b\":{\"type\":\"lucene\"}}");
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ assertTrue(definitions.hasChildNode("acme.a-1-custom-1"));
+ assertTrue(definitions.child("diff.index").getString("warn.01").contains("jcr:content is missing"));
+ }
+
+ @Test
+ public void invalidOptimizerPreservesCustomerSnapshot() {
+ NodeStore store = new MemoryNodeStore();
+ NodeBuilder definitions = store.getRoot().builder().child(INDEX_DEFINITIONS_NAME);
+ diffFile(definitions, "diff.index", "a-diff.json", "{\"acme.a\":{\"type\":\"lucene\"}}");
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ diffFile(definitions, "diff.index.optimizer", "b-diff.json", "{broken");
+ diffFile(definitions, "diff.index", "a-diff.json", "{\"acme.a\":{\"type\":\"lucene\",\"new\":true}}");
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ assertTrue(definitions.hasChildNode("acme.a-1-custom-1"));
+ assertFalse(definitions.hasChildNode("acme.a-1-custom-2"));
+ assertTrue(definitions.child("diff.index").hasProperty("warn.01"));
+ }
+
+ @Test
+ public void missingFileReferencePreservesIndexesUntilFilesChange() {
+ NodeStore store = new MemoryNodeStore();
+ NodeBuilder definitions = store.getRoot().builder().child(INDEX_DEFINITIONS_NAME);
+ diffFile(definitions, "diff.index", "a-diff.json", "{\"acme.a\":{\"type\":\"lucene\"}}");
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ diffFile(definitions, "diff.index", "a-diff.json",
+ "{\"acme.a\":{\"type\":\"lucene\",\"file\":{\"jcr:data\":\":file:missing.txt\"}}}");
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ assertTrue(definitions.hasChildNode("acme.a-1-custom-1"));
+ assertFalse(definitions.hasChildNode("acme.a-1-custom-2"));
+ assertEquals("Referenced file is missing: missing.txt", definitions.child("diff.index").getString("warn.01"));
+ diffFile(definitions, "diff.index", "missing.txt", "recovered");
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ assertTrue(definitions.hasChildNode("acme.a-1-custom-2"));
+ assertFalse(definitions.child("diff.index").hasProperty("warn.01"));
+ }
+
+ @Test
+ public void unchangedTimestampsSkipValidationAndMerging() {
+ NodeStore store = new MemoryNodeStore();
+ NodeBuilder definitions = store.getRoot().builder().child(INDEX_DEFINITIONS_NAME);
+ diffFile(definitions, "diff.index", "a-diff.json", "{\"acme.a\":{\"type\":\"lucene\"}}");
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ NodeBuilder content = definitions.child("diff.index").child("a-diff.json").child("jcr:content");
+ content.setProperty(BinaryPropertyState.binaryProperty("jcr:data", "{broken".getBytes(StandardCharsets.UTF_8)));
+ DiffIndexMerger merger = new DiffIndexMerger();
+ assertNull(DiffIndex.collectDiffs(definitions, merger));
+ assertTrue(merger.getAndClearWarnings().isEmpty());
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ assertTrue(definitions.hasChildNode("acme.a-1-custom-1"));
+ assertFalse(definitions.child("diff.index").hasProperty("warn.01"));
+
+ touchFile(content);
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ assertTrue(definitions.hasChildNode("acme.a-1-custom-1"));
+ assertTrue(definitions.child("diff.index").hasProperty("warn.01"));
+ }
+
+ @Test
+ public void recoveredBlobDoesNotTriggerProcessingWithoutTimestampChange() throws Exception {
+ NodeStore store = new MemoryNodeStore();
+ NodeBuilder definitions = store.getRoot().builder().child(INDEX_DEFINITIONS_NAME);
+ diffFile(definitions, "diff.index", "a-diff.json", "{\"acme.a\":{\"type\":\"lucene\"}}");
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ NodeBuilder content = definitions.child("diff.index").child("a-diff.json").child("jcr:content");
+ Blob blob = mock(Blob.class);
+ when(blob.getNewStream()).thenThrow(new IllegalStateException("Blob unavailable"));
+ content.setProperty("jcr:data", blob);
+ touchFile(content);
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ String warning = definitions.child("diff.index").getString("warn.01");
+ assertTrue(warning.contains("Blob unavailable"));
+ doAnswer(invocation -> new ByteArrayInputStream(
+ "{\"acme.a\":{\"type\":\"lucene\",\"new\":true}}".getBytes(StandardCharsets.UTF_8)))
+ .when(blob).getNewStream();
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ assertTrue(definitions.hasChildNode("acme.a-1-custom-1"));
+ assertFalse(definitions.hasChildNode("acme.a-1-custom-2"));
+ assertEquals(warning, definitions.child("diff.index").getString("warn.01"));
+ touchFile(content);
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ assertTrue(definitions.hasChildNode("acme.a-1-custom-2"));
+ assertFalse(definitions.child("diff.index").hasProperty("warn.01"));
+ }
+
+ @Test
+ public void filesWithoutTimestampsDoNotTriggerProcessing() {
+ NodeStore store = new MemoryNodeStore();
+ NodeBuilder definitions = store.getRoot().builder().child(INDEX_DEFINITIONS_NAME);
+ assertNull(DiffIndex.collectDiffs(definitions, new DiffIndexMerger()));
+ definitions.child("diff.index.optimizer").child("readme.txt");
+ diffFile(definitions, "diff.index.optimizer", "a-diff.json", "{\"acme.a\":{\"type\":\"lucene\"}}");
+ NodeBuilder content = definitions.child("diff.index.optimizer").child("a-diff.json").child("jcr:content");
+ content.removeProperty("jcr:lastModified");
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ assertFalse(definitions.hasChildNode("acme.a-1-custom-1"));
+ assertNull(DiffIndex.collectDiffs(definitions, new DiffIndexMerger()));
+ diffFile(definitions, "diff.index.optimizer", "b-diff.json", "{}");
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ assertTrue(definitions.hasChildNode("acme.a-1-custom-1"));
+ assertFalse(definitions.hasChildNode("diff.index"));
+ assertNull(DiffIndex.collectDiffs(definitions, new DiffIndexMerger()));
+ diffFile(definitions, "diff.index.optimizer", "a-diff.json", "{broken");
+ DiffIndexMerger merger = new DiffIndexMerger();
+ assertNull(DiffIndex.collectDiffs(definitions, merger));
+ assertTrue(definitions.hasChildNode("acme.a-1-custom-1"));
+ assertFalse(definitions.child("diff.index.optimizer").hasProperty("warn.01"));
+ assertFalse(definitions.hasChildNode("diff.index"));
+ assertEquals(1, merger.getAndClearWarnings().size());
+ }
+
+ @Test
+ public void ordinaryJsonFilesAreAuxiliaryAndCanBeReferenced() throws Exception {
+ NodeStore store = new MemoryNodeStore();
+ NodeBuilder definitions = store.getRoot().builder().child(INDEX_DEFINITIONS_NAME);
+ byte[] data = "{\"acme.ignored\":{\"type\":\"lucene\"}}".getBytes(StandardCharsets.UTF_8);
+ diffFile(definitions, "diff.index", "asset-diff.json",
+ "{\"acme.a\":{\"type\":\"lucene\",\"file\":{\"jcr:data\":\":file:settings.json\"}}}");
+ diffFile(definitions, "diff.index", "settings.json", data);
+ diffFile(definitions, "diff.index", "readme.json", "{broken");
+ definitions.child("diff.index").child("metadata.json");
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ assertTrue(definitions.hasChildNode("acme.a-1-custom-1"));
+ assertFalse(definitions.hasChildNode("acme.ignored-1-custom-1"));
+ assertFalse(definitions.child("diff.index").hasProperty("warn.01"));
+ assertBinaryEquals(data, definitions.child("acme.a-1-custom-1")
+ .child("file").getProperty("jcr:data"));
+ }
+
+ @Test
+ public void onlyAuxiliaryJsonFilesDoNotDefineIndexes() {
+ for (String index : List.of("diff.index", "diff.index.optimizer")) {
+ NodeStore store = new MemoryNodeStore();
+ NodeBuilder definitions = store.getRoot().builder().child(INDEX_DEFINITIONS_NAME);
+ diffFile(definitions, index, "settings.json", "{\"acme.ignored\":{\"type\":\"lucene\"}}");
+ DiffIndex.applyDiffIndexChanges(store, definitions);
+ assertFalse(definitions.hasChildNode("acme.ignored-1-custom-1"));
+ assertFalse(definitions.child(index).hasProperty("warn.01"));
+ assertNull(DiffIndex.collectDiffs(definitions, new DiffIndexMerger()));
+ }
+ }
+
+ @Test
+ public void warningsAreStoredOnlyInCustomerDiffIndex() {
+ NodeBuilder definitions = new MemoryNodeStore().getRoot().builder().child(INDEX_DEFINITIONS_NAME);
+ NodeBuilder optimizer = definitions.child(DiffIndexMerger.DIFF_INDEX_OPTIMIZER);
+ DiffIndexMerger merger = new DiffIndexMerger();
+ merger.logAndCollectWarn("optimizer warning");
+ DiffIndex.storeOrRemoveWarnings(definitions, merger);
+ assertFalse(definitions.hasChildNode(DiffIndexMerger.DIFF_INDEX));
+ assertFalse(optimizer.hasProperty("warn.01"));
+ assertEquals(List.of("optimizer warning"), merger.getAndClearWarnings());
+
+ NodeBuilder customer = definitions.child(DiffIndexMerger.DIFF_INDEX);
+ customer.setProperty("warn.99", "old warning");
+ merger.logAndCollectWarn("new warning");
+ DiffIndex.storeOrRemoveWarnings(definitions, merger);
+ assertEquals("new warning", customer.getString("warn.01"));
+ assertFalse(customer.hasProperty("warn.99"));
+ assertFalse(optimizer.hasProperty("warn.01"));
+ DiffIndex.storeOrRemoveWarnings(definitions, merger);
+ assertFalse(customer.hasProperty("warn.01"));
+ }
+
+ private static void diffFile(NodeBuilder definitions, String index, String name, String data) {
+ diffFile(definitions, index, name, data.getBytes(StandardCharsets.UTF_8));
+ }
+
+ private static void diffFile(NodeBuilder definitions, String index, String name, byte[] data) {
+ NodeBuilder content = definitions.child(index).child(name).child("jcr:content");
+ touchFile(content);
+ content.setProperty(BinaryPropertyState.binaryProperty("jcr:data", data));
+ }
+
+ private static void touchFile(NodeBuilder content) {
+ PropertyState lastModified = content.getProperty(JcrConstants.JCR_LASTMODIFIED);
+ String previous = lastModified == null ? null : lastModified.getValue(Type.DATE);
+ Instant modified = previous == null ? Instant.parse("2026-01-01T00:00:00.000Z")
+ : Instant.parse(previous).plusMillis(1);
+ content.setProperty(JcrConstants.JCR_LASTMODIFIED, modified.toString(), Type.DATE);
+ }
+
+ private static void assertBinaryEquals(byte[] expected, PropertyState property) throws IOException {
+ assertNotNull(property);
+ assertEquals(Type.BINARY, property.getType());
+ try (InputStream in = property.getValue(Type.BINARY).getNewStream()) {
+ assertArrayEquals(expected, in.readAllBytes());
+ }
+ }
+
@Test
public void testDiffIndexUpdate() throws Exception {
// Create a memory node store
@@ -481,4 +867,3 @@ public class DiffIndexTest {
}
}
-