You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by th...@apache.org on 2020/02/17 09:14:56 UTC
svn commit: r1874107 - in /jackrabbit/oak/trunk:
oak-run/src/main/java/org/apache/jackrabbit/oak/index/merge/
oak-run/src/main/java/org/apache/jackrabbit/oak/run/
oak-run/src/test/java/org/apache/jackrabbit/oak/index/merge/
oak-run/src/test/resources/o...
Author: thomasm
Date: Mon Feb 17 09:14:56 2020
New Revision: 1874107
URL: http://svn.apache.org/viewvc?rev=1874107&view=rev
Log:
OAK-8783 Merge index definitions
Added:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/merge/
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/merge/IndexDefMerger.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/merge/IndexMerge.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/IndexMergeCommand.java
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/merge/
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/merge/IndexDefMergerTest.java
jackrabbit/oak/trunk/oak-run/src/test/resources/org/
jackrabbit/oak/trunk/oak-run/src/test/resources/org/apache/
jackrabbit/oak/trunk/oak-run/src/test/resources/org/apache/jackrabbit/
jackrabbit/oak/trunk/oak-run/src/test/resources/org/apache/jackrabbit/oak/
jackrabbit/oak/trunk/oak-run/src/test/resources/org/apache/jackrabbit/oak/index/
jackrabbit/oak/trunk/oak-run/src/test/resources/org/apache/jackrabbit/oak/index/merge/
jackrabbit/oak/trunk/oak-run/src/test/resources/org/apache/jackrabbit/oak/index/merge/merge.txt
jackrabbit/oak/trunk/oak-run/src/test/resources/org/apache/jackrabbit/oak/index/merge/mergeIndexes.txt
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/AvailableModes.java
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/IndexName.java
Added: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/merge/IndexDefMerger.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/merge/IndexDefMerger.java?rev=1874107&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/merge/IndexDefMerger.java (added)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/merge/IndexDefMerger.java Mon Feb 17 09:14:56 2020
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.index.merge;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.stream.Collectors;
+
+import org.apache.jackrabbit.oak.commons.json.JsonObject;
+import org.apache.jackrabbit.oak.plugins.index.search.spi.query.IndexName;
+
+/**
+ * Utility that allows to merge index definitions.
+ */
+public class IndexDefMerger {
+
+ private static HashSet<String> IGNORE_LEVEL_0 = new HashSet<>(Arrays.asList(
+ "reindex", "refresh", "seed", "reindexCount"));
+
+ /**
+ * Merge index definition changes.
+ *
+ * @param ancestor the common ancestor (the old product index, e.g. lucene)
+ * @param custom the latest customized version (e.g. lucene-custom-1)
+ * @param product the latest product index (e.g. lucene-2)
+ * @return the merged index definition (e.g. lucene-2-custom-1)
+ */
+ public static JsonObject merge(JsonObject ancestor, JsonObject custom, JsonObject product) {
+ ArrayList<String> conflicts = new ArrayList<>();
+ JsonObject merged = merge(0, ancestor, custom, product, conflicts);
+ if (!conflicts.isEmpty()) {
+ throw new UnsupportedOperationException("Conflicts detected: " + conflicts);
+ }
+ return merged;
+ }
+
+ private static boolean isSame(String a, String b) {
+ if (a == null || b == null) {
+ return a == b;
+ }
+ return a.equals(b);
+ }
+
+ private static boolean isSame(JsonObject a, JsonObject b) {
+ if (a == null || b == null) {
+ return a == b;
+ }
+ return a.toString().equals(b.toString());
+ }
+
+ private static JsonObject merge(int level, JsonObject ancestor, JsonObject custom, JsonObject product,
+ ArrayList<String> conflicts) {
+ JsonObject merged = new JsonObject(true);
+ LinkedHashMap<String, Boolean> properties = new LinkedHashMap<>();
+ if (ancestor == null) {
+ ancestor = new JsonObject();
+ }
+ for(String p : ancestor.getProperties().keySet()) {
+ properties.put(p, true);
+ }
+ if (custom == null) {
+ custom = new JsonObject();
+ }
+ for(String p : custom.getProperties().keySet()) {
+ properties.put(p, true);
+ }
+ if (product == null) {
+ product = new JsonObject();
+ }
+ for(String p : product.getProperties().keySet()) {
+ properties.put(p, true);
+ }
+ for(String k : properties.keySet()) {
+ if (level == 0 && IGNORE_LEVEL_0.contains(k)) {
+ // ignore some properties
+ continue;
+ }
+ if (k.startsWith(":")) {
+ // ignore hidden properties
+ continue;
+ }
+ String ap = ancestor.getProperties().get(k);
+ String cp = custom.getProperties().get(k);
+ String pp = product.getProperties().get(k);
+ String result;
+ if (isSame(ap, pp) || isSame(cp, pp)) {
+ result = cp;
+ } else if (isSame(ap, cp)) {
+ result = pp;
+ } else {
+ conflicts.add("Could not merge value; property=" + k + "; ancestor=" + ap + "; custom=" + cp + "; product=" + pp);
+ result = ap;
+ }
+ if (result != null) {
+ merged.getProperties().put(k, result);
+ }
+ }
+ LinkedHashMap<String, Boolean> children = new LinkedHashMap<>();
+ for(String c : ancestor.getChildren().keySet()) {
+ children.put(c, true);
+ }
+ for(String c : custom.getChildren().keySet()) {
+ children.put(c, true);
+ }
+ for(String c : product.getChildren().keySet()) {
+ children.put(c, true);
+ }
+ for(String k : children.keySet()) {
+ if (k.startsWith(":")) {
+ // ignore hidden nodes
+ continue;
+ }
+ JsonObject a = ancestor.getChildren().get(k);
+ JsonObject c = custom.getChildren().get(k);
+ JsonObject p = product.getChildren().get(k);
+ JsonObject result;
+ if (isSame(a, p) || isSame(c, p)) {
+ result = c;
+ } else if (isSame(a, c)) {
+ result = p;
+ } else {
+ result = merge(level + 1, a, c, p, conflicts);
+ }
+ if (result != null) {
+ merged.getChildren().put(k, result);
+ }
+ }
+ return merged;
+ }
+
+ /**
+ * For indexes that were modified both by the customer and in the product, merge
+ * the changes, and create a new index.
+ *
+ * The new index (if any) is stored in the "newIndexes" object.
+ *
+ * @param newIndexes the new indexes
+ * @param allIndexes all index definitions (including the new ones)
+ */
+ public static void merge(JsonObject newIndexes, JsonObject allIndexes) {
+
+ // TODO when merging, we keep the product index, so two indexes are created.
+ // e.g. lucene, lucene-custom-1, lucene-2, lucene-2-custom-1
+ // but if we don't have lucene-2, then we can't merge lucene-3.
+
+ // TODO when merging, e.g. lucene-2-custom-1 is created. but
+ // it is only imported in the read-write repo, not in the read-only
+ // repository currently. so this new index won't be used;
+ // instead, lucene-2 will be used
+
+ List<IndexName> newNames = newIndexes.getChildren().keySet().stream().map(s -> IndexName.parse(s))
+ .collect(Collectors.toList());
+ Collections.sort(newNames);
+ List<IndexName> allNames = allIndexes.getChildren().keySet().stream().map(s -> IndexName.parse(s))
+ .collect(Collectors.toList());
+ Collections.sort(allNames);
+ HashMap<String, JsonObject> mergedMap = new HashMap<>();
+ for (IndexName n : newNames) {
+ if (n.getCustomerVersion() == 0) {
+ IndexName latest = n.getLatestCustomized(allNames);
+ IndexName ancestor = n.getLatestProduct(allNames);
+ if (latest != null && ancestor != null) {
+ if (n.compareTo(latest) <= 0 || n.compareTo(ancestor) <= 0) {
+ // ignore older versions of indexes
+ continue;
+ }
+ JsonObject latestCustomized = allIndexes.getChildren().get(latest.getNodeName());
+ JsonObject latestAncestor = allIndexes.getChildren().get(ancestor.getNodeName());
+ JsonObject newProduct = newIndexes.getChildren().get(n.getNodeName());
+ JsonObject merged = merge(latestAncestor, latestCustomized, newProduct);
+ mergedMap.put(n.nextCustomizedName(), merged);
+ }
+ }
+ }
+ for (Entry<String, JsonObject> e : mergedMap.entrySet()) {
+ newIndexes.getChildren().put(e.getKey(), e.getValue());
+ }
+ }
+
+}
\ No newline at end of file
Added: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/merge/IndexMerge.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/merge/IndexMerge.java?rev=1874107&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/merge/IndexMerge.java (added)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/merge/IndexMerge.java Mon Feb 17 09:14:56 2020
@@ -0,0 +1,331 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.index.merge;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import javax.jcr.PropertyType;
+
+import org.apache.felix.inventory.Format;
+import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.commons.PathUtils;
+import org.apache.jackrabbit.oak.commons.json.JsonObject;
+import org.apache.jackrabbit.oak.commons.json.JsopReader;
+import org.apache.jackrabbit.oak.commons.json.JsopTokenizer;
+import org.apache.jackrabbit.oak.json.TypeCodes;
+import org.apache.jackrabbit.oak.plugins.index.CompositeIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.IndexPathService;
+import org.apache.jackrabbit.oak.plugins.index.IndexPathServiceImpl;
+import org.apache.jackrabbit.oak.plugins.index.IndexUpdateProvider;
+import org.apache.jackrabbit.oak.plugins.index.counter.NodeCounterEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.inventory.IndexDefinitionPrinter;
+import org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier;
+import org.apache.jackrabbit.oak.plugins.index.lucene.IndexTracker;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.reference.ReferenceEditorProvider;
+import org.apache.jackrabbit.oak.plugins.memory.PropertyStates;
+import org.apache.jackrabbit.oak.run.cli.NodeStoreFixture;
+import org.apache.jackrabbit.oak.run.cli.NodeStoreFixtureProvider;
+import org.apache.jackrabbit.oak.run.cli.Options;
+import org.apache.jackrabbit.oak.spi.blob.BlobStore;
+import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EditorHook;
+import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider;
+import org.apache.jackrabbit.oak.spi.mount.Mounts;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import joptsimple.OptionParser;
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+
+/**
+ * Merge custom index definitions with out-of-the-box index definitions.
+ */
+public class IndexMerge {
+
+ private final static Logger LOG = LoggerFactory.getLogger(IndexMerge.class);
+
+ public static final String OAK_CHILD_ORDER = ":childOrder";
+
+ private EditorHook hook;
+ private ExecutorService executorService;
+
+ public static void main(String... args) throws Exception {
+ new IndexMerge().execute(args);
+ }
+
+ private boolean quiet;
+
+ /**
+ * Execute the command.
+ *
+ * @param args the command line arguments
+ */
+ public void execute(String... args) throws Exception {
+ OptionParser parser = new OptionParser();
+ OptionSpec<Void> quietOption = parser.accepts("quiet", "be less chatty");
+ OptionSpec<String> indexDirectory = parser.accepts("indexDir", "Index directory").
+ withRequiredArg();
+ Options opts = new Options();
+ OptionSet options = opts.parseAndConfigure(parser, args);
+ quiet = options.has(quietOption);
+ boolean isReadWrite = opts.getCommonOpts().isReadWrite();
+ boolean success = true;
+ String indexRootDir = indexDirectory.value(options);
+ if (indexRootDir == null) {
+ throw new IllegalArgumentException("Required argument indexDir missing");
+ }
+ if (!isReadWrite) {
+ log("Repository connected in read-only mode. Use '--read-write' for write operations");
+ }
+ try (NodeStoreFixture fixture = NodeStoreFixtureProvider.create(opts)) {
+ NodeStore nodeStore = fixture.getStore();
+ BlobStore blobStore = fixture.getBlobStore();
+ if (isReadWrite) {
+ if (blobStore == null) {
+ throw new IllegalArgumentException("No blob store specified");
+ }
+ if (!(blobStore instanceof GarbageCollectableBlobStore)) {
+ throw new IllegalArgumentException("Not a garbage collectable blob store: " + blobStore);
+ }
+ }
+ initHook(indexRootDir, (GarbageCollectableBlobStore) blobStore);
+
+ JsonObject indexes = getIndexDefinitions(nodeStore);
+ // the superseded indexes of the old repository
+ List<String> supersededKeys = new ArrayList<>(getSupersededIndexDefs(indexes));
+ Collections.sort(supersededKeys);
+
+ // keep only new indexes that are not superseded
+ Map<String, JsonObject> indexMap = indexes.getChildren();
+ for (String superseded : supersededKeys) {
+ if (indexMap.containsKey(superseded)) {
+ log("Ignoring superseded index " + superseded);
+ indexMap.remove(superseded);
+ }
+ }
+ Set<String> indexKeys = indexes.getChildren().keySet();
+
+ IndexDefMerger.merge(indexes, indexes);
+
+ Set<String> newIndexKeys = new HashSet<>(indexes.getChildren().keySet());
+ newIndexKeys.removeAll(indexKeys);
+ if (newIndexKeys.isEmpty()) {
+ log("No indexes to merge");
+ }
+ for (String newIndexKey : newIndexKeys) {
+ log("New index: " + newIndexKey);
+ JsonObject merged = indexMap.get(newIndexKey);
+ String def = merged.toString();
+ log("Merged definition: " + def);
+ if (isReadWrite) {
+ storeIndex(nodeStore, newIndexKey, merged);
+ }
+ }
+ }
+ if (executorService != null) {
+ executorService.shutdown();
+ }
+ if (!success) {
+ System.exit(1);
+ }
+ }
+
+ private void storeIndex(NodeStore ns, String newIndexName, JsonObject indexDef) {
+ NodeBuilder rootBuilder = ns.getRoot().builder();
+ NodeBuilder b = rootBuilder;
+ for(String p : PathUtils.elements(newIndexName)) {
+ b = b.child(p);
+ }
+ build(" ", b, indexDef);
+ try {
+ ns.merge(rootBuilder, hook, CommitInfo.EMPTY);
+ log("Added index " + newIndexName);
+ } catch (CommitFailedException e) {
+ LOG.error("Failed to add index " + newIndexName, e);
+ }
+ }
+
+ private void build(String linePrefix, NodeBuilder builder, JsonObject json) {
+ for(Entry<String, String> e : json.getProperties().entrySet()) {
+ String k = e.getKey();
+ String value = e.getValue();
+ JsopTokenizer tokenizer = new JsopTokenizer(value);
+ if (tokenizer.matches('[')) {
+ ArrayList<String> list = new ArrayList<>();
+ while (!tokenizer.matches(']')) {
+ String jsonString = tokenizer.getToken();
+ list.add(jsonString);
+ tokenizer.matches(',');
+ }
+ log(linePrefix + "array " + k + " = " + list + " (String[])");
+ builder.setProperty(k, list, Type.STRINGS);
+ } else if (tokenizer.matches(JsopReader.TRUE)) {
+ log(linePrefix + "property " + k + " = true (Boolean)");
+ builder.setProperty(k, true);
+ } else if (tokenizer.matches(JsopReader.FALSE)) {
+ log(linePrefix + "property " + k + " = false (Boolean)");
+ builder.setProperty(k, false);
+ } else if (tokenizer.matches(JsopReader.STRING)) {
+ String jsonString = tokenizer.getToken();
+ int split = TypeCodes.split(jsonString);
+ if (split != -1) {
+ int type = TypeCodes.decodeType(split, jsonString);
+ String v = TypeCodes.decodeName(split, jsonString);
+ if (type == PropertyType.BINARY) {
+ throw new UnsupportedOperationException();
+ } else {
+ builder.setProperty(PropertyStates.createProperty(k, v, type));
+ }
+ }
+ } else if (tokenizer.matches(JsopReader.NUMBER)) {
+ String num = tokenizer.getToken();
+ boolean isDouble = num.indexOf('.') >= 0;
+ if (isDouble) {
+ double d = Double.parseDouble(num);
+ log(linePrefix + "property " + k + " = " + d + " (Double)");
+ builder.setProperty(k, d);
+ } else {
+ long x = Long.parseLong(num);
+ log(linePrefix + "property " + k + " = " + x + " (Long)");
+ builder.setProperty(k, x);
+ }
+ }
+ }
+ ArrayList<String> childOrder = new ArrayList<>();
+ for(Entry<String, JsonObject> e : json.getChildren().entrySet()) {
+ String k = e.getKey();
+ JsonObject el = e.getValue();
+ log(linePrefix + "child " + k);
+ build(linePrefix + " ", builder.child(k), (JsonObject) el);
+ childOrder.add(k);
+ }
+ if (!childOrder.isEmpty()) {
+ builder.setProperty(OAK_CHILD_ORDER, childOrder, Type.NAMES);
+ }
+ }
+
+ /**
+ * Get the names of the index definitions that are superseded in one of the
+ * indexes.
+ *
+ * @param indexDefs all index definitions
+ * @return the superseded indexes
+ */
+ public static Set<String> getSupersededIndexDefs(JsonObject indexDefs) {
+ HashSet<String> supersededIndexes = new HashSet<>();
+ for(JsonObject d : indexDefs.getChildren().values()) {
+ String supersedes = d.getProperties().get("supersedes");
+ if (supersedes != null) {
+ JsopTokenizer tokenizer = new JsopTokenizer(supersedes);
+ if (tokenizer.matches('[')) {
+ while (!tokenizer.matches(']')) {
+ if (tokenizer.matches(JsopReader.STRING)) {
+ String s = tokenizer.getToken();
+ if (!s.contains("/@")) {
+ supersededIndexes.add(s);
+ }
+ } else {
+ throw new IllegalArgumentException("Unexpected token: " + tokenizer.getToken());
+ }
+ tokenizer.matches(',');
+ }
+ }
+ }
+ }
+ return supersededIndexes;
+ }
+
+ /**
+ * Get the the index definitions from a node store. It uses the index path
+ * service and index definition printer from Oak.
+ *
+ * @param nodeStore the source node store
+ * @return a JSON object with all index definitions
+ */
+ private static JsonObject getIndexDefinitions(NodeStore nodeStore) throws IOException {
+ IndexPathService imageIndexPathService = new IndexPathServiceImpl(nodeStore);
+ IndexDefinitionPrinter indexDefinitionPrinter = new IndexDefinitionPrinter(nodeStore, imageIndexPathService);
+ StringWriter writer = new StringWriter();
+ PrintWriter printWriter = new PrintWriter(writer);
+ indexDefinitionPrinter.print(printWriter, Format.JSON, false);
+ printWriter.flush();
+ writer.flush();
+ String json = writer.toString();
+ return JsonObject.fromJson(json, true);
+ }
+
+ private void log(String message) {
+ if(!quiet) {
+ System.out.println(message);
+ }
+ }
+
+ private void initHook(String indexRootDir, GarbageCollectableBlobStore blobStore) throws IOException {
+ IndexTracker tracker = new IndexTracker();
+ executorService = Executors.newFixedThreadPool(2);
+ IndexCopier indexCopier = new IndexCopier(executorService, new File(indexRootDir));
+ MountInfoProvider mip = createMountInfoProvider();
+ LuceneIndexEditorProvider luceneEditor = new LuceneIndexEditorProvider(indexCopier, tracker, null, null, mip);
+ luceneEditor.setBlobStore(blobStore);
+ CompositeIndexEditorProvider indexEditor = new CompositeIndexEditorProvider(
+ luceneEditor,
+ new PropertyIndexEditorProvider().with(mip),
+ new ReferenceEditorProvider().with(mip),
+ new NodeCounterEditorProvider().with(mip)
+ );
+ IndexUpdateProvider updateProvider = new IndexUpdateProvider(
+ indexEditor, "async", false);
+ hook = new EditorHook(updateProvider);
+ }
+
+ private static MountInfoProvider createMountInfoProvider() {
+ // TODO probably need the ability to configure mounts
+ return Mounts.newBuilder()
+ .mount("libs", true, Arrays.asList(
+ // pathsSupportingFragments
+ "/oak:index/*$"
+ ), Arrays.asList(
+ // mountedPaths
+ "/libs",
+ "/apps",
+ "/jcr:system/rep:permissionStore/oak:mount-libs-crx.default"))
+ .build();
+ }
+
+}
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/AvailableModes.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/AvailableModes.java?rev=1874107&r1=1874106&r2=1874107&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/AvailableModes.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/AvailableModes.java Mon Feb 17 09:14:56 2020
@@ -45,6 +45,7 @@ public final class AvailableModes {
.put("garbage", new GarbageCommand())
.put("help", new HelpCommand())
.put("history", new HistoryCommand())
+ .put("index-merge", new IndexMergeCommand())
.put(IndexCommand.NAME, new IndexCommand())
.put(IOTraceCommand.NAME, new IOTraceCommand())
.put(JsonIndexCommand.INDEX, new JsonIndexCommand())
Added: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/IndexMergeCommand.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/IndexMergeCommand.java?rev=1874107&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/IndexMergeCommand.java (added)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/IndexMergeCommand.java Mon Feb 17 09:14:56 2020
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.run;
+
+import org.apache.jackrabbit.oak.index.merge.IndexMerge;
+import org.apache.jackrabbit.oak.run.commons.Command;
+
+public class IndexMergeCommand implements Command {
+
+ @Override
+ public void execute(String... args) throws Exception {
+ new IndexMerge().execute(args);
+ }
+
+}
Added: jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/merge/IndexDefMergerTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/merge/IndexDefMergerTest.java?rev=1874107&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/merge/IndexDefMergerTest.java (added)
+++ jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/merge/IndexDefMergerTest.java Mon Feb 17 09:14:56 2020
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.index.merge;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.LineNumberReader;
+import java.util.ArrayList;
+
+import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.commons.json.JsonObject;
+import org.apache.jackrabbit.oak.commons.json.JsopTokenizer;
+import org.junit.Test;
+
+/**
+ * Test merging index definitions.
+ */
+public class IndexDefMergerTest {
+
+ @Test
+ public void merge() throws IOException, CommitFailedException {
+ String s = readFromResource("merge.txt");
+ JsonObject json = JsonObject.fromJson(s, true);
+ for(JsonObject e : array(json.getProperties().get("tests"))) {
+ merge(e);
+ }
+ }
+
+ @Test
+ public void mergeIndexes() throws IOException, CommitFailedException {
+ String s = readFromResource("mergeIndexes.txt");
+ JsonObject json = JsonObject.fromJson(s, true);
+ for(JsonObject e : array(json.getProperties().get("tests"))) {
+ mergeIndexes(e);
+ }
+ }
+
+ private void mergeIndexes(JsonObject e) {
+ JsonObject all = e.getChildren().get("all");
+ JsonObject newDefs = e.getChildren().get("new");
+ JsonObject expectedNew = e.getChildren().get("expectedNew");
+ IndexDefMerger.merge(newDefs, all);
+ assertEquals(
+ expectedNew.toString(),
+ newDefs.toString());
+ }
+
+ private void merge(JsonObject e) {
+ JsonObject ancestor = e.getChildren().get("ancestor");
+ JsonObject custom = e.getChildren().get("custom");
+ JsonObject product = e.getChildren().get("product");
+ try {
+ JsonObject got = IndexDefMerger.merge(ancestor, custom, product);
+ JsonObject expected = e.getChildren().get("expected");
+ assertEquals(expected.toString(), got.toString());
+ } catch (UnsupportedOperationException e2) {
+ String expected = e.getProperties().get("expected");
+ assertEquals("" + expected, "\"" + e2.getMessage() + "\"");
+ }
+ }
+
+ static String readFromResource(String resourceName) throws IOException {
+ try (InputStreamReader reader = new InputStreamReader(
+ IndexDefMergerTest.class.getResourceAsStream(resourceName))) {
+ StringBuilder buff = new StringBuilder();
+ try (LineNumberReader l = new LineNumberReader(reader)) {
+ while (true) {
+ String s = l.readLine();
+ if (s == null) {
+ break;
+ }
+ if (s.trim().startsWith("//")) {
+ // comment
+ continue;
+ }
+ buff.append(s);
+ buff.append('\n');
+ }
+ }
+ return buff.toString();
+ }
+ }
+
+ private static ArrayList<JsonObject> array(String json) {
+ ArrayList<JsonObject> list = new ArrayList<>();
+ JsopTokenizer tokenizer = new JsopTokenizer(json);
+ tokenizer.read('[');
+ while (!tokenizer.matches(']')) {
+ tokenizer.read('{');
+ JsonObject j = JsonObject.create(tokenizer, true);
+ list.add(j);
+ tokenizer.matches(',');
+ }
+ return list;
+ }
+
+}
Added: jackrabbit/oak/trunk/oak-run/src/test/resources/org/apache/jackrabbit/oak/index/merge/merge.txt
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/test/resources/org/apache/jackrabbit/oak/index/merge/merge.txt?rev=1874107&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/test/resources/org/apache/jackrabbit/oak/index/merge/merge.txt (added)
+++ jackrabbit/oak/trunk/oak-run/src/test/resources/org/apache/jackrabbit/oak/index/merge/merge.txt Mon Feb 17 09:14:56 2020
@@ -0,0 +1,204 @@
+{"tests":
+[
+{
+ "ancestor": {"value": 1, "a-old": 0},
+ "custom": {"value": 2, "b-new": 3},
+ "product": {"value": 1, "c-new": 4},
+ "expected": {"value": 2, "b-new": 3, "c-new": 4}
+},
+{
+ "ancestor": {"o": {"a": 1}},
+ "custom": {"o": {"a": 2, "c": 10}},
+ "product": {"o": {"a": 1, "p": 20}},
+ "expected": {"o": {"a": 2, "c": 10, "p": 20}}
+},
+{
+ "ancestor": {"o": {"a": 1}},
+ "custom": {},
+ "product": {"o": {"a": 2}},
+ "expected":"Conflicts detected: [Could not merge value; property=a; ancestor=1; custom=null; product=2]"
+},
+{
+ "ancestor": {"a": 1},
+ "custom": {"a": 2},
+ "product": {"a": 3},
+ "expected": "Conflicts detected: [Could not merge value; property=a; ancestor=1; custom=2; product=3]"
+},
+{
+
+
+ "ancestor": {
+ "jcr:primaryType": "nam:oak:QueryIndexDefinition",
+ "compatVersion": 2,
+ "includedPaths": ["/content/dam"],
+ "type": "lucene",
+ "async": ["async", "nrt"],
+ "evaluatePathRestrictions": true,
+ "aggregates": {
+ "jcr:primaryType": "nam:nt:unstructured",
+ "dam:Asset": {
+ "jcr:primaryType": "nam:nt:unstructured",
+ "include0": {
+ "jcr:primaryType": "nam:nt:unstructured",
+ "path": "str:jcr:content"
+ }
+ }
+ },
+ "facets": {
+ "jcr:primaryType": "nam:nt:unstructured",
+ "topChildren": "100",
+ "secure": "statistical"
+ },
+ "indexRules": {
+ "jcr:primaryType": "nam:nt:unstructured",
+ "dam:Asset": {
+ "jcr:primaryType": "nam:nt:unstructured",
+ "properties": {
+ "jcr:primaryType": "nam:nt:unstructured",
+ "cqTags": {
+ "jcr:primaryType": "nam:nt:unstructured",
+ "nodeScopeIndex": true,
+ "useInSuggest": true,
+ "propertyIndex": true,
+ "useInSpellcheck": true,
+ "name": "str:jcr:content/metadata/cq:tags"
+ }
+ }
+ }
+ }
+ }
+
+
+ , "custom": {
+// changed: facetc
+ "jcr:primaryType": "nam:oak:QueryIndexDefinition",
+ "compatVersion": 2,
+ "includedPaths": ["/content/dam"],
+ "type": "lucene",
+ "async": ["async", "nrt"],
+ "evaluatePathRestrictions": true,
+ "aggregates": {
+ "jcr:primaryType": "nam:nt:unstructured",
+ "dam:Asset": {
+ "jcr:primaryType": "nam:nt:unstructured",
+ "include0": {
+ "jcr:primaryType": "nam:nt:unstructured",
+ "path": "str:jcr:content"
+ }
+ }
+ },
+ "facets": {
+ "jcr:primaryType": "nam:nt:unstructured",
+ "topChildren": "200",
+ "secure": "exact"
+ },
+ "indexRules": {
+ "jcr:primaryType": "nam:nt:unstructured",
+ "dam:Asset": {
+ "jcr:primaryType": "nam:nt:unstructured",
+ "properties": {
+ "jcr:primaryType": "nam:nt:unstructured",
+ "cqTags": {
+ "jcr:primaryType": "nam:nt:unstructured",
+ "nodeScopeIndex": true,
+ "useInSuggest": true,
+ "propertyIndex": true,
+ "useInSpellcheck": true,
+ "name": "str:jcr:content/metadata/cq:tags"
+ }
+ }
+ }
+ }
+ }
+
+
+ , "product":
+// changed: cqTags/nodeScopeIndex = false
+ {
+ "jcr:primaryType": "nam:oak:QueryIndexDefinition",
+ "compatVersion": 2,
+ "includedPaths": ["/content/dam"],
+ "type": "lucene",
+ "async": ["async", "nrt"],
+ "evaluatePathRestrictions": true,
+ "aggregates": {
+ "jcr:primaryType": "nam:nt:unstructured",
+ "dam:Asset": {
+ "jcr:primaryType": "nam:nt:unstructured",
+ "include0": {
+ "jcr:primaryType": "nam:nt:unstructured",
+ "path": "str:jcr:content"
+ }
+ }
+ },
+ "facets": {
+ "jcr:primaryType": "nam:nt:unstructured",
+ "topChildren": "100",
+ "secure": "statistical"
+ },
+ "indexRules": {
+ "jcr:primaryType": "nam:nt:unstructured",
+ "dam:Asset": {
+ "jcr:primaryType": "nam:nt:unstructured",
+ "properties": {
+ "jcr:primaryType": "nam:nt:unstructured",
+ "cqTags": {
+ "jcr:primaryType": "nam:nt:unstructured",
+ "nodeScopeIndex": false,
+ "useInSuggest": true,
+ "propertyIndex": true,
+ "useInSpellcheck": true,
+ "name": "str:jcr:content/metadata/cq:tags"
+ }
+ }
+ }
+ }
+ }
+
+
+ , "expected":
+ {
+ "jcr:primaryType": "nam:oak:QueryIndexDefinition",
+ "compatVersion": 2,
+ "includedPaths": ["/content/dam"],
+ "type": "lucene",
+ "async": ["async", "nrt"],
+ "evaluatePathRestrictions": true,
+ "aggregates": {
+ "jcr:primaryType": "nam:nt:unstructured",
+ "dam:Asset": {
+ "jcr:primaryType": "nam:nt:unstructured",
+ "include0": {
+ "jcr:primaryType": "nam:nt:unstructured",
+ "path": "str:jcr:content"
+ }
+ }
+ },
+ "facets": {
+ "jcr:primaryType": "nam:nt:unstructured",
+ "topChildren": "200",
+ "secure": "exact"
+ },
+ "indexRules": {
+ "jcr:primaryType": "nam:nt:unstructured",
+ "dam:Asset": {
+ "jcr:primaryType": "nam:nt:unstructured",
+ "properties": {
+ "jcr:primaryType": "nam:nt:unstructured",
+ "cqTags": {
+ "jcr:primaryType": "nam:nt:unstructured",
+ "nodeScopeIndex": false,
+ "useInSuggest": true,
+ "propertyIndex": true,
+ "useInSpellcheck": true,
+ "name": "str:jcr:content/metadata/cq:tags"
+ }
+ }
+ }
+ }
+ }
+
+
+}
+]
+}
\ No newline at end of file
Added: jackrabbit/oak/trunk/oak-run/src/test/resources/org/apache/jackrabbit/oak/index/merge/mergeIndexes.txt
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/test/resources/org/apache/jackrabbit/oak/index/merge/mergeIndexes.txt?rev=1874107&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/test/resources/org/apache/jackrabbit/oak/index/merge/mergeIndexes.txt (added)
+++ jackrabbit/oak/trunk/oak-run/src/test/resources/org/apache/jackrabbit/oak/index/merge/mergeIndexes.txt Mon Feb 17 09:14:56 2020
@@ -0,0 +1,17 @@
+{"tests":
+[
+ {
+ "all": {
+ "/oak:index/lucene": { "a": 1, "b": 10, "x": 1, "z": 2},
+ "/oak:index/lucene-custom-1": { "a": 2, "b": 10, "c": 1, "x": 1}
+ },
+ "new": {
+ "/oak:index/lucene-2": { "a": 1, "b": 11, "d": 100, "z": 2 }
+ },
+ "expectedNew": {
+ "/oak:index/lucene-2": { "a": 1, "b": 11, "d": 100, "z": 2 },
+ "/oak:index/lucene-2-custom-1": { "a": 2, "b": 11, "c": 1, "d": 100 }
+ }
+ }
+]
+}
\ No newline at end of file
Modified: jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/IndexName.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/IndexName.java?rev=1874107&r1=1874106&r2=1874107&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/IndexName.java (original)
+++ jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/IndexName.java Mon Feb 17 09:14:56 2020
@@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import org.apache.jackrabbit.oak.commons.PathUtils;
import org.apache.jackrabbit.oak.spi.state.NodeState;
@@ -143,6 +144,47 @@ public class IndexName implements Compar
}
/**
+ * Get the latest index name object that matches this index base name, and
+ * is customized.
+ *
+ * @param all the list of all indexes
+ * @return the lastest customized index, or null if none
+ */
+ public IndexName getLatestCustomized(List<IndexName> all) {
+ IndexName latest = null;
+ for (IndexName n : all) {
+ if (n.baseName.equals(baseName)) {
+ if (n.customerVersion > 0) {
+ if (latest == null || n.compareTo(latest) > 0) {
+ latest = n;
+ }
+ }
+ }
+ }
+ return latest;
+ }
+
+ /**
+ * Get the latest product index that matches this index base name.
+ *
+ * @param all the list of all indexes
+ * @return the latest product index, or null if none
+ */
+ public IndexName getLatestProduct(List<IndexName> all) {
+ IndexName latest = null;
+ for (IndexName n : all) {
+ if (n.baseName.equals(baseName)) {
+ if (compareTo(n) > 0 && n.customerVersion == 0) {
+ if (latest == null || n.compareTo(latest) > 0) {
+ latest = n;
+ }
+ }
+ }
+ }
+ return latest;
+ }
+
+ /**
* Filter out index that are replaced by another index with the same base
* name but newer version.
*
@@ -181,6 +223,10 @@ public class IndexName implements Compar
return result;
}
+ public String nextCustomizedName() {
+ return baseName + "-" + productVersion + "-custom-" + (customerVersion + 1);
+ }
+
private static boolean isIndexActive(String indexPath, NodeState rootState) {
NodeState indexNode = rootState;
for(String e : PathUtils.elements(indexPath)) {