You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by to...@apache.org on 2017/08/28 16:10:45 UTC
svn commit: r1806471 - in /jackrabbit/oak/trunk:
oak-core-spi/src/main/java/org/apache/jackrabbit/oak/spi/mount/
oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/
Author: tomekr
Date: Mon Aug 28 16:10:45 2017
New Revision: 1806471
URL: http://svn.apache.org/viewvc?rev=1806471&view=rev
Log:
OAK-6595: Pre-populate the default store when running composite node store
-implemented the InitialContentMigrator
Added:
jackrabbit/oak/trunk/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/InitialContentMigrator.java
Modified:
jackrabbit/oak/trunk/oak-core-spi/src/main/java/org/apache/jackrabbit/oak/spi/mount/MountInfo.java
jackrabbit/oak/trunk/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/CompositeNodeStoreService.java
Modified: jackrabbit/oak/trunk/oak-core-spi/src/main/java/org/apache/jackrabbit/oak/spi/mount/MountInfo.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core-spi/src/main/java/org/apache/jackrabbit/oak/spi/mount/MountInfo.java?rev=1806471&r1=1806470&r2=1806471&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core-spi/src/main/java/org/apache/jackrabbit/oak/spi/mount/MountInfo.java (original)
+++ jackrabbit/oak/trunk/oak-core-spi/src/main/java/org/apache/jackrabbit/oak/spi/mount/MountInfo.java Mon Aug 28 16:10:45 2017
@@ -21,12 +21,14 @@ package org.apache.jackrabbit.oak.spi.mo
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.Collection;
+import java.util.Collections;
import java.util.List;
import java.util.NavigableSet;
import java.util.Set;
import java.util.TreeSet;
import com.google.common.base.Function;
+import com.google.common.collect.ImmutableSet;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.collect.Iterables.transform;
@@ -40,7 +42,7 @@ import static org.apache.jackrabbit.oak.
/**
* Default {@link Mount} implementation for non-default mounts.
*/
-final class MountInfo implements Mount {
+public final class MountInfo implements Mount {
private static final Function<String, String> SANITIZE_PATH = new Function<String, String>() {
@Override
@@ -64,7 +66,7 @@ final class MountInfo implements Mount {
this.readOnly = readOnly;
this.pathFragmentName = "oak:mount-" + name;
this.includedPaths = cleanCopy(includedPaths);
- this.pathsSupportingFragments = newHashSet(pathsSupportingFragments);
+ this.pathsSupportingFragments = ImmutableSet.copyOf(pathsSupportingFragments);
}
@Override
@@ -138,6 +140,14 @@ final class MountInfo implements Mount {
return newTreeSet(transform(includedPaths, SANITIZE_PATH));
}
+ public Set<String> getPathsSupportingFragments() {
+ return Collections.unmodifiableSet(pathsSupportingFragments);
+ }
+
+ public Set<String> getIncludedPaths() {
+ return Collections.unmodifiableSet(includedPaths);
+ }
+
@Override
public String toString() {
StringWriter sw = new StringWriter();
Modified: jackrabbit/oak/trunk/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/CompositeNodeStoreService.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/CompositeNodeStoreService.java?rev=1806471&r1=1806470&r2=1806471&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/CompositeNodeStoreService.java (original)
+++ jackrabbit/oak/trunk/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/CompositeNodeStoreService.java Mon Aug 28 16:10:45 2017
@@ -25,6 +25,7 @@ import org.apache.felix.scr.annotations.
import org.apache.felix.scr.annotations.Reference;
import org.apache.felix.scr.annotations.ReferenceCardinality;
import org.apache.felix.scr.annotations.ReferencePolicy;
+import org.apache.jackrabbit.oak.api.CommitFailedException;
import org.apache.jackrabbit.oak.api.jmx.CheckpointMBean;
import org.apache.jackrabbit.oak.commons.PropertiesUtil;
import org.apache.jackrabbit.oak.composite.checks.NodeStoreChecks;
@@ -42,6 +43,7 @@ import org.osgi.service.component.Compon
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Dictionary;
import java.util.HashSet;
@@ -84,6 +86,11 @@ public class CompositeNodeStoreService {
)
private static final String PROP_PARTIAL_READ_ONLY = "partialReadOnly";
+ @Property(label = "Pre-populate seed mount",
+ description = "Setting this parameter to a mount name will enable pre-populating the empty default store"
+ )
+ private static final String PROP_SEED_MOUNT = "seedMount";
+
private ComponentContext context;
private Set<NodeStoreProvider> nodeStoresInUse = newIdentityHashSet();
@@ -98,11 +105,14 @@ public class CompositeNodeStoreService {
private boolean partialReadOnly;
+ private String seedMount;
+
@Activate
- protected void activate(ComponentContext context, Map<String, ?> config) {
+ protected void activate(ComponentContext context, Map<String, ?> config) throws IOException, CommitFailedException {
this.context = context;
ignoreReadOnlyWritePaths = PropertiesUtil.toStringArray(config.get(PROP_IGNORE_READ_ONLY_WRITES), new String[0]);
partialReadOnly = PropertiesUtil.toBoolean(config.get(PROP_PARTIAL_READ_ONLY), true);
+ seedMount = PropertiesUtil.toString(config.get(PROP_SEED_MOUNT), null);
registerCompositeNodeStore();
}
@@ -111,7 +121,7 @@ public class CompositeNodeStoreService {
unregisterCompositeNodeStore();
}
- private void registerCompositeNodeStore() {
+ private void registerCompositeNodeStore() throws IOException, CommitFailedException {
if (nsReg != null) {
return; // already registered
}
@@ -155,10 +165,16 @@ public class CompositeNodeStoreService {
continue;
}
String mountName = getMountName(ns);
- if (mountName != null) {
- builder.addMount(mountName, ns.getNodeStoreProvider().getNodeStore());
- LOG.info("Mounting {} as {}", ns.getDescription(), mountName);
- nodeStoresInUse.add(ns.getNodeStoreProvider());
+ if (mountName == null) {
+ continue;
+ }
+
+ builder.addMount(mountName, ns.getNodeStoreProvider().getNodeStore());
+ LOG.info("Mounting {} as {}", ns.getDescription(), mountName);
+ nodeStoresInUse.add(ns.getNodeStoreProvider());
+
+ if (mountName.equals(seedMount)) {
+ new InitialContentMigrator(globalNs.nodeStore.getNodeStore(), ns.getNodeStoreProvider().getNodeStore(), mountInfoProvider.getMountByName(seedMount)).migrate();
}
}
@@ -219,7 +235,7 @@ public class CompositeNodeStoreService {
nodeStoresInUse.clear();
}
- protected void bindNodeStore(NodeStoreProvider ns, Map<String, ?> config) {
+ protected void bindNodeStore(NodeStoreProvider ns, Map<String, ?> config) throws IOException, CommitFailedException {
NodeStoreWithProps newNs = new NodeStoreWithProps(ns, config);
nodeStores.add(newNs);
Added: jackrabbit/oak/trunk/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/InitialContentMigrator.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/InitialContentMigrator.java?rev=1806471&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/InitialContentMigrator.java (added)
+++ jackrabbit/oak/trunk/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/InitialContentMigrator.java Mon Aug 28 16:10:45 2017
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.composite;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
+import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.plugins.migration.FilteringNodeState;
+import org.apache.jackrabbit.oak.plugins.migration.report.LoggingReporter;
+import org.apache.jackrabbit.oak.plugins.migration.report.ReportingNodeState;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
+import org.apache.jackrabbit.oak.spi.mount.Mount;
+import org.apache.jackrabbit.oak.spi.mount.MountInfo;
+import org.apache.jackrabbit.oak.spi.state.ApplyDiff;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class InitialContentMigrator {
+
+ private static final int LOG_NODE_COPY = Integer.getInteger("oak.upgrade.logNodeCopy", 10000);
+
+ private static final Logger LOG = LoggerFactory.getLogger(InitialContentMigrator.class);
+
+ private final NodeStore targetNodeStore;
+
+ private final NodeStore seedNodeStore;
+
+ private final Mount seedMount;
+
+ private final Set<String> includePaths;
+
+ private final Set<String> excludePaths;
+
+ private final Set<String> fragmentPaths;
+
+ private final Set<String> excludeFragments;
+
+ public InitialContentMigrator(NodeStore targetNodeStore, NodeStore seedNodeStore, Mount seedMount) {
+ this.targetNodeStore = targetNodeStore;
+ this.seedNodeStore = seedNodeStore;
+ this.seedMount = seedMount;
+
+ this.includePaths = FilteringNodeState.ALL;
+ this.excludeFragments = ImmutableSet.of(seedMount.getPathFragmentName());
+
+ if (seedMount instanceof MountInfo) {
+ this.excludePaths = ((MountInfo) seedMount).getIncludedPaths();
+ this.fragmentPaths = new HashSet<>();
+ for (String p : ((MountInfo) seedMount).getPathsSupportingFragments()) {
+ fragmentPaths.add(stripPatternCharacters(p));
+ }
+ } else {
+ this.excludePaths = FilteringNodeState.NONE;
+ this.fragmentPaths = FilteringNodeState.ALL;
+ }
+ }
+
+ private boolean isTargetInitialized() {
+ return targetNodeStore.getRoot().hasChildNode("jcr:system");
+ }
+
+ public void migrate() throws IOException, CommitFailedException {
+ if (isTargetInitialized()) {
+ LOG.info("The target is already initialized, no need to copy the seed mount");
+ }
+
+ LOG.info("Initializing the default mount using seed {}", seedMount.getName());
+ LOG.info("Include: {}", includePaths);
+ LOG.info("Exclude: {}", excludePaths);
+ LOG.info("Exclude fragments: {} @ {}", excludeFragments, fragmentPaths);
+
+ Map<String, String> nameToRevision = new LinkedHashMap<>();
+ Map<String, String> checkpointSegmentToDoc = new LinkedHashMap<>();
+
+ NodeState initialRoot = targetNodeStore.getRoot();
+ NodeState targetRoot = initialRoot;
+ NodeState previousRoot = initialRoot;
+ for (String checkpointName : seedNodeStore.checkpoints()) {
+ NodeState checkpointRoot = seedNodeStore.retrieve(checkpointName);
+ Map<String, String> checkpointInfo = seedNodeStore.checkpointInfo(checkpointName);
+
+ boolean tracePaths;
+ if (previousRoot == initialRoot) {
+ LOG.info("Migrating first checkpoint: {}", checkpointName);
+ tracePaths = true;
+ } else {
+ LOG.info("Applying diff to {}", checkpointName);
+ tracePaths = false;
+ }
+ LOG.info("Checkpoint metadata: {}", checkpointInfo);
+
+ targetRoot = copyDiffToTarget(previousRoot, checkpointRoot, targetRoot, tracePaths);
+ previousRoot = checkpointRoot;
+
+ String newCheckpointName = targetNodeStore.checkpoint(Long.MAX_VALUE, checkpointInfo);
+ if (checkpointInfo.containsKey("name")) {
+ nameToRevision.put(checkpointInfo.get("name"), newCheckpointName);
+ }
+ checkpointSegmentToDoc.put(checkpointName, newCheckpointName);
+ }
+
+ NodeState sourceRoot = seedNodeStore.getRoot();
+ boolean tracePaths;
+ if (previousRoot == initialRoot) {
+ LOG.info("No checkpoints found; migrating head");
+ tracePaths = true;
+ } else {
+ LOG.info("Applying diff to head");
+ tracePaths = false;
+ }
+
+ targetRoot = copyDiffToTarget(previousRoot, sourceRoot, targetRoot, tracePaths);
+
+ LOG.info("Rewriting checkpoint names in /:async {}", nameToRevision);
+ NodeBuilder targetBuilder = targetRoot.builder();
+ NodeBuilder async = targetBuilder.getChildNode(":async");
+ for (Map.Entry<String, String> e : nameToRevision.entrySet()) {
+ async.setProperty(e.getKey(), e.getValue(), Type.STRING);
+
+ PropertyState temp = async.getProperty(e.getKey() + "-temp");
+ if (temp == null) {
+ continue;
+ }
+ List<String> tempValues = Lists.newArrayList(temp.getValue(Type.STRINGS));
+ for (Map.Entry<String, String> sToD : checkpointSegmentToDoc.entrySet()) {
+ if (tempValues.contains(sToD.getKey())) {
+ tempValues.set(tempValues.indexOf(sToD.getKey()), sToD.getValue());
+ }
+ }
+ async.setProperty(e.getKey() + "-temp", tempValues, Type.STRINGS);
+ }
+ targetNodeStore.merge(targetBuilder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ }
+
+ private NodeState copyDiffToTarget(NodeState before, NodeState after, NodeState targetRoot, boolean tracePaths) throws IOException, CommitFailedException {
+ NodeBuilder targetBuilder = targetRoot.builder();
+ NodeState currentRoot = wrapNodeState(after);
+ NodeState baseRoot = wrapNodeState(before);
+ currentRoot.compareAgainstBaseState(baseRoot, new ApplyDiff(targetBuilder));
+ return targetNodeStore.merge(targetBuilder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ }
+
+
+ private NodeState wrapNodeState(NodeState nodeState) {
+ NodeState wrapped = nodeState;
+ wrapped = FilteringNodeState.wrap("/", wrapped, includePaths, excludePaths, fragmentPaths, excludeFragments);
+ wrapped = ReportingNodeState.wrap(wrapped, new LoggingReporter(LOG, "Copying", LOG_NODE_COPY, -1));
+ return wrapped;
+ }
+
+ private static String stripPatternCharacters(String pathPattern) {
+ String result = pathPattern;
+ result = substringBefore(result, '*');
+ result = substringBefore(result, '$');
+ if (!result.equals(pathPattern)) {
+ int slashIndex = result.lastIndexOf('/');
+ if (slashIndex > 0) {
+ result = result.substring(0, slashIndex);
+ }
+ }
+ return result;
+ }
+
+ private static String substringBefore(String subject, char stopCharacter) {
+ int index = subject.indexOf(stopCharacter);
+ if (index > -1) {
+ return subject.substring(0, index);
+ } else {
+ return subject;
+ }
+ }
+
+}