You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by mr...@apache.org on 2014/11/25 19:05:37 UTC
svn commit: r1641662 - in
/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak:
checkpoint/ checkpoint/Checkpoints.java plugins/document/
plugins/document/CheckpointsHelper.java run/Main.java
Author: mreutegg
Date: Tue Nov 25 18:05:37 2014
New Revision: 1641662
URL: http://svn.apache.org/r1641662
Log:
OAK-2293: Add checkpoint management for MongoMK
Added:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/checkpoint/
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/checkpoint/Checkpoints.java (with props)
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/document/
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/document/CheckpointsHelper.java (with props)
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java
Added: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/checkpoint/Checkpoints.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/checkpoint/Checkpoints.java?rev=1641662&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/checkpoint/Checkpoints.java (added)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/checkpoint/Checkpoints.java Tue Nov 25 18:05:37 2014
@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.checkpoint;
+
+import java.util.List;
+import java.util.Map;
+
+import javax.annotation.CheckForNull;
+
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.plugins.document.CheckpointsHelper;
+import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore;
+import org.apache.jackrabbit.oak.plugins.document.Revision;
+import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeState;
+import org.apache.jackrabbit.oak.plugins.segment.file.FileStore;
+import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+
+import com.google.common.collect.Lists;
+
+import static org.apache.jackrabbit.oak.plugins.document.CheckpointsHelper.getCheckpoints;
+import static org.apache.jackrabbit.oak.plugins.document.CheckpointsHelper.removeOlderThan;
+
+/**
+ * A helper class to manage checkpoints on TarMK and DocumentMK.
+ */
+public abstract class Checkpoints {
+
+ public static Checkpoints onTarMK(FileStore store) {
+ return new TarMKCheckpoints(store);
+ }
+
+ public static Checkpoints onDocumentMK(DocumentNodeStore store) {
+ return new DocumentMKCheckpoints(store);
+ }
+
+ /**
+ * @return a list of all checkpoints.
+ */
+ public abstract List<CP> list();
+
+ /**
+ * Remove all checkpoints.
+ *
+ * @return the number of removed checkpoints or {@code -1} if the operation
+ * did not succeed.
+ */
+ public abstract long removeAll();
+
+ /**
+ * Remove all unreferenced checkpoints.
+ *
+ * @return the number of removed checkpoints or {@code -1} if the operation
+ * did not succeed.
+ */
+ public abstract long removeUnreferenced();
+
+ /**
+ * Removes the given checkpoint.
+ *
+ * @param cp a checkpoint string.
+ * @return {@code 1} if the checkpoint was successfully remove, {@code 0} if
+ * there is no such checkpoint or {@code -1} if the operation did
+ * not succeed.
+ */
+ public abstract int remove(String cp);
+
+ private static final class TarMKCheckpoints extends Checkpoints {
+
+ private final FileStore store;
+
+ public TarMKCheckpoints(FileStore store) {
+ this.store = store;
+ }
+
+ @Override
+ public List<CP> list() {
+ List<CP> list = Lists.newArrayList();
+ NodeState ns = store.getHead().getChildNode("checkpoints");
+ for (ChildNodeEntry cne : ns.getChildNodeEntries()) {
+ NodeState cneNs = cne.getNodeState();
+ list.add(new CP(cne.getName(),
+ cneNs.getLong("created"), cneNs.getLong("timestamp")));
+ }
+ return list;
+ }
+
+ @Override
+ public long removeAll() {
+ SegmentNodeState head = store.getHead();
+ NodeBuilder builder = head.builder();
+
+ NodeBuilder cps = builder.getChildNode("checkpoints");
+ long cnt = cps.getChildNodeCount(Integer.MAX_VALUE);
+ builder.setChildNode("checkpoints");
+ if (store.setHead(head, asSegmentNodeState(builder))) {
+ return cnt;
+ } else {
+ return -1;
+ }
+ }
+
+ @Override
+ public long removeUnreferenced() {
+ SegmentNodeState head = store.getHead();
+
+ String ref = getReferenceCheckpoint(head.getChildNode("root"));
+
+ NodeBuilder builder = head.builder();
+ NodeBuilder cps = builder.getChildNode("checkpoints");
+ long cnt = 0;
+ for (String c : cps.getChildNodeNames()) {
+ if (c.equals(ref)) {
+ continue;
+ }
+ cps.getChildNode(c).remove();
+ cnt++;
+ }
+
+ if (store.setHead(head, asSegmentNodeState(builder))) {
+ return cnt;
+ } else {
+ return -1;
+ }
+ }
+
+ @Override
+ public int remove(String cp) {
+ SegmentNodeState head = store.getHead();
+ NodeBuilder builder = head.builder();
+
+ NodeBuilder cpn = builder.getChildNode("checkpoints")
+ .getChildNode(cp);
+ if (cpn.exists()) {
+ cpn.remove();
+ if (store.setHead(head, asSegmentNodeState(builder))) {
+ return 1;
+ } else {
+ return -1;
+ }
+ } else {
+ return 0;
+ }
+ }
+
+ private static SegmentNodeState asSegmentNodeState(NodeBuilder builder) {
+ return (SegmentNodeState) builder.getNodeState();
+ }
+ }
+
+ private static final class DocumentMKCheckpoints extends Checkpoints {
+
+ private final DocumentNodeStore store;
+
+ private DocumentMKCheckpoints(DocumentNodeStore store) {
+ this.store = store;
+ }
+
+ @Override
+ public List<CP> list() {
+ List<CP> list = Lists.newArrayList();
+ for (Map.Entry<Revision, String> entry : getCheckpoints(store).entrySet()) {
+ list.add(new CP(entry.getKey().toString(),
+ entry.getKey().getTimestamp(),
+ Long.parseLong(entry.getValue())));
+ }
+ return list;
+ }
+
+ @Override
+ public long removeAll() {
+ return CheckpointsHelper.removeAll(store);
+ }
+
+ @Override
+ public long removeUnreferenced() {
+ String ref = getReferenceCheckpoint(store.getRoot());
+ if (ref == null) {
+ return -1;
+ }
+ return removeOlderThan(store, Revision.fromString(ref));
+ }
+
+ @Override
+ public int remove(String cp) {
+ Revision r;
+ try {
+ r = Revision.fromString(cp);
+ } catch (IllegalArgumentException e) {
+ return 0;
+ }
+ return CheckpointsHelper.remove(store, r);
+ }
+ }
+
+ @CheckForNull
+ private static String getReferenceCheckpoint(NodeState root) {
+ String ref = null;
+ PropertyState refPS = root.getChildNode(":async").getProperty("async");
+ if (refPS != null) {
+ ref = refPS.getValue(Type.STRING);
+ }
+ if (ref != null) {
+ System.out.println(
+ "Referenced checkpoint from /:async@async is " + ref);
+ }
+ return ref;
+ }
+
+ public static final class CP {
+
+ public final String id;
+ public final long created;
+ public final long expires;
+
+ private CP(String id, long created, long expires) {
+ this.id = id;
+ this.created = created;
+ this.expires = expires;
+ }
+
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/checkpoint/Checkpoints.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/document/CheckpointsHelper.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/document/CheckpointsHelper.java?rev=1641662&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/document/CheckpointsHelper.java (added)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/document/CheckpointsHelper.java Tue Nov 25 18:05:37 2014
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.document;
+
+import java.util.SortedMap;
+
+/**
+ * Helper class to access package private functionality.
+ */
+public abstract class CheckpointsHelper {
+
+ public static SortedMap<Revision, String> getCheckpoints(
+ DocumentNodeStore store) {
+ return store.getCheckpoints().getCheckpoints();
+ }
+
+ public static long removeAll(DocumentNodeStore store) {
+ long cnt = 0;
+ for (Revision r : getCheckpoints(store).keySet()) {
+ store.getCheckpoints().release(r.toString());
+ cnt++;
+ }
+ return cnt;
+ }
+
+ public static long removeOlderThan(DocumentNodeStore store, Revision r) {
+ long cnt = 0;
+ for (Revision cp : getCheckpoints(store).keySet()) {
+ if (cp.getTimestamp() < r.getTimestamp()) {
+ store.getCheckpoints().release(cp.toString());
+ cnt++;
+ }
+ }
+ return cnt;
+ }
+
+ public static int remove(DocumentNodeStore store, Revision r) {
+ if (getCheckpoints(store).containsKey(r)) {
+ store.getCheckpoints().release(r.toString());
+ return 1;
+ } else {
+ return 0;
+ }
+ }
+
+}
Propchange: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/document/CheckpointsHelper.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java?rev=1641662&r1=1641661&r2=1641662&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java Tue Nov 25 18:05:37 2014
@@ -18,6 +18,7 @@ package org.apache.jackrabbit.oak.run;
import static com.google.common.collect.Sets.newHashSet;
import static java.util.Arrays.asList;
+import static org.apache.jackrabbit.oak.checkpoint.Checkpoints.CP;
import java.io.Closeable;
import java.io.File;
@@ -60,9 +61,8 @@ import org.apache.jackrabbit.core.Reposi
import org.apache.jackrabbit.core.config.RepositoryConfig;
import org.apache.jackrabbit.oak.Oak;
import org.apache.jackrabbit.oak.api.ContentRepository;
-import org.apache.jackrabbit.oak.api.PropertyState;
-import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.benchmark.BenchmarkRunner;
+import org.apache.jackrabbit.oak.checkpoint.Checkpoints;
import org.apache.jackrabbit.oak.commons.PathUtils;
import org.apache.jackrabbit.oak.commons.json.JsopBuilder;
import org.apache.jackrabbit.oak.console.Console;
@@ -93,8 +93,6 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.plugins.segment.standby.client.StandbyClient;
import org.apache.jackrabbit.oak.plugins.segment.standby.server.StandbyServer;
import org.apache.jackrabbit.oak.scalability.ScalabilityRunner;
-import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
-import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
import org.apache.jackrabbit.oak.upgrade.RepositoryUpgrade;
@@ -478,126 +476,114 @@ public class Main {
private static void checkpoints(String[] args) throws IOException {
if (args.length == 0) {
System.out
- .println("usage: checkpoints <path> [list|rm-all|rm-unreferenced|rm <checkpoint>]");
- System.exit(1);
- }
- if (!isValidFileStore(args[0])) {
- System.err.println("Invalid FileStore directory " + args[0]);
+ .println("usage: checkpoints {<path>|<mongo-uri>} [list|rm-all|rm-unreferenced|rm <checkpoint>]");
System.exit(1);
}
+ boolean success = false;
+ Checkpoints cps;
+ Closer closer = Closer.create();
+ try {
+ String op = "list";
+ if (args.length >= 2) {
+ op = args[1];
+ if (!"list".equals(op) && !"rm-all".equals(op) && !"rm-unreferenced".equals(op) && !"rm".equals(op)) {
+ failWith("Unknown command.");
+ }
+ }
- String path = args[0];
- String op = "list";
- if (args.length >= 2) {
- op = args[1];
- if (!"list".equals(op) && !"rm-all".equals(op) && !"rm-unreferenced".equals(op) && !"rm".equals(op)) {
- System.err.println("Unknown comand.");
- System.exit(1);
+ if (args[0].startsWith(MongoURI.MONGODB_PREFIX)) {
+ MongoClientURI uri = new MongoClientURI(args[0]);
+ MongoClient client = new MongoClient(uri);
+ final DocumentNodeStore store = new DocumentMK.Builder()
+ .setMongoDB(client.getDB(uri.getDatabase()))
+ .getNodeStore();
+ closer.register(new Closeable() {
+ @Override
+ public void close() throws IOException {
+ store.dispose();
+ }
+ });
+ cps = Checkpoints.onDocumentMK(store);
+ } else if (isValidFileStore(args[0])) {
+ final FileStore store = new FileStore(new File(args[0]),
+ 256, TAR_STORAGE_MEMORY_MAPPED);
+ closer.register(new Closeable() {
+ @Override
+ public void close() throws IOException {
+ store.close();
+ }
+ });
+ cps = Checkpoints.onTarMK(store);
+ } else {
+ failWith("Invalid FileStore directory " + args[0]);
+ return;
}
- }
- System.out.println("Checkpoints " + path);
- FileStore store = new FileStore(new File(path), 256, TAR_STORAGE_MEMORY_MAPPED);
- try {
+
+ System.out.println("Checkpoints " + args[0]);
if ("list".equals(op)) {
- NodeState ns = store.getHead().getChildNode("checkpoints");
- for (ChildNodeEntry cne : ns.getChildNodeEntries()) {
- NodeState cneNs = cne.getNodeState();
+ int cnt = 0;
+ for (CP cp : cps.list()) {
System.out.printf("- %s created %s expires %s%n",
- cne.getName(),
- new Timestamp(cneNs.getLong("created")),
- new Timestamp(cneNs.getLong("timestamp")));
+ cp.id,
+ new Timestamp(cp.created),
+ new Timestamp(cp.expires));
+ cnt++;
}
- System.out.println("Found "
- + ns.getChildNodeCount(Integer.MAX_VALUE)
- + " checkpoints");
+ System.out.println("Found " + cnt + " checkpoints");
}
if ("rm-all".equals(op)) {
long time = System.currentTimeMillis();
- SegmentNodeState head = store.getHead();
- NodeBuilder builder = head.builder();
-
- NodeBuilder cps = builder.getChildNode("checkpoints");
- long cnt = cps.getChildNodeCount(Integer.MAX_VALUE);
- builder.setChildNode("checkpoints");
- boolean ok = store.setHead(head,
- (SegmentNodeState) builder.getNodeState());
+ long cnt = cps.removeAll();
time = System.currentTimeMillis() - time;
- if (ok) {
- System.out.println("Removed " + cnt + " checkpoints in "
- + time + "ms.");
+ if (cnt != -1) {
+ System.out.println("Removed " + cnt + " checkpoints in " + time + "ms.");
} else {
- System.err.println("Failed to remove all checkpoints.");
+ failWith("Failed to remove all checkpoints.");
}
}
if ("rm-unreferenced".equals(op)) {
long time = System.currentTimeMillis();
- SegmentNodeState head = store.getHead();
-
- String ref = null;
- PropertyState refPS = head.getChildNode("root")
- .getChildNode(":async").getProperty("async");
- if (refPS != null) {
- ref = refPS.getValue(Type.STRING);
- }
- if (ref != null) {
- System.out
- .println("Referenced checkpoint from /:async@async is "
- + ref);
- }
-
- NodeBuilder builder = head.builder();
- NodeBuilder cps = builder.getChildNode("checkpoints");
- long cnt = 0;
- for (String c : cps.getChildNodeNames()) {
- if (c.equals(ref)) {
- continue;
- }
- cps.getChildNode(c).remove();
- cnt++;
- }
-
- boolean ok = cnt == 0 || store.setHead(head,
- (SegmentNodeState) builder.getNodeState());
+ long cnt = cps.removeUnreferenced();
time = System.currentTimeMillis() - time;
- if (ok) {
- System.out.println("Removed " + cnt + " checkpoints in "
- + time + "ms.");
+ if (cnt != -1) {
+ System.out.println("Removed " + cnt + " checkpoints in " + time + "ms.");
} else {
- System.err.println("Failed to remove unreferenced checkpoints.");
+ failWith("Failed to remove unreferenced checkpoints.");
}
}
if ("rm".equals(op)) {
if (args.length != 3) {
- System.err.println("Missing checkpoint id");
- System.exit(1);
- }
- long time = System.currentTimeMillis();
- String cp = args[2];
- SegmentNodeState head = store.getHead();
- NodeBuilder builder = head.builder();
-
- NodeBuilder cpn = builder.getChildNode("checkpoints")
- .getChildNode(cp);
- if (cpn.exists()) {
- cpn.remove();
- boolean ok = store.setHead(head,
- (SegmentNodeState) builder.getNodeState());
+ failWith("Missing checkpoint id");
+ } else {
+ String cp = args[2];
+ long time = System.currentTimeMillis();
+ int cnt = cps.remove(cp);
time = System.currentTimeMillis() - time;
- if (ok) {
- System.err.println("Removed checkpoint " + cp + " in "
- + time + "ms.");
+ if (cnt != 0) {
+ if (cnt == 1) {
+ System.out.println("Removed checkpoint " + cp + " in "
+ + time + "ms.");
+ } else {
+ failWith("Failed to remove checkpoint " + cp);
+ }
} else {
- System.err.println("Failed to remove checkpoint " + cp);
+ failWith("Checkpoint '" + cp + "' not found.");
}
- } else {
- System.err.println("Checkpoint '" + cp + "' not found.");
- System.exit(1);
}
}
+ success = true;
+ } catch (Throwable t) {
+ System.err.println(t.getMessage());
} finally {
- store.close();
+ closer.close();
+ }
+ if (!success) {
+ System.exit(1);
}
+ }
+ private static void failWith(String message) {
+ throw new RuntimeException(message);
}
private static void recovery(String[] args) throws IOException {