You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by md...@apache.org on 2014/12/09 17:26:16 UTC
svn commit: r1644108 - in /jackrabbit/oak/trunk:
oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/
oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/tooling/
oak-run/ oak-run/src/main/java/org/apache/jackrabbit/...
Author: mduerig
Date: Tue Dec 9 16:26:15 2014
New Revision: 1644108
URL: http://svn.apache.org/r1644108
Log:
OAK-2323: SegmentMK consistency check
Initial implementation
Added:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/JournalReader.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/tooling/
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/tooling/ConsistencyChecker.java
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
jackrabbit/oak/trunk/oak-run/README.md
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java
jackrabbit/oak/trunk/oak-run/src/main/resources/logback.xml
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java?rev=1644108&r1=1644107&r2=1644108&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java Tue Dec 9 16:26:15 2014
@@ -208,16 +208,7 @@ public class FileStore implements Segmen
String.format(FILE_NAME_FORMAT, writeNumber, "a"));
this.writer = new TarWriter(writeFile);
- LinkedList<String> heads = newLinkedList();
- String line = journalFile.readLine();
- while (line != null) {
- int space = line.indexOf(' ');
- if (space != -1) {
- heads.add(line.substring(0, space));
- }
- line = journalFile.readLine();
- }
-
+ LinkedList<String> heads = JournalReader.heads(journalFile);
RecordId id = null;
while (id == null && !heads.isEmpty()) {
RecordId last = RecordId.fromString(tracker, heads.removeLast());
@@ -748,4 +739,69 @@ public class FileStore implements Segmen
this.pauseCompaction = pauseCompaction;
return this;
}
+
+ private synchronized void setRevision(String rootRevision) {
+ RecordId id = RecordId.fromString(tracker, rootRevision);
+ head.set(id);
+ persistedHead.set(id);
+ }
+
+ /**
+ * A read only {@link FileStore} implementation that supports
+ * going back to old revisions.
+ * <p>
+ * All write methods are no-ops.
+ */
+ public static class ReadOnlyStore extends FileStore {
+ public ReadOnlyStore(File directory) throws IOException {
+ super(directory, 266);
+ super.flushThread.close();
+ super.compactionThread.close();
+ }
+
+ /**
+ * Go to the specified {@code revision}
+ *
+ * @param revision
+ */
+ public synchronized void setRevision(String revision) {
+ super.setRevision(revision);
+ }
+
+ /**
+ * @return false
+ */
+ @Override
+ public boolean setHead(SegmentNodeState base, SegmentNodeState head) {
+ return false;
+ }
+
+ /**
+ * no-op
+ */
+ @Override
+ public synchronized void writeSegment(SegmentId id, byte[] data, int offset, int length) {
+ // nop
+ }
+
+ /**
+ * no-op
+ */
+ @Override
+ public void flush() { /* nop */ }
+
+ /**
+ * no-op
+ */
+ @Override
+ public synchronized void cleanup() { /* nop */ }
+
+ /**
+ * no-op
+ */
+ @Override
+ public void gc() { /* nop */ }
+
+ }
+
}
Added: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/JournalReader.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/JournalReader.java?rev=1644108&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/JournalReader.java (added)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/JournalReader.java Tue Dec 9 16:26:15 2014
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.segment.file;
+
+import java.io.DataInput;
+import java.io.IOException;
+import java.util.LinkedList;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Reader for journal files of the SegmentMK.
+ */
+public final class JournalReader {
+
+ private JournalReader() { }
+
+ /**
+ * Read a journal file
+ * @param journal name of the journal file
+ * @return list of revisions listed in the journal. Oldest revision first.
+ * @throws IOException
+ */
+ public static LinkedList<String> heads(DataInput journal) throws IOException {
+ LinkedList<String> heads = Lists.newLinkedList();
+ String line = journal.readLine();
+ while (line != null) {
+ int space = line.indexOf(' ');
+ if (space != -1) {
+ heads.add(line.substring(0, space));
+ }
+ line = journal.readLine();
+ }
+ return heads;
+ }
+
+}
Added: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/tooling/ConsistencyChecker.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/tooling/ConsistencyChecker.java?rev=1644108&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/tooling/ConsistencyChecker.java (added)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/tooling/ConsistencyChecker.java Tue Dec 9 16:26:15 2014
@@ -0,0 +1,255 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.segment.file.tooling;
+
+import static com.google.common.collect.Lists.reverse;
+import static com.google.common.collect.Sets.newHashSet;
+import static org.apache.jackrabbit.oak.commons.PathUtils.concat;
+import static org.apache.jackrabbit.oak.commons.PathUtils.denotesRoot;
+import static org.apache.jackrabbit.oak.commons.PathUtils.getName;
+import static org.apache.jackrabbit.oak.commons.PathUtils.getParentPath;
+import static org.apache.jackrabbit.oak.spi.state.NodeStateUtils.getNode;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore;
+import org.apache.jackrabbit.oak.plugins.segment.file.FileStore.ReadOnlyStore;
+import org.apache.jackrabbit.oak.plugins.segment.file.JournalReader;
+import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Utility for checking the files of a
+ * {@link org.apache.jackrabbit.oak.plugins.segment.file.FileStore} for inconsistency and
+ * reporting that latest consistent revision.
+ */
+public class ConsistencyChecker {
+ private static final Logger LOG = LoggerFactory.getLogger(ConsistencyChecker.class);
+
+ private final ReadOnlyStore store;
+ private final long debugInterval;
+
+ /**
+ * Run a consistency check.
+ *
+ * @param directory directory containing the tar files
+ * @param journalFileName name of the journal file containing the revision history
+ * @param fullTraversal full traversal consistency check if {@code true}. Only try
+ * to access the root node otherwise.
+ * @param debugInterval number of seconds between printing progress information to
+ * the console during the full traversal phase.
+ * @return the latest consistent revision out of the revisions listed in the journal.
+ * @throws IOException
+ */
+ public static String checkConsistency(File directory, String journalFileName,
+ boolean fullTraversal, long debugInterval) throws IOException {
+ RandomAccessFile journalFile = new RandomAccessFile(
+ new File(directory, journalFileName), "r");
+
+ List<String> revisions;
+ try {
+ revisions = reverse(JournalReader.heads(journalFile));
+ } finally {
+ journalFile.close();
+ }
+
+ print("Searching for last good revision in {}", journalFileName);
+ Set<String> badPaths = newHashSet();
+ ConsistencyChecker checker = new ConsistencyChecker(directory, debugInterval);
+ try {
+ int revisionCount = 0;
+ for (String revision : revisions) {
+ print("Checking revision {}", revision);
+ revisionCount++;
+ String badPath = checker.check(revision, badPaths);
+ if (badPath == null && fullTraversal) {
+ badPath = checker.traverse(revision);
+ }
+ if (badPath == null) {
+ print("Found latest good revision {}", revision);
+ print("Searched through {} of {} revisions", revisionCount, revisions.size());
+ return revision;
+ } else {
+ badPaths.add(badPath);
+ print("Broken revision {}", revision);
+ }
+ }
+ } finally {
+ checker.close();
+ }
+
+ print("No good revision found");
+ return null;
+ }
+
+ /**
+ * Create a new consistency checker instance
+ *
+ * @param directory directory containing the tar files
+ * @param debugInterval number of seconds between printing progress information to
+ * the console during the full traversal phase.
+ * @throws IOException
+ */
+ public ConsistencyChecker(File directory, long debugInterval)
+ throws IOException {
+ store = new ReadOnlyStore(directory);
+ this.debugInterval = debugInterval;
+ }
+
+ /**
+ * Check whether the nodes and all its properties of all given
+ * {@code paths} are consistent at the given {@code revision}.
+ *
+ * @param revision revision to check
+ * @param paths paths to check
+ * @return Path of the first inconsistency detected or {@code null} if none.
+ */
+ public String check(String revision, Set<String> paths) {
+ store.setRevision(revision);
+ for (String path : paths) {
+ String err = checkPath(path);
+ if (err != null) {
+ return err;
+ }
+ }
+ return null;
+ }
+
+ private String checkPath(String path) {
+ try {
+ print("Checking {}", path);
+ NodeState root = new SegmentNodeStore(store).getRoot();
+ String parentPath = getParentPath(path);
+ String name = getName(path);
+ NodeState parent = getNode(root, parentPath);
+ if (!denotesRoot(path) && parent.hasChildNode(name)) {
+ return traverse(parent.getChildNode(name), path, false);
+ } else {
+ return traverse(parent, parentPath, false);
+ }
+ } catch (RuntimeException e) {
+ print("Error while checking {}: {}", path, e.getMessage());
+ return path;
+ }
+ }
+
+ private int nodeCount;
+ private int propertyCount;
+
+ /**
+ * Travers the given {@code revision}
+ * @param revision revision to travers
+ */
+ public String traverse(String revision) {
+ try {
+ store.setRevision(revision);
+ nodeCount = 0;
+ propertyCount = 0;
+ String result = traverse(new SegmentNodeStore(store).getRoot(), "/", true);
+ print("Traversed {} nodes and {} properties", nodeCount, propertyCount);
+ return result;
+ } catch (RuntimeException e) {
+ print("Error while traversing {}", revision, e.getMessage());
+ return "/";
+ }
+ }
+
+ private String traverse(NodeState node, String path, boolean deep) {
+ try {
+ debug("Traversing {}", path);
+ nodeCount++;
+ for (PropertyState propertyState : node.getProperties()) {
+ debug("Checking {}/{}", path, propertyState);
+ propertyState.getValue(propertyState.getType());
+ propertyCount++;
+ }
+ for (ChildNodeEntry cne : node.getChildNodeEntries()) {
+ String childName = cne.getName();
+ NodeState child = cne.getNodeState();
+ if (deep) {
+ String result = traverse(child, concat(path, childName), deep);
+ if (result != null) {
+ return result;
+ }
+ }
+ }
+ return null;
+ } catch (RuntimeException e) {
+ print("Error while traversing {}: {}", path, e.getMessage());
+ return path;
+ }
+ }
+
+ public void close() {
+ store.close();
+ }
+
+ private static void print(String format) {
+ LOG.info(format);
+ }
+
+ private static void print(String format, Object arg) {
+ LOG.info(format, arg);
+ }
+
+ private static void print(String format, Object arg1, Object arg2) {
+ LOG.info(format, arg1, arg2);
+ }
+
+ private long ts;
+
+ private void debug(String format, Object arg) {
+ if (debug()) {
+ LOG.debug(format, arg);
+ }
+ }
+
+ private void debug(String format, Object arg1, Object arg2) {
+ if (debug()) {
+ LOG.debug(format, arg1, arg2);
+ }
+ }
+
+ private boolean debug() {
+ // Avoid calling System.currentTimeMillis(), which is slow on some systems.
+ if (debugInterval == Long.MAX_VALUE) {
+ return false;
+ } else if (debugInterval == 0) {
+ return true;
+ }
+
+ long ts = System.currentTimeMillis();
+ if ((ts - this.ts) / 1000 > debugInterval) {
+ this.ts = ts;
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+
+}
Modified: jackrabbit/oak/trunk/oak-run/README.md
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/README.md?rev=1644108&r1=1644107&r2=1644108&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/README.md (original)
+++ jackrabbit/oak/trunk/oak-run/README.md Tue Dec 9 16:26:15 2014
@@ -14,6 +14,7 @@ The following runmodes are currently ava
* server : Run the Oak Server.
* console : Start an interactive console.
* explore : Starts a GUI browser based on java swing.
+ * check : Check the FileStore for inconsistencies
* primary : Run a TarMK Cold Standby primary instance
* standby : Run a TarMK Cold Standby standby instance
* scalability : Run scalability tests against different Oak repository fixtures.
@@ -88,6 +89,43 @@ browsing of an existing oak repository.
$ java -jar oak-run-*.jar explore /path/to/oak/repository [skip-size-check]
+Check
+-----
+
+The 'check' mode checks the storage of the FileStore for inconsistencies.
+
+ $ java -jar oak-run-*.jar check <options>
+
+ --deep [Long] enable deep consistency checking. An
+ optional long specifies the number
+ of seconds between progress
+ notifications (default:
+ 9223372036854775807)
+ --journal journal file (default: journal.log)
+ --path path to the segment store (required)
+
+For example
+
+ $ java -jar oak-run-*.jar check -p repository/segmentstore -d
+
+Checks the files in the `repository/segmentstore` directory for inconsistencies.
+It will start with the latest revision in the `journal.log` file going back revision
+by revision until a full traversal succeeds. During the traversal the current path
+will is output to the console every 1 second. When done the latest good revision is
+output.
+
+ Searching for last good revision in journal.log
+ Checking revision b82167c3-1ceb-4404-a67f-9c542e854086:240872
+ Traversing /
+ Error while traversing /home/users/foo: Segment 476e1abd-0ea0-44a8-ac3c-3a3bd
+ Traversed 50048 nodes and 303846 properties
+ Broken revision b82167c3-1ceb-4404-a67f-9c542e854086:240872
+ Checking revision 84d693cb-f214-4d19-a1cd-8b5766d50fdb:250028
+ Checking /home/users/foo
+ Traversed 11612889 nodes and 27511640 properties
+ Found latest good revision 84d693cb-f214-4d19-a1cd-8b5766d50fdb:250028
+ Searched through 2 of 390523 revisions
+
Primary
-------
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java?rev=1644108&r1=1644107&r2=1644108&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java Tue Dec 9 16:26:15 2014
@@ -19,6 +19,7 @@ package org.apache.jackrabbit.oak.run;
import static com.google.common.collect.Sets.newHashSet;
import static java.util.Arrays.asList;
import static org.apache.jackrabbit.oak.checkpoint.Checkpoints.CP;
+import static org.apache.jackrabbit.oak.plugins.segment.file.tooling.ConsistencyChecker.checkConsistency;
import java.io.Closeable;
import java.io.File;
@@ -54,6 +55,7 @@ import com.google.common.util.concurrent
import com.mongodb.MongoClient;
import com.mongodb.MongoClientURI;
import com.mongodb.MongoURI;
+import joptsimple.ArgumentAcceptingOptionSpec;
import joptsimple.OptionParser;
import joptsimple.OptionSet;
import joptsimple.OptionSpec;
@@ -147,6 +149,9 @@ public class Main {
case DEBUG:
debug(args);
break;
+ case CHECK:
+ check(args);
+ break;
case COMPACT:
compact(args);
break;
@@ -643,6 +648,40 @@ public class Main {
}
}
+ private static void check(String[] args) throws IOException {
+ OptionParser parser = new OptionParser();
+ ArgumentAcceptingOptionSpec<String> path = parser.accepts(
+ "path", "path to the segment store (required)")
+ .withRequiredArg().ofType(String.class);
+ ArgumentAcceptingOptionSpec<String> journal = parser.accepts(
+ "journal", "journal file")
+ .withRequiredArg().ofType(String.class).defaultsTo("journal.log");
+ ArgumentAcceptingOptionSpec<Long> deep = parser.accepts(
+ "deep", "enable deep consistency checking. An optional long " +
+ "specifies the number of seconds between progress notifications")
+ .withOptionalArg().ofType(Long.class).defaultsTo(Long.MAX_VALUE);
+
+ OptionSet options = parser.parse(args);
+
+ if (!options.has(path)) {
+ System.err.println("usage: check <options>");
+ parser.printHelpOn(System.err);
+ System.exit(1);
+ }
+
+ if (!isValidFileStore(path.value(options))) {
+ System.err.println("Invalid FileStore directory " + args[0]);
+ System.exit(1);
+ }
+
+ File dir = new File(path.value(options));
+ String journalFileName = journal.value(options);
+ boolean fullTraversal = options.has(deep);
+ long debugLevel = deep.value(options);
+
+ checkConsistency(dir, journalFileName, fullTraversal, debugLevel);
+ }
+
private static void debugTarFile(FileStore store, String[] args) {
File root = new File(args[0]);
for (int i = 1; i < args.length; i++) {
@@ -1075,6 +1114,7 @@ public class Main {
BENCHMARK("benchmark"),
CONSOLE("console"),
DEBUG("debug"),
+ CHECK("check"),
COMPACT("compact"),
SERVER("server"),
UPGRADE("upgrade"),
Modified: jackrabbit/oak/trunk/oak-run/src/main/resources/logback.xml
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/resources/logback.xml?rev=1644108&r1=1644107&r2=1644108&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/resources/logback.xml (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/resources/logback.xml Tue Dec 9 16:26:15 2014
@@ -36,6 +36,8 @@
<!-- Display info messages from the scalability suite -->
<logger name="org.apache.jackrabbit.oak.scalability" level="INFO"/>
+ <logger name="org.apache.jackrabbit.oak.plugins.segment.file.tooling.ConsistencyChecker" level="DEBUG"/>
+
<root level="warn">
<appender-ref ref="STDERR" />
</root>