You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ad...@apache.org on 2017/02/14 14:31:38 UTC
svn commit: r1782970 - in /jackrabbit/oak/trunk:
oak-doc/src/site/markdown/nodestore/segment/
oak-run/src/main/java/org/apache/jackrabbit/oak/run/
oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/
oak-segment-tar/src/main/ja...
Author: adulceanu
Date: Tue Feb 14 14:31:38 2017
New Revision: 1782970
URL: http://svn.apache.org/viewvc?rev=1782970&view=rev
Log:
OAK-5556 - Allow filter paths for Check command
Modified:
jackrabbit/oak/trunk/oak-doc/src/site/markdown/nodestore/segment/overview.md
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyChecker.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tooling/CheckValidRepositoryTest.java
Modified: jackrabbit/oak/trunk/oak-doc/src/site/markdown/nodestore/segment/overview.md
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-doc/src/site/markdown/nodestore/segment/overview.md?rev=1782970&r1=1782969&r2=1782970&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-doc/src/site/markdown/nodestore/segment/overview.md (original)
+++ jackrabbit/oak/trunk/oak-doc/src/site/markdown/nodestore/segment/overview.md Tue Feb 14 14:31:38 2017
@@ -531,7 +531,7 @@ This tool is the counterpart of `backup`
### <a name="check"/> Check
```
-java -jar oak-run.jar check PATH [--journal JOURNAL] [--notify SECS] [--bin] [--io-stats]
+java -jar oak-run.jar check PATH [--journal JOURNAL] [--notify SECS] [--bin] [--filter PATH1[,PATH2,..,PATHn]] [--io-stats]
```
The `check` tool inspects an existing Segment Store at `PATH` for eventual inconsistencies.
@@ -550,6 +550,12 @@ If the `--bin` option is specified, the
If not specified, the binary properties will not be traversed.
The `--bin` option has no effect on binary properties stored in an external Blob Store.
+If the `--filter` option is specified, the tool will traverse only the absolute paths specified as arguments.
+At least one argument is expected with this option; multiple arguments need to be comma-separated.
+The paths will be traversed in the same order as they were specified.
+If one of the paths is invalid, the consistency check will fail and the traversal will not continue for the rest of the paths.
+If the option is not specified, the full traversal of the repository (rooted at `/`) will be performed.
+
If the `--io-stats` option is specified, the tool will print some statistics about the I/O operations performed during the execution of the check command.
This option is optional and is disabled by default.
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java?rev=1782970&r1=1782969&r2=1782970&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java Tue Feb 14 14:31:38 2017
@@ -22,6 +22,8 @@ import static org.apache.jackrabbit.oak.
import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
+import java.util.LinkedHashSet;
+import java.util.Set;
import joptsimple.ArgumentAcceptingOptionSpec;
import joptsimple.OptionParser;
@@ -43,6 +45,9 @@ class CheckCommand implements Command {
.withRequiredArg().ofType(Long.class).defaultsTo(Long.MAX_VALUE);
OptionSpec<?> bin = parser.accepts("bin", "read the content of binary properties");
OptionSpec<?> segment = parser.accepts("segment", "Use oak-segment instead of oak-segment-tar");
+ ArgumentAcceptingOptionSpec<String> filter = parser.accepts(
+ "filter", "comma separated content paths to be checked")
+ .withRequiredArg().ofType(String.class).withValuesSeparatedBy(',').defaultsTo("/");
OptionSpec<?> ioStatistics = parser.accepts("io-stats", "Print I/O statistics (only for oak-segment-tar)");
OptionSet options = parser.parse(args);
@@ -57,17 +62,17 @@ class CheckCommand implements Command {
File dir = isValidFileStoreOrFail(new File(options.nonOptionArguments().get(0).toString()));
String journalFileName = journal.value(options);
long debugLevel = notify.value(options);
-
+ Set<String> filterPaths = new LinkedHashSet<String>(filter.values(options));
+
if (options.has(deep)) {
printUsage(parser, err, "The --deep option was deprecated! Please do not use it in the future!"
, "A deep scan of the content tree, traversing every node, will be performed by default.");
}
-
if (options.has(segment)) {
SegmentUtils.check(dir, journalFileName, debugLevel, options.has(bin));
} else {
- SegmentTarUtils.check(dir, journalFileName, debugLevel, options.has(bin), options.has(ioStatistics), out, err);
+ SegmentTarUtils.check(dir, journalFileName, debugLevel, options.has(bin), filterPaths, options.has(ioStatistics), out, err);
}
}
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java?rev=1782970&r1=1782969&r2=1782970&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java Tue Feb 14 14:31:38 2017
@@ -28,8 +28,8 @@ import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
+import java.util.Set;
-import com.google.common.io.Closer;
import org.apache.jackrabbit.oak.plugins.blob.BlobReferenceRetriever;
import org.apache.jackrabbit.oak.segment.SegmentBlobReferenceRetriever;
import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
@@ -52,6 +52,8 @@ import org.apache.jackrabbit.oak.segment
import org.apache.jackrabbit.oak.segment.tool.SegmentGraph;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import com.google.common.io.Closer;
+
final class SegmentTarUtils {
private static final boolean TAR_STORAGE_MEMORY_MAPPED = Boolean.getBoolean("tar.memoryMapped");
@@ -185,13 +187,14 @@ final class SegmentTarUtils {
.run();
}
- static void check(File dir, String journalFileName, long debugLevel, boolean checkBinaries, boolean ioStatistics,
+ static void check(File dir, String journalFileName, long debugLevel, boolean checkBinaries, Set<String> filterPaths, boolean ioStatistics,
PrintWriter outWriter, PrintWriter errWriter) {
Check.builder()
.withPath(dir)
.withJournal(journalFileName)
.withDebugInterval(debugLevel)
.withCheckBinaries(checkBinaries)
+ .withFilterPaths(filterPaths)
.withIOStatistics(ioStatistics)
.withOutWriter(outWriter)
.withErrWriter(errWriter)
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyChecker.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyChecker.java?rev=1782970&r1=1782969&r2=1782970&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyChecker.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyChecker.java Tue Feb 14 14:31:38 2017
@@ -98,7 +98,8 @@ public class ConsistencyChecker implemen
* @param journalFileName name of the journal file containing the revision history
* @param debugInterval number of seconds between printing progress information to
* the console during the full traversal phase.
- * @param checkBinaries if {@code true} full content of binary properties will be scanned
+ * @param checkBinaries if {@code true} full content of binary properties will be scanned
+ * @param filterPaths collection of repository paths to be checked
* @param ioStatistics if {@code true} prints I/O statistics gathered while consistency
* check was performed
* @param outWriter text output stream writer
@@ -110,6 +111,7 @@ public class ConsistencyChecker implemen
String journalFileName,
long debugInterval,
boolean checkBinaries,
+ Set<String> filterPaths,
boolean ioStatistics,
PrintWriter outWriter,
PrintWriter errWriter
@@ -127,7 +129,7 @@ public class ConsistencyChecker implemen
try {
revisionCount++;
- String corruptPath = checker.checkRevision(revision, corruptPaths, "/", checkBinaries);
+ String corruptPath = checker.checkRevision(revision, corruptPaths, filterPaths, checkBinaries);
if (corruptPath == null) {
checker.print("Found latest good revision {0}", revision);
@@ -190,23 +192,22 @@ public class ConsistencyChecker implemen
/**
- * Checks the consistency of the supplied {@code path} at the given {@code revision},
+ * Checks the consistency of the supplied {@code paths} at the given {@code revision},
* starting first with already known {@code corruptPaths}.
*
* @param revision revision to be checked
* @param corruptPaths already known corrupt paths from previous revisions
- * @param path initial path from which to start the consistency check,
+ * @param paths paths on which to run consistency check,
* provided there are no corrupt paths.
* @param checkBinaries if {@code true} full content of binary properties will be scanned
* @return {@code null}, if the content tree rooted at path is consistent
* in this revision or the path of the first inconsistency otherwise.
*/
- public String checkRevision(String revision, Set<String> corruptPaths, String path, boolean checkBinaries) {
+ public String checkRevision(String revision, Set<String> corruptPaths, Set<String> paths, boolean checkBinaries) {
print("Checking revision {0}", revision);
String result = null;
try {
- print("Checking {0}", path);
store.setRevision(revision);
NodeState root = SegmentNodeStoreBuilders.builder(store).build().getRoot();
@@ -222,14 +223,23 @@ public class ConsistencyChecker implemen
nodeCount = 0;
propertyCount = 0;
- NodeWrapper wrapper = NodeWrapper.deriveTraversableNodeOnPath(root, path);
- result = checkNodeAndDescendants(wrapper.node, wrapper.path, checkBinaries);
- print("Checked {0} nodes and {1} properties", nodeCount, propertyCount);
+ for (String path : paths) {
+ print("Checking {0}", path);
+
+ NodeWrapper wrapper = NodeWrapper.deriveTraversableNodeOnPath(root, path);
+ result = checkNodeAndDescendants(wrapper.node, wrapper.path, checkBinaries);
+
+ if (result != null) {
+ break;
+ }
+ }
return result;
} catch (RuntimeException e) {
printError("Error while traversing {0}: {1}", revision, e.getMessage());
- return path;
+ return "/";
+ } finally {
+ print("Checked {0} nodes and {1} properties", nodeCount, propertyCount);
}
}
@@ -313,7 +323,11 @@ public class ConsistencyChecker implemen
String name = getName(path);
NodeState parent = getNode(root, parentPath);
- if (!denotesRoot(path) && parent.hasChildNode(name)) {
+ if (!denotesRoot(path)) {
+ if (!parent.hasChildNode(name)) {
+ throw new IllegalArgumentException("Invalid path: " + path);
+ }
+
return new NodeWrapper(parent.getChildNode(name), path);
} else {
return new NodeWrapper(parent, parentPath);
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java?rev=1782970&r1=1782969&r2=1782970&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java Tue Feb 14 14:31:38 2017
@@ -22,6 +22,7 @@ import static com.google.common.base.Pre
import java.io.File;
import java.io.PrintWriter;
+import java.util.Set;
import org.apache.jackrabbit.oak.segment.file.tooling.ConsistencyChecker;
@@ -51,6 +52,8 @@ public class Check implements Runnable {
private long debugInterval = Long.MAX_VALUE;
private boolean checkBinaries;
+
+ private Set<String> filterPaths;
private boolean ioStatistics;
@@ -111,6 +114,19 @@ public class Check implements Runnable {
this.checkBinaries = checkBinaries;
return this;
}
+
+ /**
+ * Content paths to be checked. This parameter is not required and
+ * defaults to "/".
+ *
+ * @param filterPaths
+ * paths to be checked
+ * @return this builder.
+ */
+ public Builder withFilterPaths(Set<String> filterPaths) {
+ this.filterPaths = filterPaths;
+ return this;
+ }
/**
* Instruct the command to print statistics about I/O operations
@@ -168,6 +184,8 @@ public class Check implements Runnable {
private final long debugInterval;
private final boolean checkBinaries;
+
+ private final Set<String> filterPaths;
private final boolean ioStatistics;
@@ -180,6 +198,7 @@ public class Check implements Runnable {
this.journal = builder.journal;
this.debugInterval = builder.debugInterval;
this.checkBinaries = builder.checkBinaries;
+ this.filterPaths = builder.filterPaths;
this.ioStatistics = builder.ioStatistics;
this.outWriter = builder.outWriter;
this.errWriter = builder.errWriter;
@@ -188,7 +207,7 @@ public class Check implements Runnable {
@Override
public void run() {
try {
- ConsistencyChecker.checkConsistency(path, journal, debugInterval, checkBinaries, ioStatistics, outWriter, errWriter);
+ ConsistencyChecker.checkConsistency(path, journal, debugInterval, checkBinaries, filterPaths, ioStatistics, outWriter, errWriter);
} catch (Exception e) {
e.printStackTrace();
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tooling/CheckValidRepositoryTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tooling/CheckValidRepositoryTest.java?rev=1782970&r1=1782969&r2=1782970&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tooling/CheckValidRepositoryTest.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tooling/CheckValidRepositoryTest.java Tue Feb 14 14:31:38 2017
@@ -24,8 +24,10 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
import java.io.StringWriter;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Random;
+import java.util.Set;
import org.apache.jackrabbit.oak.segment.SegmentNodeStore;
import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
@@ -77,18 +79,22 @@ public class CheckValidRepositoryTest {
}
@Test
- public void testSuccessfulCheckWithBinaryTraversal() throws Exception {
+ public void testSuccessfulFullCheckWithBinaryTraversal() throws Exception {
StringWriter strOut = new StringWriter();
StringWriter strErr = new StringWriter();
PrintWriter outWriter = new PrintWriter(strOut, true);
PrintWriter errWriter = new PrintWriter(strErr, true);
+ Set<String> filterPaths = new LinkedHashSet<>();
+ filterPaths.add("/");
+
Check.builder()
.withPath(new File(temporaryFolder.getRoot().getAbsolutePath()))
.withJournal("journal.log")
.withDebugInterval(Long.MAX_VALUE)
.withCheckBinaries(true)
+ .withFilterPaths(filterPaths)
.withIOStatistics(true)
.withOutWriter(outWriter)
.withErrWriter(errWriter)
@@ -103,17 +109,56 @@ public class CheckValidRepositoryTest {
}
@Test
- public void testSuccessfulCheckWithoutBinaryTraversal() throws Exception {
+ public void testSuccessfulOnlyRootKidsCheckWithBinaryTraversalAndFilterPaths() throws Exception {
StringWriter strOut = new StringWriter();
StringWriter strErr = new StringWriter();
PrintWriter outWriter = new PrintWriter(strOut, true);
PrintWriter errWriter = new PrintWriter(strErr, true);
+ Set<String> filterPaths = new LinkedHashSet<>();
+ filterPaths.add("/a");
+ filterPaths.add("/b");
+ filterPaths.add("/c");
+ filterPaths.add("/d");
+ filterPaths.add("/e");
+ filterPaths.add("/f");
+
Check.builder()
.withPath(new File(temporaryFolder.getRoot().getAbsolutePath()))
.withJournal("journal.log")
.withDebugInterval(Long.MAX_VALUE)
+ .withCheckBinaries(true)
+ .withFilterPaths(filterPaths)
+ .withIOStatistics(true)
+ .withOutWriter(outWriter)
+ .withErrWriter(errWriter)
+ .build()
+ .run();
+
+ outWriter.close();
+ errWriter.close();
+
+ assertExpectedOutput(strOut.toString(), Lists.newArrayList("Searched through 1 revisions", "Checked 6 nodes and 45 properties"));
+ assertExpectedOutput(strErr.toString(), Lists.newArrayList(""));
+ }
+
+ @Test
+ public void testSuccessfulFullCheckWithoutBinaryTraversal() throws Exception {
+ StringWriter strOut = new StringWriter();
+ StringWriter strErr = new StringWriter();
+
+ PrintWriter outWriter = new PrintWriter(strOut, true);
+ PrintWriter errWriter = new PrintWriter(strErr, true);
+
+ Set<String> filterPaths = new LinkedHashSet<>();
+ filterPaths.add("/");
+
+ Check.builder()
+ .withPath(new File(temporaryFolder.getRoot().getAbsolutePath()))
+ .withJournal("journal.log")
+ .withDebugInterval(Long.MAX_VALUE)
+ .withFilterPaths(filterPaths)
.withIOStatistics(true)
.withOutWriter(outWriter)
.withErrWriter(errWriter)
@@ -127,6 +172,101 @@ public class CheckValidRepositoryTest {
assertExpectedOutput(strErr.toString(), Lists.newArrayList(""));
}
+ @Test
+ public void testSuccessfulPartialCheckWithoutBinaryTraversal() throws Exception {
+ StringWriter strOut = new StringWriter();
+ StringWriter strErr = new StringWriter();
+
+ PrintWriter outWriter = new PrintWriter(strOut, true);
+ PrintWriter errWriter = new PrintWriter(strErr, true);
+
+ Set<String> filterPaths = new LinkedHashSet<>();
+ filterPaths.add("/a");
+ filterPaths.add("/b");
+ filterPaths.add("/d");
+ filterPaths.add("/e");
+
+ Check.builder()
+ .withPath(new File(temporaryFolder.getRoot().getAbsolutePath()))
+ .withJournal("journal.log")
+ .withDebugInterval(Long.MAX_VALUE)
+ .withFilterPaths(filterPaths)
+ .withIOStatistics(true)
+ .withOutWriter(outWriter)
+ .withErrWriter(errWriter)
+ .build()
+ .run();
+
+ outWriter.close();
+ errWriter.close();
+
+ assertExpectedOutput(strOut.toString(), Lists.newArrayList("Searched through 1 revisions", "Checked 4 nodes and 10 properties"));
+ assertExpectedOutput(strErr.toString(), Lists.newArrayList(""));
+ }
+
+ @Test
+ public void testUnsuccessfulPartialCheckWithoutBinaryTraversal() throws Exception {
+ StringWriter strOut = new StringWriter();
+ StringWriter strErr = new StringWriter();
+
+ PrintWriter outWriter = new PrintWriter(strOut, true);
+ PrintWriter errWriter = new PrintWriter(strErr, true);
+
+ Set<String> filterPaths = new LinkedHashSet<>();
+ filterPaths.add("/g");
+
+ Check.builder()
+ .withPath(new File(temporaryFolder.getRoot().getAbsolutePath()))
+ .withJournal("journal.log")
+ .withDebugInterval(Long.MAX_VALUE)
+ .withFilterPaths(filterPaths)
+ .withIOStatistics(true)
+ .withOutWriter(outWriter)
+ .withErrWriter(errWriter)
+ .build()
+ .run();
+
+ outWriter.close();
+ errWriter.close();
+
+ assertExpectedOutput(strOut.toString(), Lists.newArrayList("Broken revision", "Checked 0 nodes and 0 properties", "No good revision found"));
+ assertExpectedOutput(strErr.toString(), Lists.newArrayList("Invalid path: /g"));
+ }
+
+ @Test
+ public void testUnsuccessfulPartialCheckWithBinaryTraversal() throws Exception {
+ StringWriter strOut = new StringWriter();
+ StringWriter strErr = new StringWriter();
+
+ PrintWriter outWriter = new PrintWriter(strOut, true);
+ PrintWriter errWriter = new PrintWriter(strErr, true);
+
+ Set<String> filterPaths = new LinkedHashSet<>();
+ filterPaths.add("/a");
+ filterPaths.add("/f");
+ filterPaths.add("/g");
+ filterPaths.add("/d");
+ filterPaths.add("/e");
+
+ Check.builder()
+ .withPath(new File(temporaryFolder.getRoot().getAbsolutePath()))
+ .withJournal("journal.log")
+ .withDebugInterval(Long.MAX_VALUE)
+ .withFilterPaths(filterPaths)
+ .withCheckBinaries(true)
+ .withIOStatistics(true)
+ .withOutWriter(outWriter)
+ .withErrWriter(errWriter)
+ .build()
+ .run();
+
+ outWriter.close();
+ errWriter.close();
+
+ assertExpectedOutput(strOut.toString(), Lists.newArrayList("Broken revision", "Checked 2 nodes and 10 properties", "No good revision found"));
+ assertExpectedOutput(strErr.toString(), Lists.newArrayList("Invalid path: /g"));
+ }
+
private static void assertExpectedOutput(String message, List<String> assertMessages) {
log.info("Assert message: {}", assertMessages);
log.info("Message logged: {}", message);