You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by fr...@apache.org on 2017/02/03 10:31:47 UTC
svn commit: r1781526 - in /jackrabbit/oak/trunk:
oak-doc/src/site/markdown/nodestore/segment/
oak-run/src/main/java/org/apache/jackrabbit/oak/run/
oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/
oak-segment-tar/src/main/ja...
Author: frm
Date: Fri Feb 3 10:31:47 2017
New Revision: 1781526
URL: http://svn.apache.org/viewvc?rev=1781526&view=rev
Log:
OAK-5580 - Show statistics about I/O operations in the check command
Modified:
jackrabbit/oak/trunk/oak-doc/src/site/markdown/nodestore/segment/overview.md
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyChecker.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java
Modified: jackrabbit/oak/trunk/oak-doc/src/site/markdown/nodestore/segment/overview.md
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-doc/src/site/markdown/nodestore/segment/overview.md?rev=1781526&r1=1781525&r2=1781526&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-doc/src/site/markdown/nodestore/segment/overview.md (original)
+++ jackrabbit/oak/trunk/oak-doc/src/site/markdown/nodestore/segment/overview.md Fri Feb 3 10:31:47 2017
@@ -531,7 +531,7 @@ This tool is the counterpart of `backup`
### <a name="check"/> Check
```
-java -jar oak-run.jar check PATH [--journal JOURNAL] [--deep] [--notify SECS] [--bin [LENGTH]]
+java -jar oak-run.jar check PATH [--journal JOURNAL] [--deep] [--notify SECS] [--bin [LENGTH]] [--io-stats]
```
The `check` tool inspects an existing Segment Store at `PATH` for eventual inconsistencies.
@@ -553,6 +553,9 @@ If `LENGTH` is set to a value greater th
If `LENGTH` is set to `0`, the traversal is disabled.
The `--bin` property has no effect on binary properties stored in an external Blob Store.
+If the `--io-stats` option is specified, the tool will print some statistics about the I/O operations performed during the execution of the check command.
+This option is optional and is disabled by default.
+
### <a name="compact"/> Compact
```
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java?rev=1781526&r1=1781525&r2=1781526&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java Fri Feb 3 10:31:47 2017
@@ -44,6 +44,7 @@ class CheckCommand implements Command {
"bin", "read the first n bytes from binary properties.")
.withRequiredArg().ofType(Long.class);
OptionSpec segment = parser.accepts("segment", "Use oak-segment instead of oak-segment-tar");
+ OptionSpec ioStatistics = parser.accepts("io-stats", "Print I/O statistics (only for oak-segment-tar)");
OptionSet options = parser.parse(args);
@@ -69,7 +70,7 @@ class CheckCommand implements Command {
if (options.has(segment)) {
SegmentUtils.check(dir, journalFileName, fullTraversal, debugLevel, binLen);
} else {
- SegmentTarUtils.check(dir, journalFileName, fullTraversal, debugLevel, binLen);
+ SegmentTarUtils.check(dir, journalFileName, fullTraversal, debugLevel, binLen, options.has(ioStatistics));
}
}
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java?rev=1781526&r1=1781525&r2=1781526&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java Fri Feb 3 10:31:47 2017
@@ -184,13 +184,14 @@ final class SegmentTarUtils {
.run();
}
- static void check(File dir, String journalFileName, boolean fullTraversal, long debugLevel, long binLen) {
+ static void check(File dir, String journalFileName, boolean fullTraversal, long debugLevel, long binLen, boolean ioStatistics) {
Check.builder()
.withPath(dir)
.withJournal(journalFileName)
.withFullTraversal(fullTraversal)
.withDebugInterval(debugLevel)
.withMinimumBinaryLength(binLen)
+ .withIOStatistics(ioStatistics)
.build()
.run();
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyChecker.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyChecker.java?rev=1781526&r1=1781525&r2=1781526&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyChecker.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyChecker.java Fri Feb 3 10:31:47 2017
@@ -23,6 +23,7 @@ import static com.google.common.collect.
import static java.lang.Math.min;
import static org.apache.jackrabbit.oak.api.Type.BINARIES;
import static org.apache.jackrabbit.oak.api.Type.BINARY;
+import static org.apache.jackrabbit.oak.commons.IOUtils.humanReadableByteCount;
import static org.apache.jackrabbit.oak.commons.PathUtils.concat;
import static org.apache.jackrabbit.oak.commons.PathUtils.denotesRoot;
import static org.apache.jackrabbit.oak.commons.PathUtils.getName;
@@ -35,6 +36,7 @@ import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;
import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.api.PropertyState;
@@ -42,6 +44,8 @@ import org.apache.jackrabbit.oak.api.Typ
import org.apache.jackrabbit.oak.segment.SegmentBlob;
import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
import org.apache.jackrabbit.oak.segment.file.FileStore;
+import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
+import org.apache.jackrabbit.oak.segment.file.IOMonitorAdapter;
import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException;
import org.apache.jackrabbit.oak.segment.file.JournalReader;
import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore;
@@ -56,9 +60,27 @@ import org.slf4j.LoggerFactory;
* reporting that latest consistent revision.
*/
public class ConsistencyChecker implements Closeable {
+
+ private static class StatisticsIOMonitor extends IOMonitorAdapter {
+
+ private final AtomicLong ioOperations = new AtomicLong(0);
+
+ private final AtomicLong bytesRead = new AtomicLong(0);
+
+ @Override
+ public void onSegmentRead(File file, long msb, long lsb, int length) {
+ ioOperations.incrementAndGet();
+ bytesRead.addAndGet(length);
+ }
+
+ }
+
private static final Logger LOG = LoggerFactory.getLogger(ConsistencyChecker.class);
+ private final StatisticsIOMonitor statisticsIOMonitor = new StatisticsIOMonitor();
+
private final ReadOnlyFileStore store;
+
private final long debugInterval;
/**
@@ -71,18 +93,26 @@ public class ConsistencyChecker implemen
* @param debugInterval number of seconds between printing progress information to
* the console during the full traversal phase.
* @param binLen number of bytes to read from binary properties. -1 for all.
- * @return the latest consistent revision out of the revisions listed in the journal.
* @throws IOException
*/
- public static String checkConsistency(File directory, String journalFileName,
- boolean fullTraversal, long debugInterval, long binLen) throws IOException, InvalidFileStoreVersionException {
+ public static void checkConsistency(
+ File directory,
+ String journalFileName,
+ boolean fullTraversal,
+ long debugInterval,
+ long binLen,
+ boolean ioStatistics
+ ) throws IOException, InvalidFileStoreVersionException {
print("Searching for last good revision in {}", journalFileName);
- Set<String> badPaths = newHashSet();
try (
- JournalReader journal = new JournalReader(new File(directory, journalFileName));
- ConsistencyChecker checker = new ConsistencyChecker(directory, debugInterval)) {
+ JournalReader journal = new JournalReader(new File(directory, journalFileName));
+ ConsistencyChecker checker = new ConsistencyChecker(directory, debugInterval, ioStatistics)
+ ) {
+ Set<String> badPaths = newHashSet();
+ String latestGoodRevision = null;
int revisionCount = 0;
- while (journal.hasNext()) {
+
+ while (journal.hasNext() && latestGoodRevision == null) {
String revision = journal.next();
try {
print("Checking revision {}", revision);
@@ -94,7 +124,7 @@ public class ConsistencyChecker implemen
if (badPath == null) {
print("Found latest good revision {}", revision);
print("Searched through {} revisions", revisionCount);
- return revision;
+ latestGoodRevision = revision;
} else {
badPaths.add(badPath);
print("Broken revision {}", revision);
@@ -103,10 +133,23 @@ public class ConsistencyChecker implemen
print("Skipping invalid record id {}", revision);
}
}
- }
- print("No good revision found");
- return null;
+ if (ioStatistics) {
+ print(
+ "[I/O] Segment read operations: {}",
+ checker.statisticsIOMonitor.ioOperations
+ );
+ print(
+ "[I/O] Segment bytes read: {} ({} bytes)",
+ humanReadableByteCount(checker.statisticsIOMonitor.bytesRead.get()),
+ checker.statisticsIOMonitor.bytesRead
+ );
+ }
+
+ if (latestGoodRevision == null) {
+ print("No good revision found");
+ }
+ }
}
/**
@@ -117,9 +160,12 @@ public class ConsistencyChecker implemen
* the console during the full traversal phase.
* @throws IOException
*/
- public ConsistencyChecker(File directory, long debugInterval)
- throws IOException, InvalidFileStoreVersionException {
- store = fileStoreBuilder(directory).buildReadOnly();
+ public ConsistencyChecker(File directory, long debugInterval, boolean ioStatistics) throws IOException, InvalidFileStoreVersionException {
+ FileStoreBuilder builder = fileStoreBuilder(directory);
+ if (ioStatistics) {
+ builder.withIOMonitor(statisticsIOMonitor);
+ }
+ this.store = builder.buildReadOnly();
this.debugInterval = debugInterval;
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java?rev=1781526&r1=1781525&r2=1781526&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java Fri Feb 3 10:31:47 2017
@@ -53,6 +53,8 @@ public class Check implements Runnable {
private long minimumBinaryLength;
+ private boolean ioStatistics;
+
private Builder() {
// Prevent external instantiation.
}
@@ -115,7 +117,7 @@ public class Check implements Runnable {
* binary properties. If this parameter is
* set to {@code -1}, every binary property
* is read in its entirety.
- * @return
+ * @return this builder.
*/
public Builder withMinimumBinaryLength(long minimumBinaryLength) {
this.minimumBinaryLength = minimumBinaryLength;
@@ -123,6 +125,20 @@ public class Check implements Runnable {
}
/**
+ * Instruct the command to print statistics about I/O operations
+ * performed during the check. This parameter is not required and
+ * defaults to {@code false}.
+ *
+ * @param ioStatistics {@code true} if I/O statistics should be
+ * provided, {@code false} otherwise.
+ * @return this builder.
+ */
+ public Builder withIOStatistics(boolean ioStatistics) {
+ this.ioStatistics = ioStatistics;
+ return this;
+ }
+
+ /**
* Create an executable version of the {@link Check} command.
*
* @return an instance of {@link Runnable}.
@@ -145,18 +161,21 @@ public class Check implements Runnable {
private final long minimumBinaryLength;
+ private final boolean ioStatistics;
+
private Check(Builder builder) {
this.path = builder.path;
this.journal = builder.journal;
this.fullTraversal = builder.fullTraversal;
this.debugInterval = builder.debugInterval;
this.minimumBinaryLength = builder.minimumBinaryLength;
+ this.ioStatistics = builder.ioStatistics;
}
@Override
public void run() {
try {
- ConsistencyChecker.checkConsistency(path, journal, fullTraversal, debugInterval, minimumBinaryLength);
+ ConsistencyChecker.checkConsistency(path, journal, fullTraversal, debugInterval, minimumBinaryLength, ioStatistics);
} catch (Exception e) {
e.printStackTrace();
}