You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ad...@apache.org on 2017/02/13 12:50:08 UTC
svn commit: r1782761 - in /jackrabbit/oak/trunk:
oak-run/src/main/java/org/apache/jackrabbit/oak/run/
oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/
oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/ oa...
Author: adulceanu
Date: Mon Feb 13 12:50:07 2017
New Revision: 1782761
URL: http://svn.apache.org/viewvc?rev=1782761&view=rev
Log:
OAK-5600 - Test coverage for CheckCommand
Added tests for checking a valid repository
Added:
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tooling/
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tooling/CheckValidRepositoryTest.java
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyChecker.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java?rev=1782761&r1=1782760&r2=1782761&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java Mon Feb 13 12:50:07 2017
@@ -21,6 +21,7 @@ import static org.apache.jackrabbit.oak.
import java.io.File;
import java.io.IOException;
+import java.io.PrintWriter;
import joptsimple.ArgumentAcceptingOptionSpec;
import joptsimple.OptionParser;
@@ -65,10 +66,13 @@ class CheckCommand implements Command {
, "A deep scan of the content tree, traversing every node, will be performed by default.");
}
+ PrintWriter out = new PrintWriter(System.out, true);
+ PrintWriter err = new PrintWriter(System.err, true);
+
if (options.has(segment)) {
SegmentUtils.check(dir, journalFileName, true, debugLevel, binLen);
} else {
- SegmentTarUtils.check(dir, journalFileName, true, debugLevel, binLen, options.has(ioStatistics));
+ SegmentTarUtils.check(dir, journalFileName, true, debugLevel, binLen, options.has(ioStatistics), out, err);
}
}
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java?rev=1782761&r1=1782760&r2=1782761&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java Mon Feb 13 12:50:07 2017
@@ -24,6 +24,7 @@ import static org.apache.jackrabbit.oak.
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
+import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
@@ -184,7 +185,8 @@ final class SegmentTarUtils {
.run();
}
- static void check(File dir, String journalFileName, boolean fullTraversal, long debugLevel, long binLen, boolean ioStatistics) {
+ static void check(File dir, String journalFileName, boolean fullTraversal, long debugLevel, long binLen,
+ boolean ioStatistics, PrintWriter outWriter, PrintWriter errWriter) {
Check.builder()
.withPath(dir)
.withJournal(journalFileName)
@@ -192,6 +194,8 @@ final class SegmentTarUtils {
.withDebugInterval(debugLevel)
.withMinimumBinaryLength(binLen)
.withIOStatistics(ioStatistics)
+ .withOutWriter(outWriter)
+ .withErrWriter(errWriter)
.build()
.run();
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyChecker.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyChecker.java?rev=1782761&r1=1782760&r2=1782761&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyChecker.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyChecker.java Mon Feb 13 12:50:07 2017
@@ -35,6 +35,8 @@ import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
+import java.io.PrintWriter;
+import java.text.MessageFormat;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
@@ -51,8 +53,6 @@ import org.apache.jackrabbit.oak.segment
import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore;
import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
import org.apache.jackrabbit.oak.spi.state.NodeState;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* Utility for checking the files of a
@@ -75,13 +75,15 @@ public class ConsistencyChecker implemen
}
- private static final Logger LOG = LoggerFactory.getLogger(ConsistencyChecker.class);
-
private final StatisticsIOMonitor statisticsIOMonitor = new StatisticsIOMonitor();
private final ReadOnlyFileStore store;
private final long debugInterval;
+
+ private final PrintWriter outWriter;
+
+ private final PrintWriter errWriter;
/**
* Run a consistency check.
@@ -93,6 +95,10 @@ public class ConsistencyChecker implemen
* @param debugInterval number of seconds between printing progress information to
* the console during the full traversal phase.
* @param binLen number of bytes to read from binary properties. -1 for all.
+ * @param ioStatistics if {@code true} prints I/O statistics gathered while consistency
+ * check was performed
+ * @param outWriter text output stream writer
+ * @param errWriter text error stream writer
* @throws IOException
*/
public static void checkConsistency(
@@ -101,13 +107,15 @@ public class ConsistencyChecker implemen
boolean fullTraversal,
long debugInterval,
long binLen,
- boolean ioStatistics
+ boolean ioStatistics,
+ PrintWriter outWriter,
+ PrintWriter errWriter
) throws IOException, InvalidFileStoreVersionException {
- print("Searching for last good revision in {}", journalFileName);
try (
JournalReader journal = new JournalReader(new File(directory, journalFileName));
- ConsistencyChecker checker = new ConsistencyChecker(directory, debugInterval, ioStatistics)
+ ConsistencyChecker checker = new ConsistencyChecker(directory, debugInterval, ioStatistics, outWriter, errWriter)
) {
+ checker.print("Searching for last good revision in {0}", journalFileName);
Set<String> badPaths = newHashSet();
String latestGoodRevision = null;
int revisionCount = 0;
@@ -115,39 +123,39 @@ public class ConsistencyChecker implemen
while (journal.hasNext() && latestGoodRevision == null) {
String revision = journal.next();
try {
- print("Checking revision {}", revision);
+ checker.print("Checking revision {0}", revision);
revisionCount++;
String badPath = checker.check(revision, badPaths, binLen);
if (badPath == null && fullTraversal) {
badPath = checker.traverse(revision, binLen);
}
if (badPath == null) {
- print("Found latest good revision {}", revision);
- print("Searched through {} revisions", revisionCount);
+ checker.print("Found latest good revision {0}", revision);
+ checker.print("Searched through {0} revisions", revisionCount);
latestGoodRevision = revision;
} else {
badPaths.add(badPath);
- print("Broken revision {}", revision);
+ checker.print("Broken revision {0}", revision);
}
} catch (IllegalArgumentException e) {
- print("Skipping invalid record id {}", revision);
+ checker.printError("Skipping invalid record id {0}", revision);
}
}
if (ioStatistics) {
- print(
- "[I/O] Segment read operations: {}",
+ checker.print(
+ "[I/O] Segment read operations: {0}",
checker.statisticsIOMonitor.ioOperations
);
- print(
- "[I/O] Segment bytes read: {} ({} bytes)",
+ checker.print(
+ "[I/O] Segment bytes read: {0} ({1} bytes)",
humanReadableByteCount(checker.statisticsIOMonitor.bytesRead.get()),
checker.statisticsIOMonitor.bytesRead
);
}
if (latestGoodRevision == null) {
- print("No good revision found");
+ checker.print("No good revision found");
}
}
}
@@ -155,18 +163,25 @@ public class ConsistencyChecker implemen
/**
* Create a new consistency checker instance
*
- * @param directory directory containing the tar files
+ * @param directory directory containing the tar files
* @param debugInterval number of seconds between printing progress information to
* the console during the full traversal phase.
+ * @param ioStatistics if {@code true} prints I/O statistics gathered while consistency
+ * check was performed
+ * @param outWriter text output stream writer
+ * @param errWriter text error stream writer
* @throws IOException
*/
- public ConsistencyChecker(File directory, long debugInterval, boolean ioStatistics) throws IOException, InvalidFileStoreVersionException {
+ public ConsistencyChecker(File directory, long debugInterval, boolean ioStatistics, PrintWriter outWriter,
+ PrintWriter errWriter) throws IOException, InvalidFileStoreVersionException {
FileStoreBuilder builder = fileStoreBuilder(directory);
if (ioStatistics) {
builder.withIOMonitor(statisticsIOMonitor);
}
this.store = builder.buildReadOnly();
this.debugInterval = debugInterval;
+ this.outWriter = outWriter;
+ this.errWriter = errWriter;
}
/**
@@ -191,7 +206,7 @@ public class ConsistencyChecker implemen
private String checkPath(String path, long binLen) {
try {
- print("Checking {}", path);
+ print("Checking {0}", path);
NodeState root = SegmentNodeStoreBuilders.builder(store).build().getRoot();
String parentPath = getParentPath(path);
String name = getName(path);
@@ -202,7 +217,7 @@ public class ConsistencyChecker implemen
return traverse(parent, parentPath, false, binLen);
}
} catch (RuntimeException e) {
- print("Error while checking {}: {}", path, e.getMessage());
+ printError("Error while checking {0}: {1}", path, e.getMessage());
return path;
}
}
@@ -211,7 +226,7 @@ public class ConsistencyChecker implemen
private int propertyCount;
/**
- * Travers the given {@code revision}
+ * Traverse the given {@code revision}
* @param revision revision to travers
* @param binLen number of bytes to read from binary properties. -1 for all.
*/
@@ -222,20 +237,20 @@ public class ConsistencyChecker implemen
propertyCount = 0;
String result = traverse(SegmentNodeStoreBuilders.builder(store).build()
.getRoot(), "/", true, binLen);
- print("Checked {} nodes and {} properties", nodeCount, propertyCount);
+ print("Checked {0} nodes and {1} properties", nodeCount, propertyCount);
return result;
} catch (RuntimeException e) {
- print("Error while traversing {}", revision, e.getMessage());
+ printError("Error while traversing {0}", revision, e.getMessage());
return "/";
}
}
private String traverse(NodeState node, String path, boolean deep, long binLen) {
try {
- debug("Traversing {}", path);
+ debug("Traversing {0}", path);
nodeCount++;
for (PropertyState propertyState : node.getProperties()) {
- debug("Checking {}/{}", path, propertyState);
+ debug("Checking {0}/{1}", path, propertyState);
Type<?> type = propertyState.getType();
if (type == BINARY) {
traverse(propertyState.getValue(BINARY), binLen);
@@ -260,7 +275,7 @@ public class ConsistencyChecker implemen
}
return null;
} catch (RuntimeException | IOException e) {
- print("Error while traversing {}: {}", path, e.getMessage());
+ printError("Error while traversing {0}: {1}", path, e.getMessage());
return path;
}
}
@@ -297,29 +312,37 @@ public class ConsistencyChecker implemen
store.close();
}
- private static void print(String format) {
- LOG.info(format);
+ private void print(String format) {
+ outWriter.println(format);
}
- private static void print(String format, Object arg) {
- LOG.info(format, arg);
+ private void print(String format, Object arg) {
+ outWriter.println(MessageFormat.format(format, arg));
}
- private static void print(String format, Object arg1, Object arg2) {
- LOG.info(format, arg1, arg2);
+ private void print(String format, Object arg1, Object arg2) {
+ outWriter.println(MessageFormat.format(format, arg1, arg2));
+ }
+
+ private void printError(String format, Object arg) {
+ errWriter.println(MessageFormat.format(format, arg));
+ }
+
+ private void printError(String format, Object arg1, Object arg2) {
+ errWriter.println(MessageFormat.format(format, arg1, arg2));
}
private long ts;
private void debug(String format, Object arg) {
if (debug()) {
- LOG.debug(format, arg);
+ print(format, arg);
}
}
private void debug(String format, Object arg1, Object arg2) {
if (debug()) {
- LOG.debug(format, arg1, arg2);
+ print(format, arg1, arg2);
}
}
@@ -339,6 +362,4 @@ public class ConsistencyChecker implemen
return false;
}
}
-
-
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java?rev=1782761&r1=1782760&r2=1782761&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java Mon Feb 13 12:50:07 2017
@@ -21,6 +21,7 @@ import static com.google.common.base.Pre
import static com.google.common.base.Preconditions.checkNotNull;
import java.io.File;
+import java.io.PrintWriter;
import org.apache.jackrabbit.oak.segment.file.tooling.ConsistencyChecker;
@@ -54,6 +55,10 @@ public class Check implements Runnable {
private long minimumBinaryLength;
private boolean ioStatistics;
+
+ private PrintWriter outWriter;
+
+ private PrintWriter errWriter;
private Builder() {
// Prevent external instantiation.
@@ -137,6 +142,28 @@ public class Check implements Runnable {
this.ioStatistics = ioStatistics;
return this;
}
+
+ /**
+ * The text output stream writer used to print normal output.
+ * @param outWriter the output writer.
+ * @return this builder.
+ */
+ public Builder withOutWriter(PrintWriter outWriter) {
+ this.outWriter = outWriter;
+
+ return this;
+ }
+
+ /**
+ * The text error stream writer used to print erroneous output.
+ * @param errWriter the error writer.
+ * @return this builder.
+ */
+ public Builder withErrWriter(PrintWriter errWriter) {
+ this.errWriter = errWriter;
+
+ return this;
+ }
/**
* Create an executable version of the {@link Check} command.
@@ -162,6 +189,10 @@ public class Check implements Runnable {
private final long minimumBinaryLength;
private final boolean ioStatistics;
+
+ private final PrintWriter outWriter;
+
+ private final PrintWriter errWriter;
private Check(Builder builder) {
this.path = builder.path;
@@ -170,12 +201,14 @@ public class Check implements Runnable {
this.debugInterval = builder.debugInterval;
this.minimumBinaryLength = builder.minimumBinaryLength;
this.ioStatistics = builder.ioStatistics;
+ this.outWriter = builder.outWriter;
+ this.errWriter = builder.errWriter;
}
@Override
public void run() {
try {
- ConsistencyChecker.checkConsistency(path, journal, fullTraversal, debugInterval, minimumBinaryLength, ioStatistics);
+ ConsistencyChecker.checkConsistency(path, journal, fullTraversal, debugInterval, minimumBinaryLength, ioStatistics, outWriter, errWriter);
} catch (Exception e) {
e.printStackTrace();
}
Added: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tooling/CheckValidRepositoryTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tooling/CheckValidRepositoryTest.java?rev=1782761&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tooling/CheckValidRepositoryTest.java (added)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tooling/CheckValidRepositoryTest.java Mon Feb 13 12:50:07 2017
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.segment.file.tooling;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.jackrabbit.oak.segment.SegmentNodeStore;
+import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
+import org.apache.jackrabbit.oak.segment.file.FileStore;
+import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
+import org.apache.jackrabbit.oak.segment.tool.Check;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Tests for {@link CheckCommand}
+ */
+public class CheckValidRepositoryTest {
+ private static final Logger log = LoggerFactory.getLogger(CheckValidRepositoryTest.class);
+
+ @Rule
+ public final TemporaryFolder temporaryFolder = new TemporaryFolder(new File("target"));
+
+ @Before
+ public void setup() throws Exception {
+ FileStore fileStore = FileStoreBuilder.fileStoreBuilder(temporaryFolder.getRoot())
+ .withMaxFileSize(256)
+ .withSegmentCacheSize(64)
+ .build();
+
+ SegmentNodeStore nodeStore = SegmentNodeStoreBuilders.builder(fileStore).build();
+ NodeBuilder builder = nodeStore.getRoot().builder();
+
+ addChildWithBlobProperties(nodeStore, builder, "a", 5);
+ addChildWithBlobProperties(nodeStore, builder, "b", 10);
+ addChildWithBlobProperties(nodeStore, builder, "c", 15);
+
+ addChildWithProperties(nodeStore, builder, "d", 5);
+ addChildWithProperties(nodeStore, builder, "e", 5);
+ addChildWithProperties(nodeStore, builder, "f", 5);
+
+ nodeStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ fileStore.close();
+ }
+
+ @Test
+ public void testSuccessfulCheckWithBinaryTraversal() throws Exception {
+ StringWriter strOut = new StringWriter();
+ StringWriter strErr = new StringWriter();
+
+ PrintWriter outWriter = new PrintWriter(strOut, true);
+ PrintWriter errWriter = new PrintWriter(strErr, true);
+
+ Check.builder()
+ .withPath(new File(temporaryFolder.getRoot().getAbsolutePath()))
+ .withJournal("journal.log")
+ .withFullTraversal(true)
+ .withDebugInterval(Long.MAX_VALUE)
+ .withMinimumBinaryLength(Long.MAX_VALUE)
+ .withIOStatistics(true)
+ .withOutWriter(outWriter)
+ .withErrWriter(errWriter)
+ .build()
+ .run();
+
+ outWriter.close();
+ errWriter.close();
+
+ assertExpectedOutput(strOut.toString(), Lists.newArrayList("Searched through 1 revisions", "Checked 7 nodes and 45 properties"));
+ assertExpectedOutput(strErr.toString(), Lists.newArrayList(""));
+ }
+
+ @Test
+ public void testSuccessfulCheckWithoutBinaryTraversal() throws Exception {
+ StringWriter strOut = new StringWriter();
+ StringWriter strErr = new StringWriter();
+
+ PrintWriter outWriter = new PrintWriter(strOut, true);
+ PrintWriter errWriter = new PrintWriter(strErr, true);
+
+ Check.builder()
+ .withPath(new File(temporaryFolder.getRoot().getAbsolutePath()))
+ .withJournal("journal.log")
+ .withFullTraversal(true)
+ .withDebugInterval(Long.MAX_VALUE)
+ .withMinimumBinaryLength(0L)
+ .withIOStatistics(true)
+ .withOutWriter(outWriter)
+ .withErrWriter(errWriter)
+ .build()
+ .run();
+
+ outWriter.close();
+ errWriter.close();
+
+ assertExpectedOutput(strOut.toString(), Lists.newArrayList("Searched through 1 revisions", "Checked 7 nodes and 15 properties"));
+ assertExpectedOutput(strErr.toString(), Lists.newArrayList(""));
+ }
+
+ private static void assertExpectedOutput(String message, List<String> assertMessages) {
+ log.info("Assert message: {}", assertMessages);
+ log.info("Message logged: {}", message);
+
+
+ for (String msg : assertMessages) {
+ Assert.assertTrue(message.contains(msg));
+ }
+ }
+
+ private static void addChildWithBlobProperties(SegmentNodeStore nodeStore, NodeBuilder builder, String childName,
+ int propCount) throws IOException {
+ NodeBuilder child = builder.child(childName);
+ for (int i = 0; i < propCount; i++) {
+ child.setProperty(childName + i, nodeStore.createBlob(randomStream(i, 2000)));
+ }
+ }
+
+ private static void addChildWithProperties(SegmentNodeStore nodeStore, NodeBuilder builder, String childName,
+ int propCount) throws IOException {
+ NodeBuilder child = builder.child(childName);
+ for (int i = 0; i < propCount; i++) {
+ child.setProperty(childName + i, childName + i);
+ }
+ }
+
+ private static InputStream randomStream(int seed, int size) {
+ Random r = new Random(seed);
+ byte[] data = new byte[size];
+ r.nextBytes(data);
+ return new ByteArrayInputStream(data);
+ }
+}