You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by am...@apache.org on 2016/12/02 10:15:57 UTC
svn commit: r1772323 - in /jackrabbit/oak/branches/1.4: ./
oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/
oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/
oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/ oak-core...
Author: amitj
Date: Fri Dec 2 10:15:57 2016
New Revision: 1772323
URL: http://svn.apache.org/viewvc?rev=1772323&view=rev
Log:
OAK-4476: Option to check datastore consistency in oak-run
Merge r1750887, r1751396, r1751419 from trunk
Added:
jackrabbit/oak/branches/1.4/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java
- copied, changed from r1750887, jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java
jackrabbit/oak/branches/1.4/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java
- copied, changed from r1750887, jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java
jackrabbit/oak/branches/1.4/oak-run/src/test/resources/
- copied from r1751419, jackrabbit/oak/trunk/oak-run/src/test/resources/
Modified:
jackrabbit/oak/branches/1.4/ (props changed)
jackrabbit/oak/branches/1.4/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java
jackrabbit/oak/branches/1.4/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileIOUtilsTest.java
jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/FileLineDifferenceIteratorTest.java
jackrabbit/oak/branches/1.4/oak-run/README.md
jackrabbit/oak/branches/1.4/oak-run/pom.xml
jackrabbit/oak/branches/1.4/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Mode.java
jackrabbit/oak/branches/1.4/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java
Propchange: jackrabbit/oak/branches/1.4/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 2 10:15:57 2016
@@ -1,3 +1,3 @@
/jackrabbit/oak/branches/1.0:1665962
-/jackrabbit/oak/trunk:1733615,1733875,1733913,1733929,1734230,1734254,1734279,1734941,1735052,1735081,1735109,1735141,1735267,1735405,1735484,1735549,1735564,1735588,1735622,1735638,1735919,1735983,1736176,1737309-1737310,1737334,1737349,1737998,1738004,1738136,1738138,1738207,1738234,1738252,1738775,1738795,1738833,1738950,1738957,1738963,1739712,1739760,1739867,1739894,1739959-1739960,1740114,1740116,1740250,1740333,1740349,1740360,1740625-1740626,1740774,1740837,1740879,1740971,1741016,1741032,1741339,1741343,1742077,1742117,1742125,1742363,1742520,1742888,1742916,1743097,1743172,1743343,1743674,1744265,1744292,1744589,1744670,1744672,1744959,1745038,1745127,1745197,1745336,1745368,1746086,1746117,1746342,1746345,1746408,1746696,1746981,1747198,1747200,1747341-1747342,1747380,1747387,1747406,1747492,1747512,1747654,1748505,1748553,1748722,1748870,1749275,1749350,1749424,1749443,1749464,1749475,1749645,1749662,1749815,1749872,1749875,1749899,1750052,1750076-1750077,1750287,1750457
,1750462,1750465,1750495,1750626,1750809,1750886,1751410,1751445-1751446,1751478,1751753,1751755,1751871,1752198,1752202,1752259,1752273-1752274,1752283,1752292,1752438,1752447-1752448,1752508,1752596,1752616,1752659,1752672,1753262,1753331-1753332,1753335-1753336,1753355,1753444,1754117,1754239,1755157,1755191,1756520,1756580,1757119,1757166,1758213,1758713,1759433,1759795,1759826,1760326,1760340,1760373,1760387,1760486,1760492,1760494,1760661-1760662,1761412,1761444,1761571,1761762,1761787,1761876,1762453,1762612,1762632,1762635,1763347,1763355-1763356,1763378,1763465,1763735,1764678,1764705,1764814,1764898,1765817,1765983,1766071,1766423,1766496,1766519,1766554,1766644,1767265,1768446,1768637,1770982,1771022,1771093,1771098,1771739,1771852,1771870,1771902,1772228
+/jackrabbit/oak/trunk:1733615,1733875,1733913,1733929,1734230,1734254,1734279,1734941,1735052,1735081,1735109,1735141,1735267,1735405,1735484,1735549,1735564,1735588,1735622,1735638,1735919,1735983,1736176,1737309-1737310,1737334,1737349,1737998,1738004,1738136,1738138,1738207,1738234,1738252,1738775,1738795,1738833,1738950,1738957,1738963,1739712,1739760,1739867,1739894,1739959-1739960,1740114,1740116,1740250,1740333,1740349,1740360,1740625-1740626,1740774,1740837,1740879,1740971,1741016,1741032,1741339,1741343,1742077,1742117,1742125,1742363,1742520,1742888,1742916,1743097,1743172,1743343,1743674,1744265,1744292,1744589,1744670,1744672,1744959,1745038,1745127,1745197,1745336,1745368,1746086,1746117,1746342,1746345,1746408,1746696,1746981,1747198,1747200,1747341-1747342,1747380,1747387,1747406,1747492,1747512,1747654,1748505,1748553,1748722,1748870,1749275,1749350,1749424,1749443,1749464,1749475,1749645,1749662,1749815,1749872,1749875,1749899,1750052,1750076-1750077,1750287,1750457
,1750462,1750465,1750495,1750626,1750809,1750886-1750887,1751396,1751410,1751419,1751445-1751446,1751478,1751753,1751755,1751871,1752198,1752202,1752259,1752273-1752274,1752283,1752292,1752438,1752447-1752448,1752508,1752596,1752616,1752659,1752672,1753262,1753331-1753332,1753335-1753336,1753355,1753444,1754117,1754239,1755157,1755191,1756520,1756580,1757119,1757166,1758213,1758713,1759433,1759795,1759826,1760326,1760340,1760373,1760387,1760486,1760492,1760494,1760661-1760662,1761412,1761444,1761571,1761762,1761787,1761876,1762453,1762612,1762632,1762635,1763347,1763355-1763356,1763378,1763465,1763735,1764678,1764705,1764814,1764898,1765817,1765983,1766071,1766423,1766496,1766519,1766554,1766644,1767265,1768446,1768637,1770982,1771022,1771093,1771098,1771739,1771852,1771870,1771902,1772228
/jackrabbit/trunk:1345480
Modified: jackrabbit/oak/branches/1.4/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java?rev=1772323&r1=1772322&r2=1772323&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.4/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java (original)
+++ jackrabbit/oak/branches/1.4/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java Fri Dec 2 10:15:57 2016
@@ -18,24 +18,35 @@ package org.apache.jackrabbit.oak.common
import java.io.BufferedReader;
import java.io.BufferedWriter;
+import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.Comparator;
import java.util.Iterator;
+import java.util.List;
import java.util.Set;
import javax.annotation.Nullable;
import com.google.common.base.Charsets;
import com.google.common.base.Function;
+import com.google.common.collect.AbstractIterator;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.PeekingIterator;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.LineIterator;
import static com.google.common.collect.Sets.newHashSet;
import static com.google.common.io.Closeables.close;
+import static com.google.common.io.Files.move;
import static com.google.common.io.Files.newWriter;
+import static java.io.File.createTempFile;
import static org.apache.jackrabbit.oak.commons.sort.EscapeUtils.escapeLineBreak;
import static org.apache.jackrabbit.oak.commons.sort.EscapeUtils.unescapeLineBreaks;
+import static org.apache.jackrabbit.oak.commons.sort.ExternalSort.mergeSortedFiles;
+import static org.apache.jackrabbit.oak.commons.sort.ExternalSort.sortInBatch;
/**
* Simple File utils
@@ -45,6 +56,49 @@ public final class FileIOUtils {
private FileIOUtils() {
}
+ public final static Comparator<String> lexComparator = new Comparator<String>() {
+ @Override public int compare(String s1, String s2) {
+ return s1.compareTo(s2);
+ }
+ };
+
+ /**
+ * Sorts the given file externally using the {@link #lexComparator} and removes duplicates.
+ *
+ * @param file file whose contents needs to be sorted
+ */
+ public static void sort(File file) throws IOException {
+ File sorted = createTempFile("temp", null);
+ merge(sortInBatch(file, lexComparator, true), sorted);
+ move(sorted, file);
+ }
+
+ /**
+ * Sorts the given file externally with the given comparator and removes duplicates.
+ *
+ * @param file file whose contents needs to be sorted
+ * @param comparator to compare
+ * @throws IOException
+ */
+ public static void sort(File file, Comparator<String> comparator) throws IOException {
+ File sorted = createTempFile("temp", null);
+ merge(sortInBatch(file, comparator, true), sorted);
+ move(sorted, file);
+ }
+
+ /**
+ * Merges a list of files after sorting with the {@link #lexComparator}.
+ *
+ * @param files files to merge
+ * @param output merge output file
+ * @throws IOException
+ */
+ public static void merge(List<File> files, File output) throws IOException {
+ mergeSortedFiles(
+ files,
+ output, lexComparator, true);
+ }
+
/**
* Writes a string as a new line into the given buffered writer and optionally
* escapes the line for line breaks.
@@ -156,4 +210,94 @@ public final class FileIOUtils {
return delegate.compare(func.apply(s1), func.apply(s2));
}
}
+
+ /**
+ * FileLineDifferenceIterator class which iterates over the difference of 2 files line by line.
+ *
+ * If there is a scope for lines in files containing line break characters it should be
+ * ensured that both the file are written with
+ * {@link #writeAsLine(BufferedWriter, String, boolean)} with true to escape line break
+ * characters.
+ */
+ public static class FileLineDifferenceIterator extends AbstractIterator<String> implements Closeable {
+ private final PeekingIterator<String> peekMarked;
+ private final LineIterator marked;
+ private final LineIterator all;
+ private Function<String, String> transformer = new Function<String, String>() {
+ @Override
+ public String apply(String input) {
+ return input;
+ }
+ };
+
+ public FileLineDifferenceIterator(LineIterator marked, LineIterator available) throws IOException {
+ this(marked, available, null);
+ }
+
+ public FileLineDifferenceIterator(File marked, File available,
+ @Nullable Function<String, String> transformer) throws IOException {
+ this(FileUtils.lineIterator(marked), FileUtils.lineIterator(available), transformer);
+ }
+
+ public FileLineDifferenceIterator(LineIterator marked, LineIterator available,
+ @Nullable Function<String, String> transformer) throws IOException {
+ this.marked = marked;
+ this.peekMarked = Iterators.peekingIterator(marked);
+ this.all = available;
+ if (transformer != null) {
+ this.transformer = transformer;
+ }
+ }
+
+ @Override
+ protected String computeNext() {
+ String diff = computeNextDiff();
+ if (diff == null) {
+ close();
+ return endOfData();
+ }
+ return diff;
+ }
+
+ @Override
+ public void close() {
+ LineIterator.closeQuietly(marked);
+ LineIterator.closeQuietly(all);
+ }
+
+ private String computeNextDiff() {
+ if (!all.hasNext()) {
+ return null;
+ }
+
+ //Marked finish the rest of all are part of diff
+ if (!peekMarked.hasNext()) {
+ return all.next();
+ }
+
+ String diff = null;
+ while (all.hasNext() && diff == null) {
+ diff = all.next();
+ while (peekMarked.hasNext()) {
+ String marked = peekMarked.peek();
+ int comparisonResult = transformer.apply(diff).compareTo(transformer.apply((marked)));
+ if (comparisonResult > 0) {
+ //Extra entries in marked. Ignore them and move on
+ peekMarked.next();
+ } else if (comparisonResult == 0) {
+ //Matching entry found in marked move past it. Not a
+ //dif candidate
+ peekMarked.next();
+ diff = null;
+ break;
+ } else {
+ //This entry is not found in marked entries
+ //hence part of diff
+ return diff;
+ }
+ }
+ }
+ return diff;
+ }
+ }
}
Modified: jackrabbit/oak/branches/1.4/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileIOUtilsTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileIOUtilsTest.java?rev=1772323&r1=1772322&r2=1772323&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.4/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileIOUtilsTest.java (original)
+++ jackrabbit/oak/branches/1.4/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileIOUtilsTest.java Fri Dec 2 10:15:57 2016
@@ -18,13 +18,17 @@
*/
package org.apache.jackrabbit.oak.commons;
+import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
+import java.io.FileReader;
import java.io.IOException;
+import java.io.InputStreamReader;
import java.nio.CharBuffer;
import java.nio.charset.CharacterCodingException;
import java.nio.charset.Charset;
import java.nio.charset.CharsetEncoder;
+import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
@@ -32,17 +36,21 @@ import java.util.Random;
import java.util.Set;
import com.google.common.base.Charsets;
-import com.google.common.collect.Lists;
-import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
+import static com.google.common.collect.Lists.newArrayList;
import static com.google.common.collect.Sets.newHashSet;
+import static org.apache.jackrabbit.oak.commons.FileIOUtils.lexComparator;
+import static org.apache.jackrabbit.oak.commons.FileIOUtils.lineBreakAwareComparator;
import static org.apache.jackrabbit.oak.commons.FileIOUtils.readStringsAsSet;
+import static org.apache.jackrabbit.oak.commons.FileIOUtils.sort;
import static org.apache.jackrabbit.oak.commons.FileIOUtils.writeStrings;
import static org.apache.jackrabbit.oak.commons.sort.EscapeUtils.escapeLineBreak;
import static org.apache.jackrabbit.oak.commons.sort.EscapeUtils.unescapeLineBreaks;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
/**
@@ -61,11 +69,11 @@ public class FileIOUtilsTest {
File f = folder.newFile();
int count = writeStrings(added.iterator(), f, false);
- Assert.assertEquals(added.size(), count);
+ assertEquals(added.size(), count);
Set<String> retrieved = readStringsAsSet(new FileInputStream(f), false);
- Assert.assertEquals(added, retrieved);
+ assertEquals(added, retrieved);
}
@Test
@@ -73,10 +81,10 @@ public class FileIOUtilsTest {
Set<String> added = newHashSet(getLineBreakStrings());
File f = folder.newFile();
int count = writeStrings(added.iterator(), f, true);
- Assert.assertEquals(added.size(), count);
+ assertEquals(added.size(), count);
Set<String> retrieved = readStringsAsSet(new FileInputStream(f), true);
- Assert.assertEquals(added, retrieved);
+ assertEquals(added, retrieved);
}
@Test
@@ -88,38 +96,71 @@ public class FileIOUtilsTest {
added.add(getRandomTestString());
}
int count = writeStrings(added.iterator(), f, true);
- Assert.assertEquals(added.size(), count);
+ assertEquals(added.size(), count);
Set<String> retrieved = readStringsAsSet(new FileInputStream(f), true);
- Assert.assertEquals(added, retrieved);
+ assertEquals(added, retrieved);
}
@Test
public void compareWithLineBreaks() throws Exception {
- Comparator<String> lexCmp = new Comparator<String>() {
- @Override public int compare(String s1, String s2) {
- return s1.compareTo(s2);
- }
- };
- Comparator<String> cmp = FileIOUtils.lineBreakAwareComparator(lexCmp);
+ Comparator<String> cmp = lineBreakAwareComparator(lexComparator);
List<String> strs = getLineBreakStrings();
- Collections.sort(strs, lexCmp);
+ Collections.sort(strs);
// Escape line breaks and then compare with string sorted
List<String> escapedStrs = escape(getLineBreakStrings());
Collections.sort(escapedStrs, cmp);
- Assert.assertEquals(strs, unescape(escapedStrs));
+ assertEquals(strs, unescape(escapedStrs));
+ }
+
+ @Test
+ public void sortTest() throws IOException {
+ List<String> list = newArrayList("a", "z", "e", "b");
+ File f = folder.newFile();
+ writeStrings(list.iterator(), f, false);
+ sort(f);
+
+ BufferedReader reader =
+ new BufferedReader(new InputStreamReader(new FileInputStream(f), Charsets.UTF_8));
+ String line = null;
+ List<String> retrieved = newArrayList();
+ while ((line = reader.readLine()) != null) {
+ retrieved.add(line);
+ }
+ IOUtils.closeQuietly(reader);
+ Collections.sort(list);
+ assertArrayEquals(Arrays.toString(list.toArray()), list.toArray(), retrieved.toArray());
+ }
+
+ @Test
+ public void sortCustomComparatorTest() throws IOException {
+ List<String> list = getLineBreakStrings();
+ File f = folder.newFile();
+ writeStrings(list.iterator(), f, true);
+ sort(f, lineBreakAwareComparator(lexComparator));
+
+ BufferedReader reader =
+ new BufferedReader(new InputStreamReader(new FileInputStream(f), Charsets.UTF_8));
+ String line = null;
+ List<String> retrieved = newArrayList();
+ while ((line = reader.readLine()) != null) {
+ retrieved.add(unescapeLineBreaks(line));
+ }
+ IOUtils.closeQuietly(reader);
+ Collections.sort(list);
+ assertArrayEquals(Arrays.toString(list.toArray()), list.toArray(), retrieved.toArray());
}
private static List<String> getLineBreakStrings() {
- return Lists.newArrayList("ab\nc\r", "ab\\z", "a\\\\z\nc",
+ return newArrayList("ab\nc\r", "ab\\z", "a\\\\z\nc",
"/a", "/a/b\nc", "/a/b\rd", "/a/b\r\ne", "/a/c");
}
private static List<String> escape(List<String> list) {
- List<String> escaped = Lists.newArrayList();
+ List<String> escaped = newArrayList();
for (String s : list) {
escaped.add(escapeLineBreak(s));
}
@@ -127,7 +168,7 @@ public class FileIOUtilsTest {
}
private static List<String> unescape(List<String> list) {
- List<String> unescaped = Lists.newArrayList();
+ List<String> unescaped = newArrayList();
for (String s : list) {
unescaped.add(unescapeLineBreaks(s));
}
Modified: jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java?rev=1772323&r1=1772322&r2=1772323&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java (original)
+++ jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java Fri Dec 2 10:15:57 2016
@@ -18,7 +18,6 @@ package org.apache.jackrabbit.oak.plugin
import java.io.BufferedWriter;
import java.io.ByteArrayInputStream;
-import java.io.Closeable;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
@@ -46,11 +45,8 @@ import com.google.common.base.Function;
import com.google.common.base.Joiner;
import com.google.common.base.StandardSystemProperty;
import com.google.common.base.Stopwatch;
-import com.google.common.collect.AbstractIterator;
-import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
-import com.google.common.collect.PeekingIterator;
import com.google.common.io.Closeables;
import com.google.common.io.Files;
import com.google.common.util.concurrent.ListenableFutureTask;
@@ -59,6 +55,7 @@ import org.apache.commons.io.LineIterato
import org.apache.jackrabbit.core.data.DataRecord;
import org.apache.jackrabbit.core.data.DataStoreException;
import org.apache.jackrabbit.oak.commons.FileIOUtils;
+import org.apache.jackrabbit.oak.commons.FileIOUtils.FileLineDifferenceIterator;
import org.apache.jackrabbit.oak.commons.IOUtils;
import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils;
import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils.SharedStoreRecordType;
@@ -79,14 +76,22 @@ public class MarkSweepGarbageCollector i
public static final Logger LOG = LoggerFactory.getLogger(MarkSweepGarbageCollector.class);
- public static final String NEWLINE = StandardSystemProperty.LINE_SEPARATOR.value();
-
public static final String TEMP_DIR = StandardSystemProperty.JAVA_IO_TMPDIR.value();
public static final int DEFAULT_BATCH_COUNT = 2048;
public static final String DELIM = ",";
-
+
+ private static final Function<String, String> transformer = new Function<String, String>() {
+ @Nullable
+ @Override
+ public String apply(@Nullable String input) {
+ if (input != null) {
+ return input.split(DELIM)[0];
+ }
+ return "";
+ }};
+
/** The last modified time before current time of blobs to consider for garbage collection. */
private final long maxLastModifiedInterval;
@@ -296,7 +301,8 @@ public class MarkSweepGarbageCollector i
FileLineDifferenceIterator iter = new FileLineDifferenceIterator(
fs.getMarkedRefs(),
- fs.getAvailableRefs());
+ fs.getAvailableRefs(),
+ transformer);
calculateDifference(fs, iter);
LOG.debug("Ending difference phase of the garbage collector");
@@ -543,12 +549,12 @@ public class MarkSweepGarbageCollector i
"Blob garbage collection [{}]", count.get());
// sort the marked references with the first part of the key
GarbageCollectorFileState.sort(fs.getMarkedRefs(),
- new Comparator<String>() {
- @Override
- public int compare(String s1, String s2) {
- return s1.split(DELIM)[0].compareTo(s2.split(DELIM)[0]);
- }
- });
+ new Comparator<String>() {
+ @Override
+ public int compare(String s1, String s2) {
+ return s1.split(DELIM)[0].compareTo(s2.split(DELIM)[0]);
+ }
+ });
} finally {
IOUtils.closeQuietly(writer);
}
@@ -567,7 +573,6 @@ public class MarkSweepGarbageCollector i
long candidates = 0;
try {
- Stopwatch sw = Stopwatch.createStarted();
LOG.info("Starting blob consistency check");
// Find all blobs available in the blob store
@@ -586,7 +591,10 @@ public class MarkSweepGarbageCollector i
}
LOG.trace("Starting difference phase of the consistency check");
- FileLineDifferenceIterator iter = new FileLineDifferenceIterator(fs.getAvailableRefs(), fs.getMarkedRefs());
+ FileLineDifferenceIterator iter = new FileLineDifferenceIterator(
+ fs.getAvailableRefs(),
+ fs.getMarkedRefs(),
+ transformer);
candidates = calculateDifference(fs, iter);
LOG.trace("Ending difference phase of the consistency check");
@@ -648,81 +656,6 @@ public class MarkSweepGarbageCollector i
}
}
-
- /**
- * FileLineDifferenceIterator class which iterates over the difference of 2 files line by line.
- */
- static class FileLineDifferenceIterator extends AbstractIterator<String> implements Closeable {
- private final PeekingIterator<String> peekMarked;
- private final LineIterator marked;
- private final LineIterator all;
-
- public FileLineDifferenceIterator(File marked, File available) throws IOException {
- this(FileUtils.lineIterator(marked), FileUtils.lineIterator(available));
- }
-
- public FileLineDifferenceIterator(LineIterator marked, LineIterator available) throws IOException {
- this.marked = marked;
- this.peekMarked = Iterators.peekingIterator(marked);
- this.all = available;
- }
-
- @Override
- protected String computeNext() {
- String diff = computeNextDiff();
- if (diff == null) {
- close();
- return endOfData();
- }
- return diff;
- }
-
- @Override
- public void close() {
- LineIterator.closeQuietly(marked);
- LineIterator.closeQuietly(all);
- }
-
- private String getKey(String row) {
- return row.split(DELIM)[0];
- }
-
- private String computeNextDiff() {
- if (!all.hasNext()) {
- return null;
- }
-
- //Marked finish the rest of all are part of diff
- if (!peekMarked.hasNext()) {
- return all.next();
- }
-
- String diff = null;
- while (all.hasNext() && diff == null) {
- diff = all.next();
- while (peekMarked.hasNext()) {
- String marked = peekMarked.peek();
- int comparisonResult = getKey(diff).compareTo(getKey(marked));
- if (comparisonResult > 0) {
- //Extra entries in marked. Ignore them and move on
- peekMarked.next();
- } else if (comparisonResult == 0) {
- //Matching entry found in marked move past it. Not a
- //dif candidate
- peekMarked.next();
- diff = null;
- break;
- } else {
- //This entry is not found in marked entries
- //hence part of diff
- return diff;
- }
- }
- }
- return diff;
- }
- }
-
/**
* Provides a readable string for given timestamp
*/
Modified: jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/FileLineDifferenceIteratorTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/FileLineDifferenceIteratorTest.java?rev=1772323&r1=1772322&r2=1772323&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/FileLineDifferenceIteratorTest.java (original)
+++ jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/FileLineDifferenceIteratorTest.java Fri Dec 2 10:15:57 2016
@@ -28,17 +28,20 @@ import java.util.List;
import java.util.Random;
import java.util.TreeSet;
+import javax.annotation.Nullable;
+
+import com.google.common.base.Function;
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.base.StandardSystemProperty;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import org.apache.commons.io.LineIterator;
+import org.apache.jackrabbit.oak.commons.FileIOUtils.FileLineDifferenceIterator;
import org.junit.Test;
import static java.util.Arrays.asList;
import static org.apache.jackrabbit.oak.commons.sort.EscapeUtils.escapeLineBreak;
-import static org.apache.jackrabbit.oak.plugins.blob.MarkSweepGarbageCollector.FileLineDifferenceIterator;
import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertThat;
@@ -105,7 +108,7 @@ public class FileLineDifferenceIteratorT
public void testDiffLineBreakChars() throws IOException {
List<String> all = getLineBreakStrings();
List<String> marked = getLineBreakStrings();
- List<String> diff = remove(marked, 3, 2);
+ remove(marked, 3, 2);
// without escaping, the line breaks will be resolved
assertDiff(Joiner.on(",").join(marked), Joiner.on(",").join(all),
@@ -122,6 +125,18 @@ public class FileLineDifferenceIteratorT
assertDiff(Joiner.on(",").join(marked), Joiner.on(",").join(all), diff);
}
+ @Test
+ public void testDiffTransform() throws IOException {
+ assertTransformed("a:x,b:y", "a:1,b:2,c:3,e:4,h", asList("c:3", "e:4", "h"));
+ assertTransformed("a,b,d,e", "a,b,c", asList("c"));
+ assertTransformed("a:1,b:2,d:3,e:4,f:5", "a:z,b:y,c:x,f:w", asList("c:x"));
+ assertTransformed("a,b,d,e,f", "a,b,c,f,h", asList("c", "h"));
+ assertTransformed("3:1,7:6", "2:0,3:6,5:3,9:1", asList("2:0", "5:3", "9:1"));
+ assertTransformed("", "", Collections.<String> emptyList());
+ assertTransformed("", "a, b", asList("a", "b"));
+ assertTransformed("", "a:4, b:1", asList("a:4", "b:1"));
+ }
+
private static List<String> getLineBreakStrings() {
return Lists.newArrayList("ab\nc\r", "ab\\z", "a\\\\z\nc",
"/a", "/a/b\nc", "/a/b\rd", "/a/b\r\ne", "/a/c");
@@ -149,7 +164,7 @@ public class FileLineDifferenceIteratorT
Iterator<String> itr = createItr(all, marked);
assertThat("marked: " + marked + " all: " + all, ImmutableList.copyOf(itr), is(diff));
}
-
+
private static void assertDiff(String marked, String all, List<String> diff) throws IOException {
Iterator<String> itr = createItr(marked, all);
assertThat("marked: " + marked + " all: " + all, ImmutableList.copyOf(itr), is(diff));
@@ -165,4 +180,18 @@ public class FileLineDifferenceIteratorT
return new LineIterator(new StringReader(lines));
}
+ private static void assertTransformed(String marked, String all, List<String> diff) throws IOException {
+ Iterator<String> itr = new FileLineDifferenceIterator(lineItr(marked), lineItr(all),
+ new Function<String, String>() {
+ @Nullable @Override
+ public String apply(@Nullable String input) {
+ if (input != null) {
+ return input.split(":")[0];
+ }
+ return null;
+ }
+ });
+
+ assertThat("marked: " + marked + " all: " + all, ImmutableList.copyOf(itr), is(diff));
+ }
}
Modified: jackrabbit/oak/branches/1.4/oak-run/README.md
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-run/README.md?rev=1772323&r1=1772322&r2=1772323&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.4/oak-run/README.md (original)
+++ jackrabbit/oak/branches/1.4/oak-run/README.md Fri Dec 2 10:15:57 2016
@@ -26,7 +26,7 @@ The following runmodes are currently ava
* garbage : Identifies blob garbage on a DocumentMK repository
* tarmkdiff : Show changes between revisions on TarMk
* tarmkrecovery : Lists candidates for head journal entries
- * dumpdatastorerefs : Dump all the blob references used to a file
+ * datastorecheck : Consistency checker for data store
* resetclusterid : Resets the cluster id
* help : Print a list of available runmodes
@@ -965,15 +965,60 @@ The following options are available:
--version-v10 - Uses V10 version repository reading (see OAK-2527)
-Oak Dump DataStore References
------------------------------
+Oak DataStore Check
+-------------------
-Dumps all the DataStore/BlobStore references used. Use the following commmand
+Consistency checker for the DataStore.
+Also can be used to list all the blob references in the node store and all the blob ids available in the data store.
+Use the following command:
+
+ $ java -jar oak-run-*.jar datastorecheck [--id] [--ref] [--consistency] \
+ [--store <path>|<mongo_uri>] \
+ [--s3ds <s3ds_config>|--fds <fds_config>] \
+ [--dump <path>]
- $ java -jar oak-run-*.jar dumpdatastorerefs \
- { /path/to/oak/repository | mongodb://host:port/database } [/path/to/dump]
+The following options are available:
+
+ --id - List all the ids in the data store
+ --ref - List all the blob references in the node store
+ --consistency - Lists all the missing blobs by doind a consistency check
+ Atleast one of the above should be specified
+
+ --store - Path to the segment store of mongo uri (Required for --ref & --consistency option above)
+ --dump - Path where to dump the files (Optional). Otherwise, files will be dumped in the user tmp directory.
+ --s3ds - Path to the S3DataStore configuration file
+ --fds - Path to the FileDataStore configuration file ('path' property is mandatory)
+
+Note:
+For using S3DataStore the following additional jars have to be downloaded
+ - [commons-logging-1.1.3.jar](http://central.maven.org/maven2/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar)
+ - [httpcore-4.4.4.jar](http://central.maven.org/maven2/org/apache/httpcomponents/httpcore/4.4.4/httpcore-4.4.4.jar)
+ - [aws-java-sdk-1.10.76.jar](http://central.maven.org/maven2/com/amazonaws/aws-java-sdk/1.10.76/aws-java-sdk-1.10.76.jar)
+
+The command to be executed for S3DataStore
+
+ java -classpath oak-run-*.jar:httpcore-4.4.4.jar:aws-java-sdk-osgi-1.10.76.jar:commons-logging-1.1.3.jar \
+ org.apache.jackrabbit.oak.run.Main \
+ datastorecheck --id --ref --consistency \
+ --store <path>|<mongo_uri> \
+ --s3ds <s3ds_config> \
+ --dump <dump_path>
+
+The config files should be formatted according to the OSGi configuration admin specification
+
+ E.g.
+ cat > org.apache.jackrabbit.oak.plugins.S3DataStore.config << EOF
+ accessKey="XXXXXXXXX"
+ secretKey="YYYYYY"
+ s3Bucket="bucket1"
+ s3Region="region1"
+ EOF
+
+ cat > org.apache.jackrabbit.oak.plugins.FileDataStore.config << EOF
+ path="/data/datastore"
+ EOF
+
-This will create a dump file with name starting with 'marked-'.The dump path is optional and if not specified the file will be created in the user tmp directory.
Reset Cluster Id
---------------
Modified: jackrabbit/oak/branches/1.4/oak-run/pom.xml
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-run/pom.xml?rev=1772323&r1=1772322&r2=1772323&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.4/oak-run/pom.xml (original)
+++ jackrabbit/oak/branches/1.4/oak-run/pom.xml Fri Dec 2 10:15:57 2016
@@ -412,11 +412,24 @@
</dependency>
<dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-blob-cloud</artifactId>
+ <version>${project.version}</version>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.sling</groupId>
<artifactId>org.apache.sling.testing.osgi-mock</artifactId>
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>org.apache.felix.configadmin</artifactId>
+ <version>1.8.8</version>
+ </dependency>
+
<!-- Findbugs annotations -->
<dependency>
<groupId>com.google.code.findbugs</groupId>
@@ -429,7 +442,7 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
-
+
</dependencies>
<profiles>
Copied: jackrabbit/oak/branches/1.4/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java (from r1750887, jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java)
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java?p2=jackrabbit/oak/branches/1.4/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java&p1=jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java&r1=1750887&r2=1772323&rev=1772323&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java (original)
+++ jackrabbit/oak/branches/1.4/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java Fri Dec 2 10:15:57 2016
@@ -91,8 +91,6 @@ public class DataStoreCheckCommand imple
// Optional argument to specify the dump path
ArgumentAcceptingOptionSpec<String> dump = parser.accepts("dump", "Dump Path")
.withRequiredArg().ofType(String.class);
- OptionSpec
- segmentTar = parser.accepts("segment-tar", "Use oak-segment-tar instead of oak-segment");
OptionSpec<?> help = parser.acceptsAll(asList("h", "?", "help"),
"show help").forHelp();
@@ -133,8 +131,6 @@ public class DataStoreCheckCommand imple
closer.register(Utils.asCloseable(nodeStore));
blobStore = (GarbageCollectableBlobStore) nodeStore.getBlobStore();
marker = new DocumentBlobReferenceRetriever(nodeStore);
- } else if (options.has(segmentTar)) {
- marker = SegmentTarUtils.newBlobReferenceRetriever(source, closer);
} else {
FileStore fileStore = openFileStore(source);
closer.register(Utils.asCloseable(fileStore));
Modified: jackrabbit/oak/branches/1.4/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Mode.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Mode.java?rev=1772323&r1=1772322&r2=1772323&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.4/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Mode.java (original)
+++ jackrabbit/oak/branches/1.4/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Mode.java Fri Dec 2 10:15:57 2016
@@ -41,7 +41,7 @@ enum Mode {
GARBAGE("garbage", new GarbageCommand()),
TARMKDIFF("tarmkdiff", new FileStoreDiffCommand()),
TARMKRECOVERY("tarmkrecovery", new FileStoreRevisionRecoveryCommand()),
- DUMPDATASTOREREFS("dumpdatastorerefs", new DumpDataStoreReferencesCommand()),
+ DUMPDATASTOREREFS("dumpdatastorerefs", new DataStoreCheckCommand()),
RESETCLUSTERID("resetclusterid", new ResetClusterIdCommand()),
HELP("help", new HelpCommand());
Modified: jackrabbit/oak/branches/1.4/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java?rev=1772323&r1=1772322&r2=1772323&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.4/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java (original)
+++ jackrabbit/oak/branches/1.4/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java Fri Dec 2 10:15:57 2016
@@ -19,22 +19,39 @@ package org.apache.jackrabbit.oak.run;
import static java.util.Arrays.asList;
import static org.apache.jackrabbit.oak.plugins.segment.FileStoreHelper.openFileStore;
+import static org.apache.jackrabbit.oak.commons.PropertiesUtil.populate;
import java.io.Closeable;
+import java.io.FileInputStream;
import java.io.IOException;
+import java.util.Dictionary;
+import java.util.Enumeration;
import java.util.List;
+import java.util.Properties;
+import javax.annotation.Nullable;
+import javax.jcr.RepositoryException;
+
+import com.google.common.collect.Maps;
import com.google.common.io.Closer;
import com.mongodb.MongoClientURI;
import com.mongodb.MongoURI;
+import joptsimple.ArgumentAcceptingOptionSpec;
import joptsimple.OptionParser;
import joptsimple.OptionSet;
import joptsimple.OptionSpec;
+import org.apache.felix.cm.file.ConfigurationHandler;
+import org.apache.jackrabbit.core.data.DataStore;
+import org.apache.jackrabbit.core.data.DataStoreException;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.OakFileDataStore;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedS3DataStore;
import org.apache.jackrabbit.oak.plugins.document.DocumentMK;
import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore;
import org.apache.jackrabbit.oak.plugins.document.util.MongoConnection;
import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore;
import org.apache.jackrabbit.oak.plugins.segment.file.FileStore;
+import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
class Utils {
@@ -86,6 +103,44 @@ class Utils {
return SegmentNodeStore.newSegmentNodeStore(fs).create();
}
+ @Nullable
+ public static GarbageCollectableBlobStore bootstrapDataStore(String[] args, Closer closer)
+ throws IOException, RepositoryException {
+ OptionParser parser = new OptionParser();
+ parser.allowsUnrecognizedOptions();
+
+ ArgumentAcceptingOptionSpec<String> s3dsConfig =
+ parser.accepts("s3ds", "S3DataStore config").withRequiredArg().ofType(String.class);
+ ArgumentAcceptingOptionSpec<String> fdsConfig =
+ parser.accepts("fds", "FileDataStore config").withRequiredArg().ofType(String.class);
+
+ OptionSet options = parser.parse(args);
+
+ if (!options.has(s3dsConfig) && !options.has(fdsConfig)) {
+ return null;
+ }
+
+ DataStore delegate;
+ if (options.has(s3dsConfig)) {
+ SharedS3DataStore s3ds = new SharedS3DataStore();
+ String cfgPath = s3dsConfig.value(options);
+ Properties props = loadAndTransformProps(cfgPath);
+ s3ds.setProperties(props);
+ s3ds.init(null);
+ delegate = s3ds;
+ } else {
+ delegate = new OakFileDataStore();
+ String cfgPath = fdsConfig.value(options);
+ Properties props = loadAndTransformProps(cfgPath);
+ populate(delegate, Maps.fromProperties(props), true);
+ delegate.init(null);
+ }
+ DataStoreBlobStore blobStore = new DataStoreBlobStore(delegate);
+ closer.register(Utils.asCloseable(blobStore));
+
+ return blobStore;
+ }
+
static Closeable asCloseable(final FileStore fs) {
return new Closeable() {
@@ -115,4 +170,30 @@ class Utils {
}
};
}
+
+ static Closeable asCloseable(final DataStoreBlobStore blobStore) {
+ return new Closeable() {
+
+ @Override
+ public void close() throws IOException {
+ try {
+ blobStore.close();
+ } catch (DataStoreException e) {
+ throw new IOException(e);
+ }
+ }
+ };
+ }
+
+
+ private static Properties loadAndTransformProps(String cfgPath) throws IOException {
+ Dictionary dict = ConfigurationHandler.read(new FileInputStream(cfgPath));
+ Properties props = new Properties();
+ Enumeration keys = dict.keys();
+ while (keys.hasMoreElements()) {
+ String key = (String) keys.nextElement();
+ props.put(key, dict.get(key));
+ }
+ return props;
+ }
}
Copied: jackrabbit/oak/branches/1.4/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java (from r1750887, jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java)
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java?p2=jackrabbit/oak/branches/1.4/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java&p1=jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java&r1=1750887&r2=1772323&rev=1772323&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java (original)
+++ jackrabbit/oak/branches/1.4/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java Fri Dec 2 10:15:57 2016
@@ -40,6 +40,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.common.io.Files;
import org.apache.commons.io.filefilter.FileFilterUtils;
+import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.jackrabbit.oak.commons.FileIOUtils;
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
import org.apache.jackrabbit.oak.plugins.blob.datastore.OakFileDataStore;
@@ -59,6 +60,7 @@ import org.junit.rules.TemporaryFolder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static com.google.common.base.Charsets.UTF_8;
import static org.junit.Assert.assertEquals;
/**
@@ -88,11 +90,11 @@ public class DataStoreCheckTest {
File storeFile = temporaryFolder.newFolder();
storePath = storeFile.getAbsolutePath();
- FileStore.Builder builder = FileStore.builder(storeFile)
+ FileStore.Builder builder = FileStore.newFileStore(storeFile)
.withBlobStore(blobStore).withMaxFileSize(256)
.withCacheSize(64).withMemoryMapping(false);
- FileStore fileStore = builder.build();
- NodeStore store = SegmentNodeStore.builder(fileStore).build();
+ FileStore fileStore = builder.create();
+ NodeStore store = new SegmentNodeStore(fileStore);
/* Create nodes with blobs stored in DS*/
NodeBuilder a = store.getRoot().builder();
@@ -112,8 +114,8 @@ public class DataStoreCheckTest {
log.info("Created blobs : {}", blobsAdded);
File cfgFile = temporaryFolder.newFile();
- BufferedWriter writer = Files.newWriter(cfgFile, Charsets.UTF_8);
- FileIOUtils.writeAsLine(writer, "path=\"" + dsPath + "\"",false);
+ BufferedWriter writer = Files.newWriter(cfgFile, UTF_8);
+ FileIOUtils.writeAsLine(writer, "path=\"" + StringEscapeUtils.escapeJava(dsPath) + "\"",false);
writer.close();
cfgFilePath = cfgFile.getAbsolutePath();
@@ -123,7 +125,7 @@ public class DataStoreCheckTest {
@After
public void tearDown() {
- System.setErr(new PrintStream(new FileOutputStream(FileDescriptor.out)));
+ System.setErr(new PrintStream(new FileOutputStream(FileDescriptor.err)));
}
@Test
@@ -170,8 +172,8 @@ public class DataStoreCheckTest {
List<String> argsList = Lists
.newArrayList("--fds", cfgFilePath, "--store", storePath,
"--dump", dump.getAbsolutePath());
- testIncorrectParams(argsList, "Missing "
- + "required option(s) ['id', 'ref', 'consistency']");
+ log.info("Running testMissinOpParams: {}", argsList);
+ testIncorrectParams(argsList, "Missing required option(s) ['id', 'ref', 'consistency']");
}
@Test
@@ -200,14 +202,17 @@ public class DataStoreCheckTest {
public static void testIncorrectParams(List<String> argList, String assertMsg) throws Exception {
ByteArrayOutputStream buffer = new ByteArrayOutputStream();
- System.setErr(new PrintStream(buffer));
+ System.setErr(new PrintStream(buffer, true, UTF_8.toString()));
DataStoreCheckCommand checkCommand = new DataStoreCheckCommand();
checkCommand.execute(argList.toArray(new String[0]));
- String message = buffer.toString(Charsets.UTF_8.toString());
+ String message = buffer.toString(UTF_8.toString());
+ log.info("Assert message: {}", assertMsg);
+ log.info("Message logged in System.err: {}", message);
+
Assert.assertTrue(message.contains(assertMsg));
- System.setErr(new PrintStream(new FileOutputStream(FileDescriptor.out)));
+ System.setErr(new PrintStream(new FileOutputStream(FileDescriptor.err)));
}
private static void assertFileEquals(File dump, String prefix, Set<String> blobsAdded)