You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by am...@apache.org on 2017/12/01 05:48:57 UTC
svn commit: r1816797 - in /jackrabbit/oak/trunk:
oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/
oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/
oak-run/src/main/java/org/apache/jackrabbit/oak/run/
oak-run/src/test/java/org/ap...
Author: amitj
Date: Fri Dec 1 05:48:57 2017
New Revision: 1816797
URL: http://svn.apache.org/viewvc?rev=1816797&view=rev
Log:
OAK-6489: Datastorecheck command should output the path of missing datastore
Introduced a --verbose option which outputs backend friendly blob ids in the file created. This information can be combined with the datastore path (in a script) to construct the full path of the id easily.
Modified:
jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java
jackrabbit/oak/trunk/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileIOUtilsTest.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java
Modified: jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java?rev=1816797&r1=1816796&r2=1816797&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java (original)
+++ jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java Fri Dec 1 05:48:57 2017
@@ -30,6 +30,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Set;
+import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import com.google.common.base.Charsets;
@@ -75,6 +76,12 @@ public final class FileIOUtils {
}
};
+ public final static Function<String, String> passThruTransformer = new Function<String, String>() {
+ @Nullable @Override public String apply(@Nullable String input) {
+ return input;
+ }
+ };
+
/**
* Sorts the given file externally using the {@link #lexComparator} and removes duplicates.
*
@@ -223,13 +230,31 @@ public final class FileIOUtils {
*/
public static int writeStrings(Iterator<String> iterator, File f, boolean escape,
@Nullable Logger logger, @Nullable String message) throws IOException {
+ return writeStrings(iterator, f, escape, passThruTransformer, logger, message);
+ }
+
+ /**
+ * Writes string from the given iterator to the given file and optionally
+ * escape the written strings for line breaks.
+ *
+ * @param iterator the source of the strings
+ * @param f file to write to
+ * @param escape escape whether to escape for line breaks
+ * @param transformer any transformation on the input
+ * @param logger logger to log progress
+ * @param message message to log
+ * @return
+ * @throws IOException
+ */
+ public static int writeStrings(Iterator<String> iterator, File f, boolean escape,
+ @Nonnull Function<String, String> transformer, @Nullable Logger logger, @Nullable String message) throws IOException {
BufferedWriter writer = newWriter(f, UTF_8);
boolean threw = true;
int count = 0;
try {
while (iterator.hasNext()) {
- writeAsLine(writer, iterator.next(), escape);
+ writeAsLine(writer, transformer.apply(iterator.next()), escape);
count++;
if (logger != null) {
if (count % 1000 == 0) {
Modified: jackrabbit/oak/trunk/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileIOUtilsTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileIOUtilsTest.java?rev=1816797&r1=1816796&r2=1816797&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileIOUtilsTest.java (original)
+++ jackrabbit/oak/trunk/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileIOUtilsTest.java Fri Dec 1 05:48:57 2017
@@ -40,6 +40,7 @@ import java.util.Set;
import javax.annotation.Nullable;
import com.google.common.base.Function;
+import com.google.common.base.Splitter;
import com.google.common.collect.Iterators;
import com.google.common.collect.Sets;
import com.google.common.primitives.Longs;
@@ -91,6 +92,23 @@ public class FileIOUtilsTest {
}
@Test
+ public void writeCustomReadOrgStrings() throws Exception {
+ Set<String> added = newHashSet("a-", "z-", "e-", "b-");
+ Set<String> actual = newHashSet("a", "z", "e", "b");
+
+ File f = folder.newFile();
+ int count = writeStrings(added.iterator(), f, false, new Function<String, String>() {
+ @Nullable @Override public String apply(@Nullable String input) {
+ return Splitter.on("-").trimResults().omitEmptyStrings().splitToList(input).get(0);
+ }
+ }, null, null);
+ assertEquals(added.size(), count);
+
+ Set<String> retrieved = readStringsAsSet(new FileInputStream(f), false);
+ assertEquals(actual, retrieved);
+ }
+
+ @Test
public void writeReadStringsWithLineBreaks() throws IOException {
Set<String> added = newHashSet(getLineBreakStrings());
File f = assertWrite(added.iterator(), true, added.size());
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java?rev=1816797&r1=1816796&r2=1816797&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java Fri Dec 1 05:48:57 2017
@@ -17,6 +17,7 @@
package org.apache.jackrabbit.oak.run;
import static com.google.common.base.StandardSystemProperty.JAVA_IO_TMPDIR;
+import static com.google.common.base.StandardSystemProperty.FILE_SEPARATOR;
import static com.google.common.base.Stopwatch.createStarted;
import static com.google.common.io.Closeables.close;
import static java.io.File.createTempFile;
@@ -35,14 +36,17 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Comparator;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import javax.annotation.Nullable;
+import com.google.common.base.Splitter;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
+import org.apache.commons.io.LineIterator;
import org.apache.commons.io.filefilter.FileFilterUtils;
import org.apache.jackrabbit.oak.commons.FileIOUtils;
import org.apache.jackrabbit.oak.commons.FileIOUtils.FileLineDifferenceIterator;
@@ -77,6 +81,11 @@ import joptsimple.OptionSpecBuilder;
*/
public class DataStoreCheckCommand implements Command {
private static final String DELIM = ",";
+ private static final String FDS = "--fds";
+ private static final String S3DS = "--s3ds";
+ private static final String AZUREDS = "--azureblobds";
+ private static final String DASH = "-";
+ private static final String HASH = "#";
@Override
public void execute(String... args) throws Exception {
@@ -86,7 +95,7 @@ public class DataStoreCheckCommand imple
String helpStr =
"datastorecheck [--id] [--ref] [--consistency] [--store <path>|<mongo_uri>] "
+ "[--s3ds <s3ds_config>|--fds <fds_config>|--azureblobds <azureblobds_config>]"
- + " [--dump <path>] [--repoHome <repo_home>] [--track]";
+ + " [--dump <path>] [--repoHome <repo_home>] [--track] [--verbose]";
Closer closer = Closer.create();
try {
@@ -109,6 +118,8 @@ public class DataStoreCheckCommand imple
ArgumentAcceptingOptionSpec<String> repoHome = parser.accepts("repoHome", "Local repository home folder")
.requiredIf(trackOverride, consistencyOp).withRequiredArg().ofType(String.class);
+ // Optional argument to specify tracking
+ OptionSpecBuilder verbose = parser.accepts("verbose", "Output backend formatted ids/paths");
OptionSpec<?> help = parser.acceptsAll(asList("h", "?", "help"),
"show help").forHelp();
@@ -159,8 +170,10 @@ public class DataStoreCheckCommand imple
}
// Initialize S3/FileDataStore if configured
+ String dsType = "";
GarbageCollectableBlobStore dataStore = Utils.bootstrapDataStore(args, closer);
if (dataStore != null) {
+ dsType = getDSType(args);
blobStore = dataStore;
}
@@ -175,8 +188,15 @@ public class DataStoreCheckCommand imple
closer.register(register);
if (options.has(idOp) || options.has(consistencyOp)) {
+ File idTemp = createTempFile("ids", null);
+ closer.register(new Closeable() {
+ @Override public void close() throws IOException {
+ forceDelete(idTemp);
+ }
+ });
+
File dumpFile = register.createFile(idOp, dumpPath);
- retrieveBlobIds(blobStore, dumpFile);
+ retrieveBlobIds(blobStore, idTemp);
// If track path and track override specified copy the file to the location
if (options.has(repoHome) && options.has(trackOverride)) {
@@ -184,18 +204,24 @@ public class DataStoreCheckCommand imple
File trackingFileParent = new File(FilenameUtils.concat(trackPath, "blobids"));
File trackingFile = new File(trackingFileParent,
"blob-" + String.valueOf(System.currentTimeMillis()) + ".gen");
- FileUtils.copyFile(dumpFile, trackingFile);
+ FileUtils.copyFile(idTemp, trackingFile);
+ }
+
+ if (options.has(verbose)) {
+ verboseIds(closer, dsType, idTemp, dumpFile);
+ } else {
+ FileUtils.copyFile(idTemp, dumpFile);
}
}
if (options.has(refOp) || options.has(consistencyOp)) {
retrieveBlobReferences(blobStore, marker,
- register.createFile(refOp, dumpPath));
+ register.createFile(refOp, dumpPath), dsType, options.has(verbose));
}
if (options.has(consistencyOp)) {
checkConsistency(register.get(idOp), register.get(refOp),
- register.createFile(consistencyOp, dumpPath), options.valueOf(repoHome));
+ register.createFile(consistencyOp, dumpPath), options.valueOf(repoHome), dsType);
}
} catch (Throwable t) {
t.printStackTrace();
@@ -204,6 +230,57 @@ public class DataStoreCheckCommand imple
}
}
+ private static void verboseIds(Closer closer, final String dsType, File readFile, File writeFile) throws IOException {
+ LineIterator idIterator = FileUtils.lineIterator(readFile);
+ // Create a temp file to write real ids and register with closer
+ File longIdTemp = createTempFile("longids", null);
+ closer.register(new Closeable() {
+ @Override public void close() throws IOException {
+ forceDelete(longIdTemp);
+ }
+ });
+
+ // Read and write the converted ids
+ FileIOUtils.writeStrings(idIterator, longIdTemp, true,
+ new Function<String, String>() {
+ @Nullable @Override public String apply(@Nullable String input) {
+ return encodeId(input, dsType);
+ }
+ }, null, null);
+ FileUtils.copyFile(longIdTemp, writeFile);
+ }
+
+ private static String getDSType(String[] args) {
+ List<String> argList = Arrays.asList(args);
+ if (argList.contains(S3DS)) {
+ return S3DS;
+ } else if (argList.contains(FDS)) {
+ return FDS;
+ } else if (argList.contains(AZUREDS)) {
+ return AZUREDS;
+ }
+ return "";
+ }
+
+ static String encodeId(String id, String dsType) {
+ List<String> idLengthSepList = Splitter.on(HASH).trimResults().omitEmptyStrings().splitToList(id);
+ String blobId = idLengthSepList.get(0);
+
+ if (dsType.equals(FDS)) {
+ return (blobId.substring(0, 2) + FILE_SEPARATOR.value() + blobId.substring(2, 4) + FILE_SEPARATOR.value() + blobId
+ .substring(4, 6) + FILE_SEPARATOR.value() + blobId);
+ } else if (dsType.equals(S3DS) || dsType.equals(AZUREDS)) {
+ return (blobId.substring(0, 4) + DASH + blobId.substring(4));
+ }
+ return id;
+ }
+
+ private static String decodeId(String id) {
+ List<String> list = Splitter.on(FILE_SEPARATOR.value()).trimResults().omitEmptyStrings().splitToList(id);
+ String pathStrippedId = list.get(list.size() -1);
+ return Joiner.on("").join(Splitter.on(DASH).omitEmptyStrings().trimResults().splitToList(pathStrippedId));
+ }
+
static class FileRegister implements Closeable {
Map<OptionSpec, File> opFiles = Maps.newHashMap();
String suffix = String.valueOf(System.currentTimeMillis());
@@ -240,7 +317,8 @@ public class DataStoreCheckCommand imple
}
}
- private static void checkConsistency(File ids, File refs, File missing, String trackRoot) throws IOException {
+ private static void checkConsistency(File ids, File refs, File missing, String trackRoot, String dsType)
+ throws IOException {
System.out.println("Starting consistency check");
Stopwatch watch = createStarted();
@@ -272,7 +350,7 @@ public class DataStoreCheckCommand imple
FileLineDifferenceIterator filteringIter = new FileLineDifferenceIterator(delFile, candTemp, new Function<String, String>() {
@Nullable @Override public String apply(@Nullable String input) {
if (input != null) {
- return input.split(DELIM)[0];
+ return encodeId(decodeId(input.split(DELIM)[0]), dsType);
}
return "";
}
@@ -294,8 +372,8 @@ public class DataStoreCheckCommand imple
System.out.println("Finished in " + watch.elapsed(TimeUnit.SECONDS) + " seconds");
}
- private static void retrieveBlobReferences(GarbageCollectableBlobStore blobStore,
- BlobReferenceRetriever marker, File marked) throws IOException {
+ private static void retrieveBlobReferences(GarbageCollectableBlobStore blobStore, BlobReferenceRetriever marker,
+ File marked, String dsType, boolean isVerbose) throws IOException {
final BufferedWriter writer = Files.newWriter(marked, Charsets.UTF_8);
final AtomicInteger count = new AtomicInteger();
boolean threw = true;
@@ -314,9 +392,13 @@ public class DataStoreCheckCommand imple
Iterator<String> idIter = finalBlobStore.resolveChunks(blobId);
while (idIter.hasNext()) {
- String id = delimJoiner.join(idIter.next(), nodeId);
+ String id = idIter.next();
+ if (isVerbose) {
+ id = encodeId(id, dsType);
+ }
+ String combinedId = delimJoiner.join(id, nodeId);
count.getAndIncrement();
- writeAsLine(writer, id, true);
+ writeAsLine(writer, combinedId, true);
}
} catch (Exception e) {
throw new RuntimeException("Error in retrieving references", e);
Modified: jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java?rev=1816797&r1=1816796&r2=1816797&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java Fri Dec 1 05:48:57 2017
@@ -38,6 +38,9 @@ import java.util.Properties;
import java.util.Random;
import java.util.Set;
+import javax.annotation.Nullable;
+
+import com.google.common.base.Function;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import com.google.common.collect.Iterators;
@@ -79,7 +82,11 @@ public class DataStoreCheckTest {
private static final Logger log = LoggerFactory.getLogger(DataStoreCheckTest.class);
@Rule
- public final TemporaryFolder temporaryFolder = new TemporaryFolder(new File("target"));
+ public final TemporaryFolder temporaryFolder = new TemporaryFolder(new File("target")) {
+ @Override
+ public void delete() {}
+
+ };
private String storePath;
@@ -205,6 +212,25 @@ public class DataStoreCheckTest {
}
@Test
+ public void testConsistencyVerbose() throws Exception {
+ File dump = temporaryFolder.newFolder();
+ File repoHome = temporaryFolder.newFolder();
+
+ Random rand = new Random();
+ String deletedBlobId = Iterables.get(blobsAdded, rand.nextInt(blobsAdded.size()));
+ blobsAdded.remove(deletedBlobId);
+ long count = setupDataStore.countDeleteChunks(ImmutableList.of(deletedBlobId), 0);
+ assertEquals(1, count);
+ setupDataStore.close();
+
+ testAllParamsVerbose(dump, repoHome);
+
+ assertFileEquals(dump, "[id]", encodedIds(blobsAdded, dsOption));
+ assertFileEquals(dump, "[ref]", encodedIds(Sets.union(blobsAdded, Sets.newHashSet(deletedBlobId)), dsOption));
+ assertFileEquals(dump, "[consistency]", encodedIds(Sets.newHashSet(deletedBlobId), dsOption));
+ }
+
+ @Test
public void testConsistencyWithDeleteTracker() throws Exception {
File dump = temporaryFolder.newFolder();
File repoHome = temporaryFolder.newFolder();
@@ -235,6 +261,38 @@ public class DataStoreCheckTest {
assertFileEquals(dump, "[consistency]", Sets.newHashSet(deletedBlobId));
}
+ @Test
+ public void testConsistencyVerboseWithDeleteTracker() throws Exception {
+ File dump = temporaryFolder.newFolder();
+ File repoHome = temporaryFolder.newFolder();
+
+ File trackerFolder = new File(repoHome, "blobids");
+ FileUtils.forceMkdir(trackerFolder);
+
+ File delTracker = new File(trackerFolder, "activedeletions.del");
+ Random rand = new Random();
+ String deletedBlobId = Iterables.get(blobsAdded, rand.nextInt(blobsAdded.size()));
+ blobsAdded.remove(deletedBlobId);
+ long count = setupDataStore.countDeleteChunks(ImmutableList.of(deletedBlobId), 0);
+
+ String activeDeletedBlobId = Iterables.get(blobsAdded, rand.nextInt(blobsAdded.size()));
+ blobsAdded.remove(activeDeletedBlobId);
+ count += setupDataStore.countDeleteChunks(ImmutableList.of(activeDeletedBlobId), 0);
+ assertEquals(2, count);
+
+ // artificially put the deleted id in the tracked .del file
+ FileIOUtils.writeStrings(Iterators.singletonIterator(activeDeletedBlobId), delTracker, false);
+
+ setupDataStore.close();
+
+ testAllParamsVerbose(dump, repoHome);
+
+ assertFileEquals(dump, "[id]", encodedIds(blobsAdded, dsOption));
+ assertFileEquals(dump, "[ref]",
+ encodedIds(Sets.union(blobsAdded, Sets.newHashSet(deletedBlobId, activeDeletedBlobId)), dsOption));
+ assertFileEquals(dump, "[consistency]", encodedIds(Sets.newHashSet(deletedBlobId), dsOption));
+ }
+
private void testAllParams(File dump, File repoHome) throws Exception {
DataStoreCheckCommand checkCommand = new DataStoreCheckCommand();
List<String> argsList = Lists
@@ -244,6 +302,15 @@ public class DataStoreCheckTest {
checkCommand.execute(argsList.toArray(new String[0]));
}
+ private void testAllParamsVerbose(File dump, File repoHome) throws Exception {
+ DataStoreCheckCommand checkCommand = new DataStoreCheckCommand();
+ List<String> argsList = Lists
+ .newArrayList("--id", "--ref", "--consistency", "--" + dsOption, cfgFilePath, "--store", storePath,
+ "--dump", dump.getAbsolutePath(), "--repoHome", repoHome.getAbsolutePath(), "--verbose");
+
+ checkCommand.execute(argsList.toArray(new String[0]));
+ }
+
@Test
public void testMissingOpParams() throws Exception {
setupDataStore.close();
@@ -352,4 +419,12 @@ public class DataStoreCheckTest {
ConfigurationHandler.write(fos, props);
return cfgFile.getAbsolutePath();
}
+
+ private static Set<String> encodedIds(Set<String> ids, String dsOption) {
+ return Sets.newHashSet(Iterators.transform(ids.iterator(), new Function<String, String>() {
+ @Nullable @Override public String apply(@Nullable String input) {
+ return DataStoreCheckCommand.encodeId(input, "--"+dsOption);
+ }
+ }));
+ }
}