You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ng...@apache.org on 2020/02/20 15:28:04 UTC
svn commit: r1874259 - in /jackrabbit/oak/trunk/oak-run: ./
src/main/java/org/apache/jackrabbit/oak/run/
src/test/java/org/apache/jackrabbit/oak/run/
Author: ngupta
Date: Thu Feb 20 15:28:04 2020
New Revision: 1874259
URL: http://svn.apache.org/viewvc?rev=1874259&view=rev
Log:
OAK-8902 | Selective listing of blobs under specific paths | Adding implementation to DataStoreCommand
Modified:
jackrabbit/oak/trunk/oak-run/README.md
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreOptions.java
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java
Modified: jackrabbit/oak/trunk/oak-run/README.md
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/README.md?rev=1874259&r1=1874258&r2=1874259&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/README.md (original)
+++ jackrabbit/oak/trunk/oak-run/README.md Thu Feb 20 15:28:04 2020
@@ -555,6 +555,7 @@ Maintenance commands for the DataStore:
[--work-dir <temporary_path>] \
[--max-age <seconds>] \
[--verbose] \
+ [--verboseRootPath]
[<store_path>|<mongo_uri>]
[--metrics] [--export-metrics]
@@ -576,6 +577,11 @@ The following options are available:
--verbose - Outputs backend friendly blobids and also adds the node path (for SegmentNodeStore) from where referred.
This options would typically be a slower option since, it requires the whole repo traversal.
Adds the sub-directories created in FDS to the id path and the changes done to the id for S3/Azure when stored in the respective container.
+ --verboseRootPath - Paths under which backend friendly blobids are required (Optional). If not specified, then --verbose uses "/" as the default path. For example,
+ to list all blobids under /oak:index and /content/oak:index, use --verboseRootPath /oak:index,/content/oak:index (If providing more than one arguments to this option,
+ use comma as a delimiter).
+ This option is NOT available for the collect-garbage operation. If specified with collect-garbage, the command execution will throw
+ an exception.
<store_path|mongo_uri> - Path to the tar segment store or the segment azure uri as specified in
http://jackrabbit.apache.org/oak/docs/nodestore/segment/overview.html#remote-segment-stores
or if Mongo NodeStore then the mongo uri.
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java?rev=1874259&r1=1874258&r2=1874259&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java Thu Feb 20 15:28:04 2020
@@ -89,6 +89,10 @@ import org.jetbrains.annotations.Nullabl
/**
* Command to check data store consistency and also optionally retrieve ids
* and references.
+ *
+ * NOTE - OAK-7671 plans on deprecating this command to delegate internally to use
+ * @see org.apache.jackrabbit.oak.run.DataStoreCommand instead. So
+ * any new support around Datastore should be added to @see org.apache.jackrabbit.oak.run.DataStoreCommand
*/
public class DataStoreCheckCommand implements Command {
private static final String DELIM = ",";
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java?rev=1874259&r1=1874258&r2=1874259&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java Thu Feb 20 15:28:04 2020
@@ -19,7 +19,9 @@ package org.apache.jackrabbit.oak.run;
import java.io.File;
import java.io.IOException;
import java.lang.management.ManagementFactory;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
@@ -183,11 +185,13 @@ public class DataStoreCommand implements
Options opts, Closer closer) throws IOException {
BlobReferenceRetriever retriever;
- if (opts.getCommonOpts().isDocument()) {
+ if (opts.getCommonOpts().isDocument() && !dataStoreOpts.hasVerboseRootPaths()) {
retriever = new DocumentBlobReferenceRetriever((DocumentNodeStore) fixture.getStore());
} else {
if (dataStoreOpts.isVerbose()) {
- retriever = new NodeTraverserReferenceRetriever(fixture.getStore());
+ List<String> rootPathList = dataStoreOpts.getVerboseRootPaths();
+ retriever = new NodeTraverserReferenceRetriever(fixture.getStore(),
+ (String[]) rootPathList.toArray(new String[rootPathList.size()]));
} else {
ReadOnlyFileStore fileStore = getService(fixture.getWhiteboard(), ReadOnlyFileStore.class);
retriever = new SegmentBlobReferenceRetriever(fileStore);
@@ -235,9 +239,11 @@ public class DataStoreCommand implements
*/
static class NodeTraverserReferenceRetriever implements BlobReferenceRetriever {
private final NodeStore nodeStore;
+ private final String[] paths;
- public NodeTraverserReferenceRetriever(NodeStore nodeStore) {
+ public NodeTraverserReferenceRetriever(NodeStore nodeStore, String ... paths) {
this.nodeStore = nodeStore;
+ this.paths = paths;
}
private void binaryProperties(NodeState state, String path, ReferenceCollector collector) {
@@ -269,7 +275,20 @@ public class DataStoreCommand implements
@Override public void collectReferences(ReferenceCollector collector) throws IOException {
log.info("Starting dump of blob references by traversing");
- traverseChildren(nodeStore.getRoot(), "/", collector);
+ if (paths.length == 0) {
+ traverseChildren(nodeStore.getRoot(), "/", collector);
+ } else {
+ for (String path: paths) {
+ Iterable<String> nodeList = PathUtils.elements(path);
+ NodeState state = nodeStore.getRoot();
+ for (String node: nodeList) {
+ state = state.getChildNode(node);
+ }
+ traverseChildren(state, path, collector);
+ }
+ }
+
+
}
}
@@ -289,15 +308,18 @@ public class DataStoreCommand implements
private final BlobStoreOptions optionBean;
private final BlobStoreOptions.Type blobStoreType;
private final File outDir;
- private final File outFile;
+ private final List<File> outFileList = new ArrayList<File>();
public VerboseIdLogger(Options options) {
this.optionBean = options.getOptionBean(BlobStoreOptions.class);
this.blobStoreType = optionBean.getBlobStoreType();
outDir = options.getOptionBean(DataStoreOptions.class).getOutDir();
- outFile = filterFiles(outDir, "gccand-");
- if (outFile == null) {
+ outFileList.add(filterFiles(outDir, "marked-"));
+ outFileList.add(filterFiles(outDir, "gccand-"));
+ outFileList.removeAll(Collections.singleton(null));
+
+ if (outFileList.size() == 0) {
throw new IllegalArgumentException("No candidate file found");
}
}
@@ -337,13 +359,16 @@ public class DataStoreCommand implements
}
public void log() throws IOException {
- File tempFile = new File(outDir, outFile.getName() + "-temp");
- FileUtils.moveFile(outFile, tempFile);
- try (BurnOnCloseFileIterator<String> iterator =
+
+ for (File outFile : outFileList) {
+ File tempFile = new File(outDir, outFile.getName() + "-temp");
+ FileUtils.moveFile(outFile, tempFile);
+ try (BurnOnCloseFileIterator<String> iterator =
new BurnOnCloseFileIterator<String>(FileUtils.lineIterator(tempFile, UTF_8.toString()), tempFile,
(Function<String, String>) input -> encodeId(input, blobStoreType))) {
FileIOUtils.writeStrings(iterator, outFile, true, log, "Transformed to verbose ids - ");
}
+ }
}
}
}
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreOptions.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreOptions.java?rev=1874259&r1=1874258&r2=1874259&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreOptions.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreOptions.java Thu Feb 20 15:28:04 2020
@@ -22,6 +22,7 @@ package org.apache.jackrabbit.oak.run;
import java.io.File;
import java.io.IOException;
import java.util.HashSet;
+import java.util.List;
import java.util.Set;
import com.google.common.collect.ImmutableSet;
@@ -47,8 +48,10 @@ public class DataStoreOptions implements
private final Set<String> operationNames;
private final OptionSpec<Long> blobGcMaxAgeInSecs;
private final OptionSpec<Void> verbose;
+ private final OptionSpec<String> verboseRootPath;
private final OptionSpec<Boolean> resetLoggingConfig;
private OptionSpec<String> exportMetrics;
+ private static final String DELIM = ",";
public DataStoreOptions(OptionParser parser) {
collectGarbage = parser.accepts("collect-garbage",
@@ -76,6 +79,11 @@ public class DataStoreOptions implements
verbose =
parser.accepts("verbose", "Option to get all the paths and implementation specific blob ids");
+ // Option NOT available for garbage collection operation - we throw an exception if both --collect-garbage and
+ // --verboseRootPath are provided in the command.
+ verboseRootPath = parser.accepts("verboseRootPath",
+ "Root path to output backend formatted ids/paths").availableUnless(collectGarbage).availableIf(verbose).withRequiredArg().withValuesSeparatedBy(DELIM).ofType(String.class);;
+
resetLoggingConfig =
parser.accepts("reset-log-config", "Reset logging config for testing purposes only").withOptionalArg()
.ofType(Boolean.class).defaultsTo(Boolean.TRUE);
@@ -159,6 +167,10 @@ public class DataStoreOptions implements
return options.has(verbose);
}
+ public boolean hasVerboseRootPaths() {
+ return options.has(verboseRootPath);
+ }
+
public boolean isResetLoggingConfig() {
return resetLoggingConfig.value(options);
}
@@ -178,4 +190,9 @@ public class DataStoreOptions implements
public String exportMetricsArgs() {
return exportMetrics.value(options);
}
+
+ public List<String> getVerboseRootPaths() {
+ return options.valuesOf(verboseRootPath);
+ }
+
}
Modified: jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java?rev=1874259&r1=1874258&r2=1874259&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java Thu Feb 20 15:28:04 2020
@@ -44,6 +44,7 @@ import com.google.common.collect.Iterato
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
+import joptsimple.OptionException;
import org.apache.commons.io.FileUtils;
import org.apache.felix.cm.file.ConfigurationHandler;
import org.apache.jackrabbit.core.data.DataStore;
@@ -106,6 +107,8 @@ import static org.apache.jackrabbit.oak.
import static org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.defaultGCOptions;
import static org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreBuilder;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assume.assumeFalse;
/**
@@ -323,6 +326,71 @@ public class DataStoreCommandTest {
}
@Test
+ public void testConsistencyVerboseRootPathNoMissing() throws Exception {
+ File dump = temporaryFolder.newFolder();
+ Data data = prepareData(storeFixture, blobFixture, 10, 0, 0);
+ storeFixture.close();
+ additionalParams += " --verboseRootPath /c1,/c2";
+
+
+ for (String id : data.idToPath.keySet()) {
+ if (data.idToPath.get(id).equals("/c1") || data.idToPath.get(id).equals("/c2")) {
+ data.addedSubset.add(id);
+ }
+ }
+ testConsistency(dump, data, true, true);
+ }
+
+ @Test
+ public void testConsistencyVerboseRootPathWithDeleted() throws Exception {
+ File dump = temporaryFolder.newFolder();
+ Data data = prepareData(storeFixture, blobFixture, 10, 4, 1);
+ storeFixture.close();
+ additionalParams += " --verboseRootPath /c1,/c2";
+
+
+ for (String id : data.idToPath.keySet()) {
+ if (data.idToPath.get(id).equals("/c1") || data.idToPath.get(id).equals("/c2")) {
+ data.addedSubset.add(id);
+ }
+ }
+ testConsistency(dump, data, true, true);
+ }
+
+ @Test
+ public void testConsistencyVerboseRootPath() throws Exception {
+ File dump = temporaryFolder.newFolder();
+ Data data = prepareData(storeFixture, blobFixture, 10, 0, 2);
+ storeFixture.close();
+ additionalParams += " --verboseRootPath /c1,/c2";
+ for (String id : data.idToPath.keySet()) {
+ if (data.idToPath.get(id).equals("/c1") || data.idToPath.get(id).equals("/c2")) {
+ data.addedSubset.add(id);
+ }
+ }
+ testConsistency(dump, data, true, true);
+ }
+
+ @Test
+ public void testConsistencyVerboseRootPathWithoutVerbose() throws Exception {
+ File dump = temporaryFolder.newFolder();
+ Data data = prepareData(storeFixture, blobFixture, 10, 5, 1);
+ storeFixture.close();
+
+ additionalParams += " --verboseRootPath /a";
+ try {
+ // Create exec command without --verbose - It should throw an exception
+ // since we used --verboseRootPath without --verbose
+ testConsistency(dump, data, false, true);
+ } catch (OptionException e) {
+ assertTrue(e.getMessage().equals("Option(s) [verboseRootPath] are unavailable " +
+ "given other options on the command line"));
+ return;
+ }
+ assertFalse(true);
+ }
+
+ @Test
public void testConsistencyNoMissing() throws Exception {
File dump = temporaryFolder.newFolder();
Data data = prepareData(storeFixture, blobFixture, 10, 5, 0);
@@ -353,6 +421,26 @@ public class DataStoreCommandTest {
testGc(dump, data, 0, false);
}
+ /*
+ Command should throw and exception if --verboseRootPath specified
+ with --collect-garbage
+ */
+ @Test
+ public void gcWithVerboseRootPathOption() throws Exception {
+ File dump = temporaryFolder.newFolder();
+ Data data = prepareData(storeFixture, blobFixture, 10, 5, 1);
+ storeFixture.close();
+
+ additionalParams += " --verboseRootPath /a";
+ try {
+ testGc(dump, data, 0, false);
+ } catch (OptionException e) {
+ assertTrue(e.getMessage().equals("Option(s) [verboseRootPath] are unavailable " +
+ "given other options on the command line"));
+ return;
+ }
+ assertFalse(true);
+ }
@Test
public void gcNoDeletion() throws Exception {
@@ -450,13 +538,18 @@ public class DataStoreCommandTest {
assertFileEquals(dump, "marked-", Sets.difference(data.added, data.deleted));
}
+
private void testConsistency(File dump, Data data, boolean verbose) throws Exception {
+ testConsistency(dump, data, verbose, false);
+ }
+
+ private void testConsistency(File dump, Data data, boolean verbose, boolean verboseRootPath) throws Exception {
List<String> argsList = Lists
.newArrayList("--check-consistency", "--" + getOption(blobFixture.getType()), blobFixture.getConfigPath(),
storeFixture.getConnectionString(), "--out-dir", dump.getAbsolutePath(), "--work-dir",
temporaryFolder.newFolder().getAbsolutePath());
if (!Strings.isNullOrEmpty(additionalParams)) {
- argsList.add(additionalParams);
+ argsList.addAll(Splitter.on(" ").splitToList(additionalParams));
}
if (verbose) {
@@ -467,14 +560,18 @@ public class DataStoreCommandTest {
assertFileEquals(dump, "avail-", Sets.difference(data.added, data.missingDataStore));
- // Only verbose or Document would have paths suffixed
- assertFileEquals(dump, "marked-", (verbose || storeFixture instanceof StoreFixture.MongoStoreFixture) ?
- encodedIdsAndPath(Sets.difference(data.added, data.deleted), blobFixture.getType(), data.idToPath, false) :
- Sets.difference(data.added, data.deleted));
+ // Verbose would have paths as well as ids changed but normally only DocumentNS would have paths suffixed
+ assertFileEquals(dump, "marked-", verbose ?
+ encodedIdsAndPath(Sets.difference(verboseRootPath ? data.addedSubset :
+ data.added, data.deleted), blobFixture.getType(), data.idToPath, true) :
+ (storeFixture instanceof StoreFixture.MongoStoreFixture) ?
+ encodedIdsAndPath(Sets.difference(data.added, data.deleted), blobFixture.getType(), data.idToPath, false) :
+ Sets.difference(data.added, data.deleted));
// Verbose would have paths as well as ids changed but normally only DocumentNS would have paths suffixed
assertFileEquals(dump, "gccand-", verbose ?
- encodedIdsAndPath(data.missingDataStore, blobFixture.getType(), data.idToPath, true) :
+ encodedIdsAndPath(verboseRootPath ? Sets.intersection(data.addedSubset, data.missingDataStore) :
+ data.missingDataStore, blobFixture.getType(), data.idToPath, true) :
(storeFixture instanceof StoreFixture.MongoStoreFixture) ?
encodedIdsAndPath(data.missingDataStore, blobFixture.getType(), data.idToPath, false) :
data.missingDataStore);
@@ -635,12 +732,14 @@ public class DataStoreCommandTest {
private Map<String, String> idToPath;
private Set<String> deleted;
private Set<String> missingDataStore;
+ private Set<String> addedSubset;
public Data() {
added = Sets.newHashSet();
idToPath = Maps.newHashMap();
deleted = Sets.newHashSet();
missingDataStore = Sets.newHashSet();
+ addedSubset = Sets.newHashSet();
}
}