You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ng...@apache.org on 2020/02/19 06:14:49 UTC
svn commit: r1874183 - in /jackrabbit/oak/trunk/oak-run: README.md
src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java
Author: ngupta
Date: Wed Feb 19 06:14:49 2020
New Revision: 1874183
URL: http://svn.apache.org/viewvc?rev=1874183&view=rev
Log:
OAK-8902 | Add support in oak-run to list blob ids under specific paths.
Modified:
jackrabbit/oak/trunk/oak-run/README.md
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java
Modified: jackrabbit/oak/trunk/oak-run/README.md
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/README.md?rev=1874183&r1=1874182&r2=1874183&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/README.md (original)
+++ jackrabbit/oak/trunk/oak-run/README.md Wed Feb 19 06:14:49 2020
@@ -491,6 +491,7 @@ Use the following command:
[--repoHome <local_repository_root>]
[--track]
[--verbose]
+ [--verboseRootPath]
The following options are available:
@@ -502,6 +503,7 @@ The following options are available:
--store - Path to the segment store of mongo uri (Required for --ref & --consistency option above)
--dump - Path where to dump the files (Optional). Otherwise, files will be dumped in the user tmp directory.
--s3ds - Path to the S3DataStore configuration file
+ --azureblobds - Path to the AzureDataStore configuration file
--fds - Path to the FileDataStore configuration file ('path' property is mandatory)
--nods - To check for misconfigured external references when no data store should be there (Use instead of --s3ds or --fds)
--repoHome - Path of the local reposity home folder. Mandatory for --consistency & --track options
@@ -509,6 +511,9 @@ The following options are available:
--verbose - Outputs backend friendly blobids and also adds the node path (for SegmentNodeStore) from where referred.
This options would typically be a slower option since, it requires the whole repo traversal.
Adds the sub-directories created in FDS and the changes done for S3/Azure when stored in the respective container.
+ --verboseRootPath- Paths under which backend friendly blobids are required (Optional). If not specified, then --verbose uses "/" as the default path. For example,
+ to list all blobids under /oak:index and /content/oak:index, use --verboseRootPath /oak:index,/content/oak:index (If providing more than one arguments to this option,
+ use comma as a delimiter).
Note:
The command to be executed for S3DataStore
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java?rev=1874183&r1=1874182&r2=1874183&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java Wed Feb 19 06:14:49 2020
@@ -117,7 +117,8 @@ public class DataStoreCheckCommand imple
String helpStr =
"datastorecheck [--id] [--ref] [--consistency] [--store <path>|<mongo_uri>] "
+ "[--s3ds <s3ds_config>|--fds <fds_config>|--azureblobds <azureblobds_config>|--nods]"
- + " [--dump <path>] [--repoHome <repo_home>] [--track] [--verbose]";
+ + " [--dump <path>] [--repoHome <repo_home>] [--track] " +
+ "[--verbose] [--verboseRootPath <verbose_root_path>]";
try (Closer closer = Utils.createCloserWithShutdownHook()) {
// Options for operations requested
@@ -142,6 +143,11 @@ public class DataStoreCheckCommand imple
// Optional argument to specify tracking
OptionSpecBuilder verbose = parser.accepts("verbose", "Output backend formatted ids/paths");
+ // Optional argument to specify root path under which tracking if to be done. Defaults to "/" if not specified
+ ArgumentAcceptingOptionSpec verboseRootPath = parser.accepts("verboseRootPath",
+ "Root path to output backend formatted ids/paths")
+ .withRequiredArg().withValuesSeparatedBy(DELIM).ofType(String.class);
+
OptionSpec<?> help = parser.acceptsAll(asList("h", "?", "help"),
"show help").forHelp();
@@ -242,12 +248,19 @@ public class DataStoreCheckCommand imple
}
if (options.has(refOp) || options.has(consistencyOp)) {
- if (options.has(verbose) &&
+
+ // Find blob ids by traversal for verbose mode + Segment Store or if verboseRootPath option
+ // is present (find blob references under a specific root path.)
+ if ((options.has(verbose) &&
(nodeStore instanceof SegmentNodeStore ||
- nodeStore instanceof org.apache.jackrabbit.oak.segment.SegmentNodeStore)) {
+ nodeStore instanceof org.apache.jackrabbit.oak.segment.SegmentNodeStore)) ||
+ options.has(verboseRootPath)) {
NodeTraverser traverser = new NodeTraverser(nodeStore, dsType);
closer.register(traverser);
- traverser.traverse();
+
+ List<String> rootPathList = options.valuesOf(verboseRootPath);
+ traverser.traverse((String[]) rootPathList.toArray(new String[rootPathList.size()]));
+
FileUtils.copyFile(traverser.references, register.createFile(refOp, dumpPath));
} else {
retrieveBlobReferences(blobStore, marker,
@@ -486,18 +499,29 @@ public class DataStoreCheckCommand imple
for (PropertyState p : state.getProperties()) {
String propPath = PathUtils.concat(path, p.getName());
try {
+ String id ;
if (p.getType() == Type.BINARY) {
- count.incrementAndGet();
+ id = p.getValue(Type.BINARY).getContentIdentity();
+ // Ignore inline encoded binaries in document mk and null references in segment mk
+ if (id == null || p.getValue(Type.BINARY).isInlined()) {
+ continue;
+ }
writeAsLine(writer,
- getLine(p.getValue(Type.BINARY).getContentIdentity(), propPath), false);
+ getLine(id, propPath), false);
+ count.incrementAndGet();
+
} else if (p.getType() == Type.BINARIES && p.count() > 0) {
Iterator<Blob> iterator = p.getValue(Type.BINARIES).iterator();
while (iterator.hasNext()) {
- count.incrementAndGet();
- String id = iterator.next().getContentIdentity();
+ id = iterator.next().getContentIdentity();
+ // Ignore inline encoded binaries in document mk
+ if (id == null || p.getValue(Type.BINARY).isInlined()) {
+ continue;
+ }
writeAsLine(writer,
getLine(id, propPath), false);
+ count.incrementAndGet();
}
}
} catch (Exception e) {
@@ -517,7 +541,7 @@ public class DataStoreCheckCommand imple
}
}
- public void traverse() throws IOException {
+ public void traverse(String ... paths) throws IOException {
BufferedWriter writer = null;
final AtomicInteger count = new AtomicInteger();
boolean threw = true;
@@ -526,7 +550,18 @@ public class DataStoreCheckCommand imple
try {
writer = Files.newWriter(references, Charsets.UTF_8);
- traverseChildren(nodeStore.getRoot(), "/", writer, count);
+ if (paths.length == 0) {
+ traverseChildren(nodeStore.getRoot(), "/", writer, count);
+ } else {
+ for (String path: paths ) {
+ Iterable<String> nodeList = PathUtils.elements(path);
+ NodeState state = nodeStore.getRoot();
+ for (String node: nodeList) {
+ state = state.getChildNode(node);
+ }
+ traverseChildren(state, path, writer, count);
+ }
+ }
writer.flush();
sort(references, idComparator);