You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@jackrabbit.apache.org by GitBox <gi...@apache.org> on 2020/07/02 16:32:14 UTC
[GitHub] [jackrabbit-oak] stefan-egli opened a new pull request #238: OAK-9131 : introducing scanners for references to nt:frozenNode
stefan-egli opened a new pull request #238:
URL: https://github.com/apache/jackrabbit-oak/pull/238
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [jackrabbit-oak] stefan-egli commented on pull request #238: OAK-9131 : introducing scanners for references to nt:frozenNode
Posted by GitBox <gi...@apache.org>.
stefan-egli commented on pull request #238:
URL: https://github.com/apache/jackrabbit-oak/pull/238#issuecomment-657620022
> > Could you please also update the README.md?
>
> Yes, will add next, thx for the pointer
Now done in [this commit](https://github.com/apache/jackrabbit-oak/pull/238/commits/7a0e090c200b4081416393d0a8bcbb643dd87945)
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [jackrabbit-oak] stefan-egli commented on a change in pull request #238: OAK-9131 : introducing scanners for references to nt:frozenNode
Posted by GitBox <gi...@apache.org>.
stefan-egli commented on a change in pull request #238:
URL: https://github.com/apache/jackrabbit-oak/pull/238#discussion_r453710617
##########
File path: oak-run/src/main/java/org/apache/jackrabbit/oak/run/FrozenNodeRefsByScanningCommand.java
##########
@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.run;
+
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+
+import javax.jcr.ItemNotFoundException;
+import javax.jcr.Node;
+import javax.jcr.NodeIterator;
+import javax.jcr.Property;
+import javax.jcr.PropertyIterator;
+import javax.jcr.PropertyType;
+import javax.jcr.Repository;
+import javax.jcr.RepositoryException;
+import javax.jcr.Session;
+import javax.jcr.SimpleCredentials;
+import javax.jcr.Value;
+import javax.jcr.query.Query;
+import javax.jcr.query.QueryManager;
+import javax.jcr.query.QueryResult;
+
+import org.apache.jackrabbit.oak.Oak;
+import org.apache.jackrabbit.oak.jcr.Jcr;
+import org.apache.jackrabbit.oak.plugins.index.lucene.IndexTracker;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.DocumentQueue;
+import org.apache.jackrabbit.oak.run.cli.CommonOptions;
+import org.apache.jackrabbit.oak.run.cli.NodeStoreFixture;
+import org.apache.jackrabbit.oak.run.cli.NodeStoreFixtureProvider;
+import org.apache.jackrabbit.oak.run.cli.Options;
+import org.apache.jackrabbit.oak.run.commons.Command;
+import org.apache.jackrabbit.oak.spi.commit.Observer;
+import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.apache.jackrabbit.oak.stats.StatisticsProvider;
+
+import com.google.common.io.Closer;
+import com.google.common.util.concurrent.MoreExecutors;
+
+import joptsimple.OptionParser;
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+
+/**
+ * Scans and lists all references to nt:frozenNode and returns an exit code of 1 if any are found (0 otherwise).
+ * <p/>
+ * This variant does a *very expensive repository scan* for all properties formatted as uuid
+ * ( LIKE \"________-____-____-____-____________\" )
+ * and checking if any reference points to an nt:frozenNode (under /jcr:system/jcr:versionStorage
+ * at depth > 7).
+ * <p/>
+ * Note that any property with uuid that cannot be resolved will *not be reported*, as that
+ * is a legitimate use case of uuid property use. Only uuids that resolve will be analysed.
+ * <p/>
+ * Also note that this command has a few test parameters which can be used to "easily"
+ * create a property with a reference (or just a string for that matter too).
+ * <p/>
+ * Example:
+ * <pre>
+ * java -mx4g -jar oak-run-*.jar frozennoderefsbyscanning mongodb://localhost/<dbname> -user=admin -password=admin
+ * </pre>
+ * <p/>
+ * Test Example for creating a reference:
+ * <pre>
+ * java -mx4g -jar oak-run-*.jar frozennoderefsbyscanning mongodb://localhost/<dbname> -user=admin -password=admin -testCreateRefPath=<mypath> -testCreateRefProp=<mypropertyname> -testCreateRefUuid=<myuuid> -read-write=true
+ * </pre>
+ */
+public class FrozenNodeRefsByScanningCommand implements Command {
+
+ static {
+ // disable any query limits as our query is going to be a fulls can, and we are aware of it
+ System.setProperty("oak.queryLimitReads", String.valueOf(Long.MAX_VALUE));
+
+ // // disable the WARN of the TraversingCursor
+ // LoggerContext c = (LoggerContext) LoggerFactory.getILoggerFactory();
+ // Logger logger = c.getLogger("org.apache.jackrabbit.oak.plugins.index.Cursors$TraversingCursor");
+ // logger.setLevel(Level.ERROR);
+ }
+
+ public static final String NAME = "frozennoderefsbyscanning";
+
+ private final String summary = "Scans and lists all references to nt:frozenNode";
Review comment:
thx! fixed
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [jackrabbit-oak] stefan-egli closed pull request #238: OAK-9131 : introducing scanners for references to nt:frozenNode
Posted by GitBox <gi...@apache.org>.
stefan-egli closed pull request #238:
URL: https://github.com/apache/jackrabbit-oak/pull/238
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [jackrabbit-oak] stefan-egli commented on a change in pull request #238: OAK-9131 : introducing scanners for references to nt:frozenNode
Posted by GitBox <gi...@apache.org>.
stefan-egli commented on a change in pull request #238:
URL: https://github.com/apache/jackrabbit-oak/pull/238#discussion_r453712113
##########
File path: oak-run/src/main/java/org/apache/jackrabbit/oak/run/FrozenNodeRefsUsingIndexCommand.java
##########
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.run;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.commons.PathUtils;
+import org.apache.jackrabbit.oak.plugins.memory.GenericPropertyState;
+import org.apache.jackrabbit.oak.run.Utils.NodeStoreOptions;
+import org.apache.jackrabbit.oak.run.commons.Command;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
+import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+
+import com.google.common.io.Closer;
+
+/**
+ * Scans and lists all references to nt:frozenNode and returns an exit code of 1 if any are found (0 otherwise).
+ * <p/>
+ * This variant uses the /oak:index/reference by scanning through that list (only)
+ * and checking if any reference points to an nt:frozenNode (under /jcr:system/jcr:versionStorage
+ * at depth > 7).
+ * <p/>
+ * Example:
+ * <pre>
+ * java -mx4g -jar oak-run-*.jar frozennoderefsusingindex mongodb://localhost/<dbname>
+ * </pre>
+ */
+public class FrozenNodeRefsUsingIndexCommand implements Command {
+
+ public static final String NAME = "frozennoderefsusingindex";
+
+ @Override
+ public void execute(String... args) throws Exception {
+ String help = NAME + " {<path>|<mongo-uri>|<jdbc-uri>} [options]";
+ Utils.NodeStoreOptions nopts = new Utils.NodeStoreOptions(help).parse(args);
+ if (hasNtFrozenNodeReferences(nopts)) {
+ System.err.println("FAILURE: References (in /oak:index/references) to nt:frozenNode found.");
+ System.exit(1);
+ } else {
+ System.out.println("SUCCESS: No references (in /oak:index/references) to nt:frozenNode found.");
+ }
+ }
+
+ private static boolean hasNtFrozenNodeReferences(NodeStoreOptions nopts) throws IOException {
+ Closer closer = Utils.createCloserWithShutdownHook();
+ try {
+ NodeStore store = Utils.bootstrapNodeStore(nopts, closer);
+ NodeState root = store.getRoot();
+
+ NodeBuilder b = root.builder();
Review comment:
Oups, this one is particularly ugly - removed this part entirely (at https://github.com/apache/jackrabbit-oak/pull/238/commits/fb8b330c4f248078bb2dce62cfb19ae90a50030c )
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [jackrabbit-oak] stefan-egli edited a comment on pull request #238: OAK-9131 : introducing scanners for references to nt:frozenNode
Posted by GitBox <gi...@apache.org>.
stefan-egli edited a comment on pull request #238:
URL: https://github.com/apache/jackrabbit-oak/pull/238#issuecomment-658171682
> Hmm, it still looks like the scan command starts a repository in read-write mode.
Indeed, thx for the heads-up!
Fixed it now:
* the `frozennoderefsbyscanning` variant already opened in readOnly mode, unless `-read-write=true` was set. Now explicitly disabled that option and therefore always using readOnly.
* the `frozennoderefsusingindex` variant was indeed opening in readWrite (!) mode. As it is using a completely different way of accessing the NodeStore, that utils variant didn't yet support readOnly mode - so I've now added this, and used it for this case.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [jackrabbit-oak] stefan-egli commented on pull request #238: OAK-9131 : introducing scanners for references to nt:frozenNode
Posted by GitBox <gi...@apache.org>.
stefan-egli commented on pull request #238:
URL: https://github.com/apache/jackrabbit-oak/pull/238#issuecomment-658171682
> Hmm, it still looks like the scan command starts a repository in read-write mode.
Indeed, thx for the heads-up!
Fixed it now:
* the `frozennoderefsbyscanning` variant already opened in readOnly mode, unless `-read-write=true` was set. Now explicitly disabled that option and therefore always using readOnly.
* the `frozennoderefsusingindex` variant was indeed opening in readWrite (!) mode. As it is using a completely different way of accessing the NodeStore, that utils variant didn't yet support readOnly mode - so I've now added this, and used it for this case.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [jackrabbit-oak] mreutegg commented on a change in pull request #238: OAK-9131 : introducing scanners for references to nt:frozenNode
Posted by GitBox <gi...@apache.org>.
mreutegg commented on a change in pull request #238:
URL: https://github.com/apache/jackrabbit-oak/pull/238#discussion_r454161744
##########
File path: oak-run/README.md
##########
@@ -136,6 +138,42 @@ browsing of an existing oak repository.
$ java -jar oak-run-*.jar explore /path/to/oak/repository [skip-size-check]
+frozennoderefsbyscanning
+------------------------
+
+This command executes a potentially expensive (!) travering query searching for
Review comment:
```suggestion
This command executes a potentially expensive (!) traversing query searching for
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [jackrabbit-oak] stefan-egli commented on a change in pull request #238: OAK-9131 : introducing scanners for references to nt:frozenNode
Posted by GitBox <gi...@apache.org>.
stefan-egli commented on a change in pull request #238:
URL: https://github.com/apache/jackrabbit-oak/pull/238#discussion_r453711579
##########
File path: oak-run/src/main/java/org/apache/jackrabbit/oak/run/FrozenNodeRefsByScanningCommand.java
##########
@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.run;
+
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+
+import javax.jcr.ItemNotFoundException;
+import javax.jcr.Node;
+import javax.jcr.NodeIterator;
+import javax.jcr.Property;
+import javax.jcr.PropertyIterator;
+import javax.jcr.PropertyType;
+import javax.jcr.Repository;
+import javax.jcr.RepositoryException;
+import javax.jcr.Session;
+import javax.jcr.SimpleCredentials;
+import javax.jcr.Value;
+import javax.jcr.query.Query;
+import javax.jcr.query.QueryManager;
+import javax.jcr.query.QueryResult;
+
+import org.apache.jackrabbit.oak.Oak;
+import org.apache.jackrabbit.oak.jcr.Jcr;
+import org.apache.jackrabbit.oak.plugins.index.lucene.IndexTracker;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.DocumentQueue;
+import org.apache.jackrabbit.oak.run.cli.CommonOptions;
+import org.apache.jackrabbit.oak.run.cli.NodeStoreFixture;
+import org.apache.jackrabbit.oak.run.cli.NodeStoreFixtureProvider;
+import org.apache.jackrabbit.oak.run.cli.Options;
+import org.apache.jackrabbit.oak.run.commons.Command;
+import org.apache.jackrabbit.oak.spi.commit.Observer;
+import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.apache.jackrabbit.oak.stats.StatisticsProvider;
+
+import com.google.common.io.Closer;
+import com.google.common.util.concurrent.MoreExecutors;
+
+import joptsimple.OptionParser;
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+
+/**
+ * Scans and lists all references to nt:frozenNode and returns an exit code of 1 if any are found (0 otherwise).
+ * <p/>
+ * This variant does a *very expensive repository scan* for all properties formatted as uuid
+ * ( LIKE \"________-____-____-____-____________\" )
+ * and checking if any reference points to an nt:frozenNode (under /jcr:system/jcr:versionStorage
+ * at depth > 7).
+ * <p/>
+ * Note that any property with uuid that cannot be resolved will *not be reported*, as that
+ * is a legitimate use case of uuid property use. Only uuids that resolve will be analysed.
+ * <p/>
+ * Also note that this command has a few test parameters which can be used to "easily"
+ * create a property with a reference (or just a string for that matter too).
+ * <p/>
+ * Example:
+ * <pre>
+ * java -mx4g -jar oak-run-*.jar frozennoderefsbyscanning mongodb://localhost/<dbname> -user=admin -password=admin
+ * </pre>
+ * <p/>
+ * Test Example for creating a reference:
+ * <pre>
+ * java -mx4g -jar oak-run-*.jar frozennoderefsbyscanning mongodb://localhost/<dbname> -user=admin -password=admin -testCreateRefPath=<mypath> -testCreateRefProp=<mypropertyname> -testCreateRefUuid=<myuuid> -read-write=true
+ * </pre>
+ */
+public class FrozenNodeRefsByScanningCommand implements Command {
+
+ static {
+ // disable any query limits as our query is going to be a fulls can, and we are aware of it
+ System.setProperty("oak.queryLimitReads", String.valueOf(Long.MAX_VALUE));
+
+ // // disable the WARN of the TraversingCursor
+ // LoggerContext c = (LoggerContext) LoggerFactory.getILoggerFactory();
+ // Logger logger = c.getLogger("org.apache.jackrabbit.oak.plugins.index.Cursors$TraversingCursor");
+ // logger.setLevel(Level.ERROR);
+ }
+
+ public static final String NAME = "frozennoderefsbyscanning";
+
+ private final String summary = "Scans and lists all references to nt:frozenNode";
+
+ @Override
+ public void execute(String... args) throws Exception {
+ OptionParser parser = new OptionParser();
+
+ Options opts = new Options();
+ opts.setCommandName(NAME);
+ opts.setSummary(summary);
+ opts.setConnectionString(CommonOptions.DEFAULT_CONNECTION_STRING);
+
+ OptionSpec<String> userOption = parser.accepts("user", "User name").withOptionalArg().defaultsTo("admin");
+ OptionSpec<String> passwordOption = parser.accepts("password", "Password").withOptionalArg().defaultsTo("admin");
+
+ OptionSpec<String> testCreateRefPathOption = parser.accepts("testCreateRefPath", "FOR TESTING ONLY: path where to create a reference from")
+ .withOptionalArg();
+ OptionSpec<String> testCreateRefPropOption = parser.accepts("testCreateRefProp", "FOR TESTING ONLY: property name for create a reference")
+ .withOptionalArg();
+ OptionSpec<String> testCreateRefTypeOption = parser
+ .accepts("testCreateRefType", "FOR TESTING ONLY: property type: 'reference' or anything else for a plain String").withOptionalArg();
+ OptionSpec<String> testCreateRefUuidOption = parser.accepts("testCreateRefUuid", "FOR TESTING ONLY: uuid to use as the reference")
+ .withOptionalArg();
+
+ OptionSet options = opts.parseAndConfigure(parser, args);
+
+ System.out.println("Opening nodestore...");
+ NodeStoreFixture nodeStoreFixture = NodeStoreFixtureProvider.create(opts);
+ System.out.println("Nodestore opened.");
+
+ if (uuidscan(userOption, passwordOption, testCreateRefPathOption, testCreateRefPropOption, testCreateRefTypeOption, testCreateRefUuidOption,
+ options, nodeStoreFixture)) {
+ System.err.println("FAILURE: References (in any uuid formatted property value) to nt:frozenNode found.");
+ System.exit(1);
+ } else {
+ System.out.println("SUCCESS: No references (in any uuid formatted property value) to nt:frozenNode found.");
+ }
+ }
+
+ private boolean uuidscan(OptionSpec<String> userOption, OptionSpec<String> passwordOption, OptionSpec<String> testCreateRefPathOption,
+ OptionSpec<String> testCreateRefPropOption, OptionSpec<String> testCreateRefTypeOption, OptionSpec<String> testCreateRefUuidOption,
+ OptionSet options, NodeStoreFixture nodeStoreFixture) throws IOException {
+ List<FrozenNodeRef> list = new LinkedList<FrozenNodeRef>();
+
+ NodeStore nodeStore = nodeStoreFixture.getStore();
+ String user = userOption.value(options);
+ String password = passwordOption.value(options);
+
+ String createRefPath = testCreateRefPathOption.value(options);
+ String createRefProp = testCreateRefPropOption.value(options);
+ String createRefType = testCreateRefTypeOption.value(options);
+ String createRefUuid = testCreateRefUuidOption.value(options);
+
+ Closer closer = Utils.createCloserWithShutdownHook();
+ closer.register(nodeStoreFixture);
+ try {
+
+ System.out.println("Logging in...");
+ Session session = openSession(nodeStore, "crx.default", user, password);
+
+ if (createRefPath != null && createRefProp != null && createRefUuid != null) {
+ // this part of the code is only executed if any of the -testXX properties
+ // are set - which is as the name suggests only for setting up a test.
+ System.out.println(
+ "Logged in, creating test reference: " + "path=" + createRefPath + ", property=" + createRefProp + ", uuid=" + createRefUuid);
+ Node n = session.getNode(createRefPath);
+ Value v;
+ if ("reference".equals(createRefType)) {
+ v = session.getValueFactory().createValue(createRefUuid, PropertyType.REFERENCE);
+ } else {
+ v = session.getValueFactory().createValue(createRefUuid, PropertyType.STRING);
+ }
+ n.setProperty(createRefProp, v);
+ session.save();
+ System.out.println("Created. Done.");
+ session.logout();
+ closer.close();
+ System.exit(0);
+ }
+
+ System.out.println("Logged in, querying...");
+ QueryManager qm = session.getWorkspace().getQueryManager();
+
+ // query for only getting 'Reference' and 'WeakReference' would be :
+ // SELECT * FROM [nt:base] AS p WHERE PROPERTY(*, 'Reference') IS NOT NULL OR PROPERTY(*, 'WeakReference') IS NOT NULL
+ Query q = qm.createQuery("SELECT * FROM [nt:base] AS p WHERE PROPERTY(*, '*') LIKE \"________-____-____-____-____________\"", "JCR-SQL2");
+ QueryResult qr = q.execute();
+ NodeIterator it = qr.getNodes();
+
+ while (it.hasNext()) {
+ Node n = it.nextNode();
+ PropertyIterator pit = n.getProperties();
+ while (pit.hasNext()) {
+ Property p = pit.nextProperty();
+ if ("jcr:uuid".equals(p.getName())) {
+ // jcr:uuid should be skipped as that's the identifier of a node, not a reference
+ continue;
+ }
+ if (!p.isMultiple()) {
+ String propValue = p.getValue().getString();
+ if (propValue.matches("........-....-....-....-............")) {
+ verify(list, session, n, p, propValue);
+ }
+ } else {
+ for (Value v : p.getValues()) {
+ String propValue = v.getString();
+ if (propValue.matches("........-....-....-....-............")) {
+ verify(list, session, n, p, propValue);
+ }
+ }
+ }
+ }
+ }
+
+ System.out.println("logout...");
+ session.logout();
+ System.out.println("done.");
+
+ return !list.isEmpty();
+ } catch (Throwable e) {
+ throw closer.rethrow(e);
+ } finally {
+ closer.close();
+ }
+ }
+
+ private void verify(List<FrozenNodeRef> list, Session session, Node n, Property p, String propValue) throws RepositoryException {
+ try {
+ Node node = session.getNodeByIdentifier(propValue);
+ String path = node.getPath();
+ boolean candidate = FrozenNodeRef.isFrozenNodeReferenceCandidate(path);
+ if (!candidate) {
+ return;
+ }
+ Property primaryType = node.getProperty("jcr:primaryType");
+ String primaryTypeValue = primaryType.getString();
+ boolean isNtFrozenNode = "nt:frozenNode".equals(primaryTypeValue);
+ if (!isNtFrozenNode) {
+ // this is where we ultimately have to continue out in any case - as only an nt:frozenNode
+ // is what we're interested in.
+ return;
+ }
+
+ String uuid = propValue;
+ String referrerPath = n.getPath();
+ String referrerProperty = p.getName();
+ FrozenNodeRef ref = new FrozenNodeRef(referrerPath, referrerProperty, PropertyType.nameFromValue(p.getType()), uuid, path);
+ list.add(ref);
Review comment:
Good point. The list was more for reporting each find - but ultimately there's no other use case for this atm, and the reporting itself can done as it happens. So I've now changed this to only carry a `count` at https://github.com/apache/jackrabbit-oak/pull/238/commits/fb8b330c4f248078bb2dce62cfb19ae90a50030c
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [jackrabbit-oak] stefan-egli commented on pull request #238: OAK-9131 : introducing scanners for references to nt:frozenNode
Posted by GitBox <gi...@apache.org>.
stefan-egli commented on pull request #238:
URL: https://github.com/apache/jackrabbit-oak/pull/238#issuecomment-659312525
Merged to trunk in rev [1879938](http://svn.apache.org/viewvc?rev=1879938&view=rev)
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [jackrabbit-oak] stefan-egli commented on pull request #238: OAK-9131 : introducing scanners for references to nt:frozenNode
Posted by GitBox <gi...@apache.org>.
stefan-egli commented on pull request #238:
URL: https://github.com/apache/jackrabbit-oak/pull/238#issuecomment-657612134
@mreutegg thx for the review - I've done the changes in [this commit](https://github.com/apache/jackrabbit-oak/pull/238/commits/fb8b330c4f248078bb2dce62cfb19ae90a50030c)
> In general, can those two commands run in read-only mode? This would make them safe to run on data that is in use. E.g. a production system.
Yes, and with the suggested move of the test code to a test class this is now also more clear: only the test class uses the `-read-write=true` parameter - so even more reason to move test code to a test class indeed.
> Could you please also update the README.md?
Yes, will add next, thx for the pointer
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [jackrabbit-oak] stefan-egli commented on a change in pull request #238: OAK-9131 : introducing scanners for references to nt:frozenNode
Posted by GitBox <gi...@apache.org>.
stefan-egli commented on a change in pull request #238:
URL: https://github.com/apache/jackrabbit-oak/pull/238#discussion_r453711842
##########
File path: oak-run/src/main/java/org/apache/jackrabbit/oak/run/FrozenNodeRefsUsingIndexCommand.java
##########
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.run;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.commons.PathUtils;
+import org.apache.jackrabbit.oak.plugins.memory.GenericPropertyState;
+import org.apache.jackrabbit.oak.run.Utils.NodeStoreOptions;
+import org.apache.jackrabbit.oak.run.commons.Command;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
+import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+
+import com.google.common.io.Closer;
+
+/**
+ * Scans and lists all references to nt:frozenNode and returns an exit code of 1 if any are found (0 otherwise).
+ * <p/>
+ * This variant uses the /oak:index/reference by scanning through that list (only)
+ * and checking if any reference points to an nt:frozenNode (under /jcr:system/jcr:versionStorage
+ * at depth > 7).
+ * <p/>
+ * Example:
+ * <pre>
+ * java -mx4g -jar oak-run-*.jar frozennoderefsusingindex mongodb://localhost/<dbname>
+ * </pre>
+ */
+public class FrozenNodeRefsUsingIndexCommand implements Command {
+
+ public static final String NAME = "frozennoderefsusingindex";
+
+ @Override
+ public void execute(String... args) throws Exception {
+ String help = NAME + " {<path>|<mongo-uri>|<jdbc-uri>} [options]";
+ Utils.NodeStoreOptions nopts = new Utils.NodeStoreOptions(help).parse(args);
+ if (hasNtFrozenNodeReferences(nopts)) {
+ System.err.println("FAILURE: References (in /oak:index/references) to nt:frozenNode found.");
+ System.exit(1);
+ } else {
+ System.out.println("SUCCESS: No references (in /oak:index/references) to nt:frozenNode found.");
+ }
+ }
+
+ private static boolean hasNtFrozenNodeReferences(NodeStoreOptions nopts) throws IOException {
+ Closer closer = Utils.createCloserWithShutdownHook();
+ try {
+ NodeStore store = Utils.bootstrapNodeStore(nopts, closer);
+ NodeState root = store.getRoot();
+
+ NodeBuilder b = root.builder();
+ NodeBuilder content = b.child("content");
+ PropertyState refProp = GenericPropertyState.referenceProperty("myref", "dd31e965-7e74-4c04-9a64-6698ecb4c624");
+ content.setProperty(refProp);
+ store.merge(b, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ NodeState oakIndex = root.getChildNode("oak:index");
+ NodeState refIndex = oakIndex.getChildNode("reference");
+ NodeState uuidIndex = oakIndex.getChildNode("uuid");
+ NodeState uuids = uuidIndex.getChildNode(":index");
+ System.out.println("Scanning ... refindex = " + refIndex);
+ NodeState references = refIndex.getChildNode(":references");
+
+ List<FrozenNodeRef> list = new LinkedList<FrozenNodeRef>();
Review comment:
Agree - changed to a `count` at https://github.com/apache/jackrabbit-oak/pull/238/commits/fb8b330c4f248078bb2dce62cfb19ae90a50030c
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [jackrabbit-oak] stefan-egli commented on a change in pull request #238: OAK-9131 : introducing scanners for references to nt:frozenNode
Posted by GitBox <gi...@apache.org>.
stefan-egli commented on a change in pull request #238:
URL: https://github.com/apache/jackrabbit-oak/pull/238#discussion_r453710952
##########
File path: oak-run/src/main/java/org/apache/jackrabbit/oak/run/FrozenNodeRefsByScanningCommand.java
##########
@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.run;
+
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+
+import javax.jcr.ItemNotFoundException;
+import javax.jcr.Node;
+import javax.jcr.NodeIterator;
+import javax.jcr.Property;
+import javax.jcr.PropertyIterator;
+import javax.jcr.PropertyType;
+import javax.jcr.Repository;
+import javax.jcr.RepositoryException;
+import javax.jcr.Session;
+import javax.jcr.SimpleCredentials;
+import javax.jcr.Value;
+import javax.jcr.query.Query;
+import javax.jcr.query.QueryManager;
+import javax.jcr.query.QueryResult;
+
+import org.apache.jackrabbit.oak.Oak;
+import org.apache.jackrabbit.oak.jcr.Jcr;
+import org.apache.jackrabbit.oak.plugins.index.lucene.IndexTracker;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.DocumentQueue;
+import org.apache.jackrabbit.oak.run.cli.CommonOptions;
+import org.apache.jackrabbit.oak.run.cli.NodeStoreFixture;
+import org.apache.jackrabbit.oak.run.cli.NodeStoreFixtureProvider;
+import org.apache.jackrabbit.oak.run.cli.Options;
+import org.apache.jackrabbit.oak.run.commons.Command;
+import org.apache.jackrabbit.oak.spi.commit.Observer;
+import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.apache.jackrabbit.oak.stats.StatisticsProvider;
+
+import com.google.common.io.Closer;
+import com.google.common.util.concurrent.MoreExecutors;
+
+import joptsimple.OptionParser;
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+
+/**
+ * Scans and lists all references to nt:frozenNode and returns an exit code of 1 if any are found (0 otherwise).
+ * <p/>
+ * This variant does a *very expensive repository scan* for all properties formatted as uuid
+ * ( LIKE \"________-____-____-____-____________\" )
+ * and checking if any reference points to an nt:frozenNode (under /jcr:system/jcr:versionStorage
+ * at depth > 7).
+ * <p/>
+ * Note that any property with uuid that cannot be resolved will *not be reported*, as that
+ * is a legitimate use case of uuid property use. Only uuids that resolve will be analysed.
+ * <p/>
+ * Also note that this command has a few test parameters which can be used to "easily"
+ * create a property with a reference (or just a string for that matter too).
+ * <p/>
+ * Example:
+ * <pre>
+ * java -mx4g -jar oak-run-*.jar frozennoderefsbyscanning mongodb://localhost/<dbname> -user=admin -password=admin
+ * </pre>
+ * <p/>
+ * Test Example for creating a reference:
+ * <pre>
+ * java -mx4g -jar oak-run-*.jar frozennoderefsbyscanning mongodb://localhost/<dbname> -user=admin -password=admin -testCreateRefPath=<mypath> -testCreateRefProp=<mypropertyname> -testCreateRefUuid=<myuuid> -read-write=true
+ * </pre>
+ */
+public class FrozenNodeRefsByScanningCommand implements Command {
+
+ static {
+ // disable any query limits as our query is going to be a fulls can, and we are aware of it
+ System.setProperty("oak.queryLimitReads", String.valueOf(Long.MAX_VALUE));
+
+ // // disable the WARN of the TraversingCursor
+ // LoggerContext c = (LoggerContext) LoggerFactory.getILoggerFactory();
+ // Logger logger = c.getLogger("org.apache.jackrabbit.oak.plugins.index.Cursors$TraversingCursor");
+ // logger.setLevel(Level.ERROR);
+ }
+
+ public static final String NAME = "frozennoderefsbyscanning";
+
+ private final String summary = "Scans and lists all references to nt:frozenNode";
+
+ @Override
+ public void execute(String... args) throws Exception {
+ OptionParser parser = new OptionParser();
+
+ Options opts = new Options();
+ opts.setCommandName(NAME);
+ opts.setSummary(summary);
+ opts.setConnectionString(CommonOptions.DEFAULT_CONNECTION_STRING);
+
+ OptionSpec<String> userOption = parser.accepts("user", "User name").withOptionalArg().defaultsTo("admin");
+ OptionSpec<String> passwordOption = parser.accepts("password", "Password").withOptionalArg().defaultsTo("admin");
+
+ OptionSpec<String> testCreateRefPathOption = parser.accepts("testCreateRefPath", "FOR TESTING ONLY: path where to create a reference from")
+ .withOptionalArg();
+ OptionSpec<String> testCreateRefPropOption = parser.accepts("testCreateRefProp", "FOR TESTING ONLY: property name for create a reference")
+ .withOptionalArg();
+ OptionSpec<String> testCreateRefTypeOption = parser
+ .accepts("testCreateRefType", "FOR TESTING ONLY: property type: 'reference' or anything else for a plain String").withOptionalArg();
+ OptionSpec<String> testCreateRefUuidOption = parser.accepts("testCreateRefUuid", "FOR TESTING ONLY: uuid to use as the reference")
+ .withOptionalArg();
+
+ OptionSet options = opts.parseAndConfigure(parser, args);
+
+ System.out.println("Opening nodestore...");
+ NodeStoreFixture nodeStoreFixture = NodeStoreFixtureProvider.create(opts);
+ System.out.println("Nodestore opened.");
+
+ if (uuidscan(userOption, passwordOption, testCreateRefPathOption, testCreateRefPropOption, testCreateRefTypeOption, testCreateRefUuidOption,
+ options, nodeStoreFixture)) {
+ System.err.println("FAILURE: References (in any uuid formatted property value) to nt:frozenNode found.");
+ System.exit(1);
+ } else {
+ System.out.println("SUCCESS: No references (in any uuid formatted property value) to nt:frozenNode found.");
+ }
+ }
+
+ private boolean uuidscan(OptionSpec<String> userOption, OptionSpec<String> passwordOption, OptionSpec<String> testCreateRefPathOption,
+ OptionSpec<String> testCreateRefPropOption, OptionSpec<String> testCreateRefTypeOption, OptionSpec<String> testCreateRefUuidOption,
+ OptionSet options, NodeStoreFixture nodeStoreFixture) throws IOException {
+ List<FrozenNodeRef> list = new LinkedList<FrozenNodeRef>();
+
+ NodeStore nodeStore = nodeStoreFixture.getStore();
+ String user = userOption.value(options);
+ String password = passwordOption.value(options);
+
+ String createRefPath = testCreateRefPathOption.value(options);
+ String createRefProp = testCreateRefPropOption.value(options);
+ String createRefType = testCreateRefTypeOption.value(options);
+ String createRefUuid = testCreateRefUuidOption.value(options);
+
+ Closer closer = Utils.createCloserWithShutdownHook();
+ closer.register(nodeStoreFixture);
+ try {
+
+ System.out.println("Logging in...");
+ Session session = openSession(nodeStore, "crx.default", user, password);
+
+ if (createRefPath != null && createRefProp != null && createRefUuid != null) {
Review comment:
Agree, much nicer. Done now at https://github.com/apache/jackrabbit-oak/pull/238/commits/fb8b330c4f248078bb2dce62cfb19ae90a50030c
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [jackrabbit-oak] mreutegg commented on pull request #238: OAK-9131 : introducing scanners for references to nt:frozenNode
Posted by GitBox <gi...@apache.org>.
mreutegg commented on pull request #238:
URL: https://github.com/apache/jackrabbit-oak/pull/238#issuecomment-658032610
> > In general, can those two commands run in read-only mode? This would make them safe to run on data that is in use. E.g. a production system.
>
> Yes, and with the suggested move of the test code to a test class this is now also more clear: only the test class uses the `-read-write=true` parameter - so even more reason to move test code to a test class indeed.
Hmm, it still looks like the scan command starts a repository in read-write mode.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [jackrabbit-oak] stefan-egli commented on a change in pull request #238: OAK-9131 : introducing scanners for references to nt:frozenNode
Posted by GitBox <gi...@apache.org>.
stefan-egli commented on a change in pull request #238:
URL: https://github.com/apache/jackrabbit-oak/pull/238#discussion_r454279208
##########
File path: oak-run/README.md
##########
@@ -136,6 +138,42 @@ browsing of an existing oak repository.
$ java -jar oak-run-*.jar explore /path/to/oak/repository [skip-size-check]
+frozennoderefsbyscanning
+------------------------
+
+This command executes a potentially expensive (!) travering query searching for
Review comment:
committed, thx
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [jackrabbit-oak] mreutegg commented on a change in pull request #238: OAK-9131 : introducing scanners for references to nt:frozenNode
Posted by GitBox <gi...@apache.org>.
mreutegg commented on a change in pull request #238:
URL: https://github.com/apache/jackrabbit-oak/pull/238#discussion_r452323663
##########
File path: oak-run/src/main/java/org/apache/jackrabbit/oak/run/FrozenNodeRefsByScanningCommand.java
##########
@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.run;
+
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+
+import javax.jcr.ItemNotFoundException;
+import javax.jcr.Node;
+import javax.jcr.NodeIterator;
+import javax.jcr.Property;
+import javax.jcr.PropertyIterator;
+import javax.jcr.PropertyType;
+import javax.jcr.Repository;
+import javax.jcr.RepositoryException;
+import javax.jcr.Session;
+import javax.jcr.SimpleCredentials;
+import javax.jcr.Value;
+import javax.jcr.query.Query;
+import javax.jcr.query.QueryManager;
+import javax.jcr.query.QueryResult;
+
+import org.apache.jackrabbit.oak.Oak;
+import org.apache.jackrabbit.oak.jcr.Jcr;
+import org.apache.jackrabbit.oak.plugins.index.lucene.IndexTracker;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.DocumentQueue;
+import org.apache.jackrabbit.oak.run.cli.CommonOptions;
+import org.apache.jackrabbit.oak.run.cli.NodeStoreFixture;
+import org.apache.jackrabbit.oak.run.cli.NodeStoreFixtureProvider;
+import org.apache.jackrabbit.oak.run.cli.Options;
+import org.apache.jackrabbit.oak.run.commons.Command;
+import org.apache.jackrabbit.oak.spi.commit.Observer;
+import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.apache.jackrabbit.oak.stats.StatisticsProvider;
+
+import com.google.common.io.Closer;
+import com.google.common.util.concurrent.MoreExecutors;
+
+import joptsimple.OptionParser;
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+
+/**
+ * Scans and lists all references to nt:frozenNode and returns an exit code of 1 if any are found (0 otherwise).
+ * <p/>
+ * This variant does a *very expensive repository scan* for all properties formatted as uuid
+ * ( LIKE \"________-____-____-____-____________\" )
+ * and checking if any reference points to an nt:frozenNode (under /jcr:system/jcr:versionStorage
+ * at depth > 7).
+ * <p/>
+ * Note that any property with uuid that cannot be resolved will *not be reported*, as that
+ * is a legitimate use case of uuid property use. Only uuids that resolve will be analysed.
+ * <p/>
+ * Also note that this command has a few test parameters which can be used to "easily"
+ * create a property with a reference (or just a string for that matter too).
+ * <p/>
+ * Example:
+ * <pre>
+ * java -mx4g -jar oak-run-*.jar frozennoderefsbyscanning mongodb://localhost/<dbname> -user=admin -password=admin
+ * </pre>
+ * <p/>
+ * Test Example for creating a reference:
+ * <pre>
+ * java -mx4g -jar oak-run-*.jar frozennoderefsbyscanning mongodb://localhost/<dbname> -user=admin -password=admin -testCreateRefPath=<mypath> -testCreateRefProp=<mypropertyname> -testCreateRefUuid=<myuuid> -read-write=true
+ * </pre>
+ */
+public class FrozenNodeRefsByScanningCommand implements Command {
+
+ static {
+ // disable any query limits as our query is going to be a fulls can, and we are aware of it
Review comment:
```suggestion
// disable any query limits as our query is going to be a full scan, and we are aware of it
```
##########
File path: oak-run/src/main/java/org/apache/jackrabbit/oak/run/FrozenNodeRefsUsingIndexCommand.java
##########
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.run;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.commons.PathUtils;
+import org.apache.jackrabbit.oak.plugins.memory.GenericPropertyState;
+import org.apache.jackrabbit.oak.run.Utils.NodeStoreOptions;
+import org.apache.jackrabbit.oak.run.commons.Command;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
+import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+
+import com.google.common.io.Closer;
+
+/**
+ * Scans and lists all references to nt:frozenNode and returns an exit code of 1 if any are found (0 otherwise).
+ * <p/>
+ * This variant uses the /oak:index/reference by scanning through that list (only)
+ * and checking if any reference points to an nt:frozenNode (under /jcr:system/jcr:versionStorage
+ * at depth > 7).
+ * <p/>
+ * Example:
+ * <pre>
+ * java -mx4g -jar oak-run-*.jar frozennoderefsusingindex mongodb://localhost/<dbname>
+ * </pre>
+ */
+public class FrozenNodeRefsUsingIndexCommand implements Command {
+
+ public static final String NAME = "frozennoderefsusingindex";
+
+ @Override
+ public void execute(String... args) throws Exception {
+ String help = NAME + " {<path>|<mongo-uri>|<jdbc-uri>} [options]";
+ Utils.NodeStoreOptions nopts = new Utils.NodeStoreOptions(help).parse(args);
+ if (hasNtFrozenNodeReferences(nopts)) {
+ System.err.println("FAILURE: References (in /oak:index/references) to nt:frozenNode found.");
+ System.exit(1);
+ } else {
+ System.out.println("SUCCESS: No references (in /oak:index/references) to nt:frozenNode found.");
+ }
+ }
+
+ private static boolean hasNtFrozenNodeReferences(NodeStoreOptions nopts) throws IOException {
+ Closer closer = Utils.createCloserWithShutdownHook();
+ try {
+ NodeStore store = Utils.bootstrapNodeStore(nopts, closer);
+ NodeState root = store.getRoot();
+
+ NodeBuilder b = root.builder();
+ NodeBuilder content = b.child("content");
+ PropertyState refProp = GenericPropertyState.referenceProperty("myref", "dd31e965-7e74-4c04-9a64-6698ecb4c624");
+ content.setProperty(refProp);
+ store.merge(b, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ NodeState oakIndex = root.getChildNode("oak:index");
+ NodeState refIndex = oakIndex.getChildNode("reference");
+ NodeState uuidIndex = oakIndex.getChildNode("uuid");
+ NodeState uuids = uuidIndex.getChildNode(":index");
+ System.out.println("Scanning ... refindex = " + refIndex);
+ NodeState references = refIndex.getChildNode(":references");
+
+ List<FrozenNodeRef> list = new LinkedList<FrozenNodeRef>();
Review comment:
Same concern about memory as above.
##########
File path: oak-run/src/main/java/org/apache/jackrabbit/oak/run/FrozenNodeRefsByScanningCommand.java
##########
@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.run;
+
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+
+import javax.jcr.ItemNotFoundException;
+import javax.jcr.Node;
+import javax.jcr.NodeIterator;
+import javax.jcr.Property;
+import javax.jcr.PropertyIterator;
+import javax.jcr.PropertyType;
+import javax.jcr.Repository;
+import javax.jcr.RepositoryException;
+import javax.jcr.Session;
+import javax.jcr.SimpleCredentials;
+import javax.jcr.Value;
+import javax.jcr.query.Query;
+import javax.jcr.query.QueryManager;
+import javax.jcr.query.QueryResult;
+
+import org.apache.jackrabbit.oak.Oak;
+import org.apache.jackrabbit.oak.jcr.Jcr;
+import org.apache.jackrabbit.oak.plugins.index.lucene.IndexTracker;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.DocumentQueue;
+import org.apache.jackrabbit.oak.run.cli.CommonOptions;
+import org.apache.jackrabbit.oak.run.cli.NodeStoreFixture;
+import org.apache.jackrabbit.oak.run.cli.NodeStoreFixtureProvider;
+import org.apache.jackrabbit.oak.run.cli.Options;
+import org.apache.jackrabbit.oak.run.commons.Command;
+import org.apache.jackrabbit.oak.spi.commit.Observer;
+import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.apache.jackrabbit.oak.stats.StatisticsProvider;
+
+import com.google.common.io.Closer;
+import com.google.common.util.concurrent.MoreExecutors;
+
+import joptsimple.OptionParser;
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+
+/**
+ * Scans and lists all references to nt:frozenNode and returns an exit code of 1 if any are found (0 otherwise).
+ * <p/>
+ * This variant does a *very expensive repository scan* for all properties formatted as uuid
+ * ( LIKE \"________-____-____-____-____________\" )
+ * and checking if any reference points to an nt:frozenNode (under /jcr:system/jcr:versionStorage
+ * at depth > 7).
+ * <p/>
+ * Note that any property with uuid that cannot be resolved will *not be reported*, as that
+ * is a legitimate use case of uuid property use. Only uuids that resolve will be analysed.
+ * <p/>
+ * Also note that this command has a few test parameters which can be used to "easily"
+ * create a property with a reference (or just a string for that matter too).
+ * <p/>
+ * Example:
+ * <pre>
+ * java -mx4g -jar oak-run-*.jar frozennoderefsbyscanning mongodb://localhost/<dbname> -user=admin -password=admin
+ * </pre>
+ * <p/>
+ * Test Example for creating a reference:
+ * <pre>
+ * java -mx4g -jar oak-run-*.jar frozennoderefsbyscanning mongodb://localhost/<dbname> -user=admin -password=admin -testCreateRefPath=<mypath> -testCreateRefProp=<mypropertyname> -testCreateRefUuid=<myuuid> -read-write=true
+ * </pre>
+ */
+public class FrozenNodeRefsByScanningCommand implements Command {
+
+ static {
+ // disable any query limits as our query is going to be a fulls can, and we are aware of it
+ System.setProperty("oak.queryLimitReads", String.valueOf(Long.MAX_VALUE));
+
+ // // disable the WARN of the TraversingCursor
+ // LoggerContext c = (LoggerContext) LoggerFactory.getILoggerFactory();
+ // Logger logger = c.getLogger("org.apache.jackrabbit.oak.plugins.index.Cursors$TraversingCursor");
+ // logger.setLevel(Level.ERROR);
+ }
+
+ public static final String NAME = "frozennoderefsbyscanning";
+
+ private final String summary = "Scans and lists all references to nt:frozenNode";
+
+ @Override
+ public void execute(String... args) throws Exception {
+ OptionParser parser = new OptionParser();
+
+ Options opts = new Options();
+ opts.setCommandName(NAME);
+ opts.setSummary(summary);
+ opts.setConnectionString(CommonOptions.DEFAULT_CONNECTION_STRING);
+
+ OptionSpec<String> userOption = parser.accepts("user", "User name").withOptionalArg().defaultsTo("admin");
+ OptionSpec<String> passwordOption = parser.accepts("password", "Password").withOptionalArg().defaultsTo("admin");
+
+ OptionSpec<String> testCreateRefPathOption = parser.accepts("testCreateRefPath", "FOR TESTING ONLY: path where to create a reference from")
+ .withOptionalArg();
+ OptionSpec<String> testCreateRefPropOption = parser.accepts("testCreateRefProp", "FOR TESTING ONLY: property name for create a reference")
+ .withOptionalArg();
+ OptionSpec<String> testCreateRefTypeOption = parser
+ .accepts("testCreateRefType", "FOR TESTING ONLY: property type: 'reference' or anything else for a plain String").withOptionalArg();
+ OptionSpec<String> testCreateRefUuidOption = parser.accepts("testCreateRefUuid", "FOR TESTING ONLY: uuid to use as the reference")
+ .withOptionalArg();
+
+ OptionSet options = opts.parseAndConfigure(parser, args);
+
+ System.out.println("Opening nodestore...");
+ NodeStoreFixture nodeStoreFixture = NodeStoreFixtureProvider.create(opts);
+ System.out.println("Nodestore opened.");
+
+ if (uuidscan(userOption, passwordOption, testCreateRefPathOption, testCreateRefPropOption, testCreateRefTypeOption, testCreateRefUuidOption,
+ options, nodeStoreFixture)) {
+ System.err.println("FAILURE: References (in any uuid formatted property value) to nt:frozenNode found.");
+ System.exit(1);
+ } else {
+ System.out.println("SUCCESS: No references (in any uuid formatted property value) to nt:frozenNode found.");
+ }
+ }
+
+ private boolean uuidscan(OptionSpec<String> userOption, OptionSpec<String> passwordOption, OptionSpec<String> testCreateRefPathOption,
+ OptionSpec<String> testCreateRefPropOption, OptionSpec<String> testCreateRefTypeOption, OptionSpec<String> testCreateRefUuidOption,
+ OptionSet options, NodeStoreFixture nodeStoreFixture) throws IOException {
+ List<FrozenNodeRef> list = new LinkedList<FrozenNodeRef>();
+
+ NodeStore nodeStore = nodeStoreFixture.getStore();
+ String user = userOption.value(options);
+ String password = passwordOption.value(options);
+
+ String createRefPath = testCreateRefPathOption.value(options);
+ String createRefProp = testCreateRefPropOption.value(options);
+ String createRefType = testCreateRefTypeOption.value(options);
+ String createRefUuid = testCreateRefUuidOption.value(options);
+
+ Closer closer = Utils.createCloserWithShutdownHook();
+ closer.register(nodeStoreFixture);
+ try {
+
+ System.out.println("Logging in...");
+ Session session = openSession(nodeStore, "crx.default", user, password);
+
+ if (createRefPath != null && createRefProp != null && createRefUuid != null) {
+ // this part of the code is only executed if any of the -testXX properties
+ // are set - which is as the name suggests only for setting up a test.
+ System.out.println(
+ "Logged in, creating test reference: " + "path=" + createRefPath + ", property=" + createRefProp + ", uuid=" + createRefUuid);
+ Node n = session.getNode(createRefPath);
+ Value v;
+ if ("reference".equals(createRefType)) {
+ v = session.getValueFactory().createValue(createRefUuid, PropertyType.REFERENCE);
+ } else {
+ v = session.getValueFactory().createValue(createRefUuid, PropertyType.STRING);
+ }
+ n.setProperty(createRefProp, v);
+ session.save();
+ System.out.println("Created. Done.");
+ session.logout();
+ closer.close();
+ System.exit(0);
+ }
+
+ System.out.println("Logged in, querying...");
+ QueryManager qm = session.getWorkspace().getQueryManager();
+
+ // query for only getting 'Reference' and 'WeakReference' would be :
+ // SELECT * FROM [nt:base] AS p WHERE PROPERTY(*, 'Reference') IS NOT NULL OR PROPERTY(*, 'WeakReference') IS NOT NULL
+ Query q = qm.createQuery("SELECT * FROM [nt:base] AS p WHERE PROPERTY(*, '*') LIKE \"________-____-____-____-____________\"", "JCR-SQL2");
+ QueryResult qr = q.execute();
+ NodeIterator it = qr.getNodes();
+
+ while (it.hasNext()) {
+ Node n = it.nextNode();
+ PropertyIterator pit = n.getProperties();
+ while (pit.hasNext()) {
+ Property p = pit.nextProperty();
+ if ("jcr:uuid".equals(p.getName())) {
+ // jcr:uuid should be skipped as that's the identifier of a node, not a reference
+ continue;
+ }
+ if (!p.isMultiple()) {
+ String propValue = p.getValue().getString();
+ if (propValue.matches("........-....-....-....-............")) {
+ verify(list, session, n, p, propValue);
+ }
+ } else {
+ for (Value v : p.getValues()) {
+ String propValue = v.getString();
+ if (propValue.matches("........-....-....-....-............")) {
+ verify(list, session, n, p, propValue);
+ }
+ }
+ }
+ }
+ }
+
+ System.out.println("logout...");
+ session.logout();
+ System.out.println("done.");
+
+ return !list.isEmpty();
+ } catch (Throwable e) {
+ throw closer.rethrow(e);
+ } finally {
+ closer.close();
+ }
+ }
+
+ private void verify(List<FrozenNodeRef> list, Session session, Node n, Property p, String propValue) throws RepositoryException {
+ try {
+ Node node = session.getNodeByIdentifier(propValue);
+ String path = node.getPath();
+ boolean candidate = FrozenNodeRef.isFrozenNodeReferenceCandidate(path);
+ if (!candidate) {
+ return;
+ }
+ Property primaryType = node.getProperty("jcr:primaryType");
+ String primaryTypeValue = primaryType.getString();
+ boolean isNtFrozenNode = "nt:frozenNode".equals(primaryTypeValue);
+ if (!isNtFrozenNode) {
+ // this is where we ultimately have to continue out in any case - as only an nt:frozenNode
+ // is what we're interested in.
+ return;
+ }
+
+ String uuid = propValue;
+ String referrerPath = n.getPath();
+ String referrerProperty = p.getName();
+ FrozenNodeRef ref = new FrozenNodeRef(referrerPath, referrerProperty, PropertyType.nameFromValue(p.getType()), uuid, path);
+ list.add(ref);
Review comment:
Won't this consume too much memory when there are many references? The only use of the list seems to be to decide on the return of `uuidscan()`.
##########
File path: oak-run/src/main/java/org/apache/jackrabbit/oak/run/FrozenNodeRefsByScanningCommand.java
##########
@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.run;
+
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+
+import javax.jcr.ItemNotFoundException;
+import javax.jcr.Node;
+import javax.jcr.NodeIterator;
+import javax.jcr.Property;
+import javax.jcr.PropertyIterator;
+import javax.jcr.PropertyType;
+import javax.jcr.Repository;
+import javax.jcr.RepositoryException;
+import javax.jcr.Session;
+import javax.jcr.SimpleCredentials;
+import javax.jcr.Value;
+import javax.jcr.query.Query;
+import javax.jcr.query.QueryManager;
+import javax.jcr.query.QueryResult;
+
+import org.apache.jackrabbit.oak.Oak;
+import org.apache.jackrabbit.oak.jcr.Jcr;
+import org.apache.jackrabbit.oak.plugins.index.lucene.IndexTracker;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.DocumentQueue;
+import org.apache.jackrabbit.oak.run.cli.CommonOptions;
+import org.apache.jackrabbit.oak.run.cli.NodeStoreFixture;
+import org.apache.jackrabbit.oak.run.cli.NodeStoreFixtureProvider;
+import org.apache.jackrabbit.oak.run.cli.Options;
+import org.apache.jackrabbit.oak.run.commons.Command;
+import org.apache.jackrabbit.oak.spi.commit.Observer;
+import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.apache.jackrabbit.oak.stats.StatisticsProvider;
+
+import com.google.common.io.Closer;
+import com.google.common.util.concurrent.MoreExecutors;
+
+import joptsimple.OptionParser;
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+
+/**
+ * Scans and lists all references to nt:frozenNode and returns an exit code of 1 if any are found (0 otherwise).
+ * <p/>
+ * This variant does a *very expensive repository scan* for all properties formatted as uuid
+ * ( LIKE \"________-____-____-____-____________\" )
+ * and checking if any reference points to an nt:frozenNode (under /jcr:system/jcr:versionStorage
+ * at depth > 7).
+ * <p/>
+ * Note that any property with uuid that cannot be resolved will *not be reported*, as that
+ * is a legitimate use case of uuid property use. Only uuids that resolve will be analysed.
+ * <p/>
+ * Also note that this command has a few test parameters which can be used to "easily"
+ * create a property with a reference (or just a string for that matter too).
+ * <p/>
+ * Example:
+ * <pre>
+ * java -mx4g -jar oak-run-*.jar frozennoderefsbyscanning mongodb://localhost/<dbname> -user=admin -password=admin
+ * </pre>
+ * <p/>
+ * Test Example for creating a reference:
+ * <pre>
+ * java -mx4g -jar oak-run-*.jar frozennoderefsbyscanning mongodb://localhost/<dbname> -user=admin -password=admin -testCreateRefPath=<mypath> -testCreateRefProp=<mypropertyname> -testCreateRefUuid=<myuuid> -read-write=true
+ * </pre>
+ */
+public class FrozenNodeRefsByScanningCommand implements Command {
+
+ static {
+ // disable any query limits as our query is going to be a fulls can, and we are aware of it
+ System.setProperty("oak.queryLimitReads", String.valueOf(Long.MAX_VALUE));
+
+ // // disable the WARN of the TraversingCursor
+ // LoggerContext c = (LoggerContext) LoggerFactory.getILoggerFactory();
+ // Logger logger = c.getLogger("org.apache.jackrabbit.oak.plugins.index.Cursors$TraversingCursor");
+ // logger.setLevel(Level.ERROR);
+ }
+
+ public static final String NAME = "frozennoderefsbyscanning";
+
+ private final String summary = "Scans and lists all references to nt:frozenNode";
+
+ @Override
+ public void execute(String... args) throws Exception {
+ OptionParser parser = new OptionParser();
+
+ Options opts = new Options();
+ opts.setCommandName(NAME);
+ opts.setSummary(summary);
+ opts.setConnectionString(CommonOptions.DEFAULT_CONNECTION_STRING);
+
+ OptionSpec<String> userOption = parser.accepts("user", "User name").withOptionalArg().defaultsTo("admin");
+ OptionSpec<String> passwordOption = parser.accepts("password", "Password").withOptionalArg().defaultsTo("admin");
+
+ OptionSpec<String> testCreateRefPathOption = parser.accepts("testCreateRefPath", "FOR TESTING ONLY: path where to create a reference from")
+ .withOptionalArg();
+ OptionSpec<String> testCreateRefPropOption = parser.accepts("testCreateRefProp", "FOR TESTING ONLY: property name for create a reference")
+ .withOptionalArg();
+ OptionSpec<String> testCreateRefTypeOption = parser
+ .accepts("testCreateRefType", "FOR TESTING ONLY: property type: 'reference' or anything else for a plain String").withOptionalArg();
+ OptionSpec<String> testCreateRefUuidOption = parser.accepts("testCreateRefUuid", "FOR TESTING ONLY: uuid to use as the reference")
+ .withOptionalArg();
+
+ OptionSet options = opts.parseAndConfigure(parser, args);
+
+ System.out.println("Opening nodestore...");
+ NodeStoreFixture nodeStoreFixture = NodeStoreFixtureProvider.create(opts);
+ System.out.println("Nodestore opened.");
+
+ if (uuidscan(userOption, passwordOption, testCreateRefPathOption, testCreateRefPropOption, testCreateRefTypeOption, testCreateRefUuidOption,
+ options, nodeStoreFixture)) {
+ System.err.println("FAILURE: References (in any uuid formatted property value) to nt:frozenNode found.");
+ System.exit(1);
+ } else {
+ System.out.println("SUCCESS: No references (in any uuid formatted property value) to nt:frozenNode found.");
+ }
+ }
+
+ private boolean uuidscan(OptionSpec<String> userOption, OptionSpec<String> passwordOption, OptionSpec<String> testCreateRefPathOption,
+ OptionSpec<String> testCreateRefPropOption, OptionSpec<String> testCreateRefTypeOption, OptionSpec<String> testCreateRefUuidOption,
+ OptionSet options, NodeStoreFixture nodeStoreFixture) throws IOException {
+ List<FrozenNodeRef> list = new LinkedList<FrozenNodeRef>();
+
+ NodeStore nodeStore = nodeStoreFixture.getStore();
+ String user = userOption.value(options);
+ String password = passwordOption.value(options);
+
+ String createRefPath = testCreateRefPathOption.value(options);
+ String createRefProp = testCreateRefPropOption.value(options);
+ String createRefType = testCreateRefTypeOption.value(options);
+ String createRefUuid = testCreateRefUuidOption.value(options);
+
+ Closer closer = Utils.createCloserWithShutdownHook();
+ closer.register(nodeStoreFixture);
+ try {
+
+ System.out.println("Logging in...");
+ Session session = openSession(nodeStore, "crx.default", user, password);
+
+ if (createRefPath != null && createRefProp != null && createRefUuid != null) {
Review comment:
I would move this part into a dedicated test class.
##########
File path: oak-run/src/main/java/org/apache/jackrabbit/oak/run/FrozenNodeRefsByScanningCommand.java
##########
@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.run;
+
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+
+import javax.jcr.ItemNotFoundException;
+import javax.jcr.Node;
+import javax.jcr.NodeIterator;
+import javax.jcr.Property;
+import javax.jcr.PropertyIterator;
+import javax.jcr.PropertyType;
+import javax.jcr.Repository;
+import javax.jcr.RepositoryException;
+import javax.jcr.Session;
+import javax.jcr.SimpleCredentials;
+import javax.jcr.Value;
+import javax.jcr.query.Query;
+import javax.jcr.query.QueryManager;
+import javax.jcr.query.QueryResult;
+
+import org.apache.jackrabbit.oak.Oak;
+import org.apache.jackrabbit.oak.jcr.Jcr;
+import org.apache.jackrabbit.oak.plugins.index.lucene.IndexTracker;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.DocumentQueue;
+import org.apache.jackrabbit.oak.run.cli.CommonOptions;
+import org.apache.jackrabbit.oak.run.cli.NodeStoreFixture;
+import org.apache.jackrabbit.oak.run.cli.NodeStoreFixtureProvider;
+import org.apache.jackrabbit.oak.run.cli.Options;
+import org.apache.jackrabbit.oak.run.commons.Command;
+import org.apache.jackrabbit.oak.spi.commit.Observer;
+import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.apache.jackrabbit.oak.stats.StatisticsProvider;
+
+import com.google.common.io.Closer;
+import com.google.common.util.concurrent.MoreExecutors;
+
+import joptsimple.OptionParser;
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+
+/**
+ * Scans and lists all references to nt:frozenNode and returns an exit code of 1 if any are found (0 otherwise).
+ * <p/>
+ * This variant does a *very expensive repository scan* for all properties formatted as uuid
+ * ( LIKE \"________-____-____-____-____________\" )
+ * and checking if any reference points to an nt:frozenNode (under /jcr:system/jcr:versionStorage
+ * at depth > 7).
+ * <p/>
+ * Note that any property with uuid that cannot be resolved will *not be reported*, as that
+ * is a legitimate use case of uuid property use. Only uuids that resolve will be analysed.
+ * <p/>
+ * Also note that this command has a few test parameters which can be used to "easily"
+ * create a property with a reference (or just a string for that matter too).
+ * <p/>
+ * Example:
+ * <pre>
+ * java -mx4g -jar oak-run-*.jar frozennoderefsbyscanning mongodb://localhost/<dbname> -user=admin -password=admin
+ * </pre>
+ * <p/>
+ * Test Example for creating a reference:
+ * <pre>
+ * java -mx4g -jar oak-run-*.jar frozennoderefsbyscanning mongodb://localhost/<dbname> -user=admin -password=admin -testCreateRefPath=<mypath> -testCreateRefProp=<mypropertyname> -testCreateRefUuid=<myuuid> -read-write=true
+ * </pre>
+ */
+public class FrozenNodeRefsByScanningCommand implements Command {
+
+ static {
+ // disable any query limits as our query is going to be a fulls can, and we are aware of it
+ System.setProperty("oak.queryLimitReads", String.valueOf(Long.MAX_VALUE));
+
+ // // disable the WARN of the TraversingCursor
+ // LoggerContext c = (LoggerContext) LoggerFactory.getILoggerFactory();
+ // Logger logger = c.getLogger("org.apache.jackrabbit.oak.plugins.index.Cursors$TraversingCursor");
+ // logger.setLevel(Level.ERROR);
+ }
+
+ public static final String NAME = "frozennoderefsbyscanning";
+
+ private final String summary = "Scans and lists all references to nt:frozenNode";
Review comment:
```suggestion
private final String summary = "Scans repository and lists all references to nt:frozenNode";
```
##########
File path: oak-run/src/main/java/org/apache/jackrabbit/oak/run/FrozenNodeRefsUsingIndexCommand.java
##########
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.run;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.commons.PathUtils;
+import org.apache.jackrabbit.oak.plugins.memory.GenericPropertyState;
+import org.apache.jackrabbit.oak.run.Utils.NodeStoreOptions;
+import org.apache.jackrabbit.oak.run.commons.Command;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
+import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+
+import com.google.common.io.Closer;
+
+/**
+ * Scans and lists all references to nt:frozenNode and returns an exit code of 1 if any are found (0 otherwise).
+ * <p/>
+ * This variant uses the /oak:index/reference by scanning through that list (only)
+ * and checking if any reference points to an nt:frozenNode (under /jcr:system/jcr:versionStorage
+ * at depth > 7).
+ * <p/>
+ * Example:
+ * <pre>
+ * java -mx4g -jar oak-run-*.jar frozennoderefsusingindex mongodb://localhost/<dbname>
+ * </pre>
+ */
+public class FrozenNodeRefsUsingIndexCommand implements Command {
+
+ public static final String NAME = "frozennoderefsusingindex";
+
+ @Override
+ public void execute(String... args) throws Exception {
+ String help = NAME + " {<path>|<mongo-uri>|<jdbc-uri>} [options]";
+ Utils.NodeStoreOptions nopts = new Utils.NodeStoreOptions(help).parse(args);
+ if (hasNtFrozenNodeReferences(nopts)) {
+ System.err.println("FAILURE: References (in /oak:index/references) to nt:frozenNode found.");
+ System.exit(1);
+ } else {
+ System.out.println("SUCCESS: No references (in /oak:index/references) to nt:frozenNode found.");
+ }
+ }
+
+ private static boolean hasNtFrozenNodeReferences(NodeStoreOptions nopts) throws IOException {
+ Closer closer = Utils.createCloserWithShutdownHook();
+ try {
+ NodeStore store = Utils.bootstrapNodeStore(nopts, closer);
+ NodeState root = store.getRoot();
+
+ NodeBuilder b = root.builder();
Review comment:
Move to test class?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [jackrabbit-oak] stefan-egli commented on a change in pull request #238: OAK-9131 : introducing scanners for references to nt:frozenNode
Posted by GitBox <gi...@apache.org>.
stefan-egli commented on a change in pull request #238:
URL: https://github.com/apache/jackrabbit-oak/pull/238#discussion_r453710442
##########
File path: oak-run/src/main/java/org/apache/jackrabbit/oak/run/FrozenNodeRefsByScanningCommand.java
##########
@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.run;
+
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+
+import javax.jcr.ItemNotFoundException;
+import javax.jcr.Node;
+import javax.jcr.NodeIterator;
+import javax.jcr.Property;
+import javax.jcr.PropertyIterator;
+import javax.jcr.PropertyType;
+import javax.jcr.Repository;
+import javax.jcr.RepositoryException;
+import javax.jcr.Session;
+import javax.jcr.SimpleCredentials;
+import javax.jcr.Value;
+import javax.jcr.query.Query;
+import javax.jcr.query.QueryManager;
+import javax.jcr.query.QueryResult;
+
+import org.apache.jackrabbit.oak.Oak;
+import org.apache.jackrabbit.oak.jcr.Jcr;
+import org.apache.jackrabbit.oak.plugins.index.lucene.IndexTracker;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.DocumentQueue;
+import org.apache.jackrabbit.oak.run.cli.CommonOptions;
+import org.apache.jackrabbit.oak.run.cli.NodeStoreFixture;
+import org.apache.jackrabbit.oak.run.cli.NodeStoreFixtureProvider;
+import org.apache.jackrabbit.oak.run.cli.Options;
+import org.apache.jackrabbit.oak.run.commons.Command;
+import org.apache.jackrabbit.oak.spi.commit.Observer;
+import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.apache.jackrabbit.oak.stats.StatisticsProvider;
+
+import com.google.common.io.Closer;
+import com.google.common.util.concurrent.MoreExecutors;
+
+import joptsimple.OptionParser;
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+
+/**
+ * Scans and lists all references to nt:frozenNode and returns an exit code of 1 if any are found (0 otherwise).
+ * <p/>
+ * This variant does a *very expensive repository scan* for all properties formatted as uuid
+ * ( LIKE \"________-____-____-____-____________\" )
+ * and checking if any reference points to an nt:frozenNode (under /jcr:system/jcr:versionStorage
+ * at depth > 7).
+ * <p/>
+ * Note that any property with uuid that cannot be resolved will *not be reported*, as that
+ * is a legitimate use case of uuid property use. Only uuids that resolve will be analysed.
+ * <p/>
+ * Also note that this command has a few test parameters which can be used to "easily"
+ * create a property with a reference (or just a string for that matter too).
+ * <p/>
+ * Example:
+ * <pre>
+ * java -mx4g -jar oak-run-*.jar frozennoderefsbyscanning mongodb://localhost/<dbname> -user=admin -password=admin
+ * </pre>
+ * <p/>
+ * Test Example for creating a reference:
+ * <pre>
+ * java -mx4g -jar oak-run-*.jar frozennoderefsbyscanning mongodb://localhost/<dbname> -user=admin -password=admin -testCreateRefPath=<mypath> -testCreateRefProp=<mypropertyname> -testCreateRefUuid=<myuuid> -read-write=true
+ * </pre>
+ */
+public class FrozenNodeRefsByScanningCommand implements Command {
+
+ static {
+ // disable any query limits as our query is going to be a fulls can, and we are aware of it
Review comment:
thx! fixed
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org