You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by jo...@apache.org on 2024/11/18 11:20:40 UTC
(jackrabbit-oak) branch reapply-OAK1126-OAK11259 created (now 808d4f44ec)
This is an automated email from the ASF dual-hosted git repository.
joscorbe pushed a change to branch reapply-OAK1126-OAK11259
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
at 808d4f44ec Reapply "OAK-11260 Speed up Lucene cost estimation (#1858)"
This branch includes the following new commits:
new 6fa1a9e2a2 Reapply "OAK-11259: bump testcontainers to v1.20.3 / toxiproxy v2.11.0 (#1857)"
new 808d4f44ec Reapply "OAK-11260 Speed up Lucene cost estimation (#1858)"
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(jackrabbit-oak) 02/02: Reapply "OAK-11260 Speed up Lucene cost estimation (#1858)"
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joscorbe pushed a commit to branch reapply-OAK1126-OAK11259
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit 808d4f44ec42e40df7f21a29b719d1ae3f13ac5b
Author: Jose Cordero <co...@adobe.com>
AuthorDate: Mon Nov 18 12:19:47 2024 +0100
Reapply "OAK-11260 Speed up Lucene cost estimation (#1858)"
This reverts commit 34fbbff6f186ace4c9cbf25f08939a70c7a91368.
---
.../apache/jackrabbit/oak/commons/Profiler.java | 3 +
.../oak/query/SQL2OptimiseQueryTest.java | 29 +++++++
.../index/lucene/LuceneIndexLookupUtil.java | 14 ++++
.../oak/jcr/query/ManyIndexesAndUnionPlanTest.java | 96 ++++++++++++++++++++++
.../oak/plugins/index/search/IndexDefinition.java | 2 +-
5 files changed, 143 insertions(+), 1 deletion(-)
diff --git a/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/Profiler.java b/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/Profiler.java
index b1161a9fb9..bd5044d5d1 100644
--- a/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/Profiler.java
+++ b/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/Profiler.java
@@ -389,6 +389,9 @@ public class Profiler implements Runnable {
boolean packageCounts = false;
for (int j = 0, i = 0; i < dump.length && j < depth; i++) {
String el = dump[i].toString();
+ if (el.startsWith("app//")) {
+ el = el.substring("app//".length());
+ }
if (!el.equals(last) && !startsWithAny(el, ignoreLines)) {
last = el;
buff.append("at ").append(el).append(LINE_SEPARATOR);
diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/query/SQL2OptimiseQueryTest.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/query/SQL2OptimiseQueryTest.java
index c571cf01a4..f8685022ec 100644
--- a/oak-core/src/test/java/org/apache/jackrabbit/oak/query/SQL2OptimiseQueryTest.java
+++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/query/SQL2OptimiseQueryTest.java
@@ -35,6 +35,8 @@ import static org.junit.Assert.assertTrue;
import java.text.ParseException;
import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import javax.jcr.RepositoryException;
@@ -66,6 +68,33 @@ public class SQL2OptimiseQueryTest extends AbstractQueryTest {
}
};
+ @Test
+ public void limitUnionSize() throws ParseException {
+ String query = "SELECT * FROM [nt:base]\n"
+ + "WHERE (CONTAINS(*, '1') AND ([jcr:uuid] LIKE '2' OR [jcr:uuid] LIKE '3'))\n"
+ + " AND ((CONTAINS(*, '4') AND ([jcr:uuid] LIKE '5' OR [jcr:uuid] LIKE '6'))\n"
+ + " OR (CONTAINS(*, '0a') AND ([jcr:uuid] LIKE '0b' OR [jcr:uuid] LIKE '0c'))\n"
+ + " OR (CONTAINS(*, '1a') AND ([jcr:uuid] LIKE '1b' OR [jcr:uuid] LIKE '1c'))\n"
+ + " OR (CONTAINS(*, '2a') AND ([jcr:uuid] LIKE '2b' OR [jcr:uuid] LIKE '2c'))\n"
+ + " OR (CONTAINS(*, '3a') AND ([jcr:uuid] LIKE '3b' OR [jcr:uuid] LIKE '3c'))\n"
+ + " OR (CONTAINS(*, '4a') AND ([jcr:uuid] LIKE '4b' OR [jcr:uuid] LIKE '4c'))\n"
+ + " OR (CONTAINS(*, '5a') AND ([jcr:uuid] LIKE '5b' OR [jcr:uuid] LIKE '5c'))\n"
+ + " OR (CONTAINS(*, '6a') AND ([jcr:uuid] LIKE '6b' OR [jcr:uuid] LIKE '6c'))\n"
+ + " OR (CONTAINS(*, '7a') AND ([jcr:uuid] LIKE '7b' OR [jcr:uuid] LIKE '7c'))\n"
+ + " OR (CONTAINS(*, '8a') AND ([jcr:uuid] LIKE '8b' OR [jcr:uuid] LIKE '8c'))\n"
+ + " OR (CONTAINS(*, '9a') AND ([jcr:uuid] LIKE '9b' OR [jcr:uuid] LIKE '9c'))\n"
+ + " OR (CONTAINS(*, 'ea') AND ([jcr:uuid] LIKE 'eb' OR [jcr:uuid] LIKE 'ec')))\n"
+ + " AND ((CONTAINS(*, '10') AND ([jcr:uuid] LIKE '11' OR [jcr:uuid] LIKE '12'))\n"
+ + " OR (CONTAINS(*, '13') AND ([jcr:uuid] LIKE '14' OR [jcr:uuid] LIKE '15')))";
+ SQL2Parser parser = SQL2ParserTest.createTestSQL2Parser(
+ getMappings(), getNodeTypes(), qeSettings);
+ Query original;
+ original = parser.parse(query, false);
+ assertNotNull(original);
+ String alternative = original.buildAlternativeQuery().toString();
+ assertEquals(60825, alternative.length());
+ }
+
/**
* checks the {@code Query#optimise()} calls for the conversion from OR to UNION from a query
* POV; ensuring that it returns always the same, expected resultset.
diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexLookupUtil.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexLookupUtil.java
index 24c5fc9f5d..f0c2dd82a8 100644
--- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexLookupUtil.java
+++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexLookupUtil.java
@@ -27,11 +27,17 @@ import org.apache.jackrabbit.oak.plugins.index.search.IndexFormatVersion;
import org.apache.jackrabbit.oak.plugins.index.search.IndexLookup;
import org.apache.jackrabbit.oak.spi.query.Filter;
import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.state.NodeStateUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_PROPERTY_NAME;
import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.TYPE_LUCENE;
class LuceneIndexLookupUtil {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LuceneIndexLookupUtil.class);
+
static final Predicate<NodeState> LUCENE_INDEX_DEFINITION_PREDICATE =
state -> TYPE_LUCENE.equals(state.getString(TYPE_PROPERTY_NAME));
@@ -45,6 +51,14 @@ class LuceneIndexLookupUtil {
public static String getOldFullTextIndexPath(NodeState root, Filter filter, IndexTracker tracker) {
Collection<String> indexPaths = getLuceneIndexLookup(root).collectIndexNodePaths(filter, false);
for (String path : indexPaths) {
+ NodeState node = NodeStateUtils.getNode(root, path);
+ if (IndexDefinition.determineIndexFormatVersion(node) != IndexFormatVersion.V1) {
+ // shortcut to avoid reading the index definition if not needed
+ continue;
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Old V1 lucene index found at {}", path);
+ }
IndexDefinition indexDefinition = tracker.getIndexDefinition(path);
if (indexDefinition != null && indexDefinition.isFullTextEnabled()
&& indexDefinition.getVersion() == IndexFormatVersion.V1) {
diff --git a/oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/query/ManyIndexesAndUnionPlanTest.java b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/query/ManyIndexesAndUnionPlanTest.java
new file mode 100644
index 0000000000..4c6751bb46
--- /dev/null
+++ b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/query/ManyIndexesAndUnionPlanTest.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.jcr.query;
+
+import javax.jcr.Node;
+import javax.jcr.NodeIterator;
+import javax.jcr.RepositoryException;
+import javax.jcr.Session;
+import javax.jcr.query.Query;
+import javax.jcr.query.QueryManager;
+import javax.jcr.query.Row;
+import javax.jcr.query.RowIterator;
+
+import org.apache.jackrabbit.core.query.AbstractQueryTest;
+import org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants;
+import org.apache.jackrabbit.oak.plugins.index.search.IndexFormatVersion;
+import org.junit.Test;
+
+public class ManyIndexesAndUnionPlanTest extends AbstractQueryTest {
+
+ @Test
+ public void testResultSize() throws Exception {
+ createIndexes();
+ createData();
+ doTestResultSize(10);
+ }
+
+ private void createIndexes() throws RepositoryException, InterruptedException {
+ Session session = superuser;
+ Node index = session.getRootNode().getNode("oak:index");
+ for (int i = 0; i < 10; i++) {
+ Node lucene = index.addNode("lucene" + i, "oak:QueryIndexDefinition");
+ lucene.setProperty("type", "lucene");
+ lucene.setProperty("async", "async");
+ lucene.setProperty(FulltextIndexConstants.COMPAT_MODE, IndexFormatVersion.V1.getVersion());
+ }
+ session.save();
+ }
+
+ private void createData() throws RepositoryException {
+ Session session = superuser;
+ for (int i = 0; i < 10; i++) {
+ Node n = testRootNode.addNode("node" + i);
+ n.setProperty("text", "Hello World");
+ }
+ session.save();
+ }
+
+ private void doTestResultSize(int expected) throws RepositoryException {
+ Session session = superuser;
+ QueryManager qm = session.getWorkspace().getQueryManager();
+
+ String xpath;
+ xpath = "/jcr:root//*[jcr:contains(@text, 'Hello World')]";
+
+ Query q;
+ long result;
+ NodeIterator it;
+ StringBuilder buff;
+
+ q = qm.createQuery(xpath, "xpath");
+ it = q.execute().getNodes();
+ result = it.getSize();
+ assertTrue("size: " + result + " expected around " + expected,
+ result > expected - 50 &&
+ result < expected + 50);
+ buff = new StringBuilder();
+ while (it.hasNext()) {
+ Node n = it.nextNode();
+ buff.append(n.getPath()).append('\n');
+ }
+ for (int j = 0; j < 1; j++) {
+ for (int i = 0; i < 1; i++) {
+ q = qm.createQuery("explain " + xpath, "xpath");
+ RowIterator rit = q.execute().getRows();
+ Row r = rit.nextRow();
+ assertTrue(r.toString().indexOf("luceneGlobal") >= 0);
+ }
+ }
+ }
+
+}
\ No newline at end of file
diff --git a/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java b/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java
index 3accb696d5..061733d819 100644
--- a/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java
+++ b/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java
@@ -1829,7 +1829,7 @@ public class IndexDefinition implements Aggregate.AggregateMapper {
nb.setProperty(JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME);
}
- protected static IndexFormatVersion determineIndexFormatVersion(NodeState defn) {
+ public static IndexFormatVersion determineIndexFormatVersion(NodeState defn) {
//Compat mode version if specified has highest priority
if (defn.hasProperty(COMPAT_MODE)) {
return versionFrom(defn.getProperty(COMPAT_MODE));
(jackrabbit-oak) 01/02: Reapply "OAK-11259: bump testcontainers to v1.20.3 / toxiproxy v2.11.0 (#1857)"
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joscorbe pushed a commit to branch reapply-OAK1126-OAK11259
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit 6fa1a9e2a279b405af261fd6e4a2558b7059e026
Author: Jose Cordero <co...@adobe.com>
AuthorDate: Mon Nov 18 12:19:37 2024 +0100
Reapply "OAK-11259: bump testcontainers to v1.20.3 / toxiproxy v2.11.0 (#1857)"
This reverts commit 1e465c0ba8b8e68dbf9c85cfcac120c353d9cbf2.
---
oak-parent/pom.xml | 2 +-
.../document/flatfile/pipelined/PipelinedMongoConnectionFailureIT.java | 2 +-
.../jackrabbit/oak/plugins/index/elastic/ElasticReliabilityTest.java | 2 +-
.../oak/plugins/document/mongo/LeaseUpdateSocketTimeoutIT.java | 2 +-
4 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/oak-parent/pom.xml b/oak-parent/pom.xml
index 188a20c41d..95de424c72 100644
--- a/oak-parent/pom.xml
+++ b/oak-parent/pom.xml
@@ -66,7 +66,7 @@
<tika.version>1.28.5</tika.version>
<derby.version>10.15.2.0</derby.version>
<jackson.version>2.17.2</jackson.version>
- <testcontainers.version>1.20.1</testcontainers.version>
+ <testcontainers.version>1.20.3</testcontainers.version>
<pax-exam.version>4.13.1</pax-exam.version>
<groovy.version>3.0.22</groovy.version>
<!-- determines the bytecode version (i.e. the minimum JRE required to run the build artifact) -->
diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMongoConnectionFailureIT.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMongoConnectionFailureIT.java
index 9e0a554d2f..b0eab4972b 100644
--- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMongoConnectionFailureIT.java
+++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMongoConnectionFailureIT.java
@@ -72,7 +72,7 @@ import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipeline
public class PipelinedMongoConnectionFailureIT {
private static final Logger LOG = LoggerFactory.getLogger(PipelinedMongoConnectionFailureIT.class);
- private static final DockerImageName TOXIPROXY_IMAGE = DockerImageName.parse("ghcr.io/shopify/toxiproxy:2.9.0");
+ private static final DockerImageName TOXIPROXY_IMAGE = DockerImageName.parse("ghcr.io/shopify/toxiproxy:2.11.0");
// We cannot use the MongoDockerRule/MongoConnectionFactory because they don't allow customizing the docker network
// used to launch the Mongo container.
private static final int MONGODB_DEFAULT_PORT = 27017;
diff --git a/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticReliabilityTest.java b/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticReliabilityTest.java
index b6b66b8893..714734af79 100644
--- a/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticReliabilityTest.java
+++ b/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticReliabilityTest.java
@@ -49,7 +49,7 @@ public class ElasticReliabilityTest extends ElasticAbstractQueryTest {
@Rule
public final RestoreSystemProperties restoreSystemProperties = new RestoreSystemProperties();
- private static final DockerImageName TOXIPROXY_IMAGE = DockerImageName.parse("ghcr.io/shopify/toxiproxy:2.9.0");
+ private static final DockerImageName TOXIPROXY_IMAGE = DockerImageName.parse("ghcr.io/shopify/toxiproxy:2.11.0");
private ToxiproxyContainer toxiproxy;
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/LeaseUpdateSocketTimeoutIT.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/LeaseUpdateSocketTimeoutIT.java
index 8807dc90ef..d8b5477070 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/LeaseUpdateSocketTimeoutIT.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/LeaseUpdateSocketTimeoutIT.java
@@ -53,7 +53,7 @@ import static org.junit.Assume.assumeTrue;
public class LeaseUpdateSocketTimeoutIT {
- private static final DockerImageName TOXIPROXY_IMAGE = DockerImageName.parse("ghcr.io/shopify/toxiproxy:2.9.0");
+ private static final DockerImageName TOXIPROXY_IMAGE = DockerImageName.parse("ghcr.io/shopify/toxiproxy:2.11.0");
private static final int MONGODB_DEFAULT_PORT = 27017;