You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by mk...@apache.org on 2022/01/05 14:12:30 UTC
[jackrabbit-oak] branch trunk updated: OAK-9651: Protection against very large queries (#446)
This is an automated email from the ASF dual-hosted git repository.
mkataria pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new 6703be8 OAK-9651: Protection against very large queries (#446)
6703be8 is described below
commit 6703be860c863ddb96eae01eea5e220710b2ab58
Author: Mohit Kataria <mk...@apache.org>
AuthorDate: Wed Jan 5 19:42:25 2022 +0530
OAK-9651: Protection against very large queries (#446)
* OAK-9651: Protection against very large queries
---
.../jackrabbit/oak/query/QueryEngineImpl.java | 9 +-
.../jackrabbit/oak/query/QueryEngineSettings.java | 16 +++
.../oak/query/stats/QueryStatsMBeanImpl.java | 11 +-
.../jackrabbit/oak/query/QueryLimitTest.java | 111 +++++++++++++++++++++
.../jackrabbit/oak/query/stats/QueryStatsTest.java | 38 +++++++
5 files changed, 183 insertions(+), 2 deletions(-)
diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryEngineImpl.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryEngineImpl.java
index c9bcb18..2a4e1a1 100644
--- a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryEngineImpl.java
+++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryEngineImpl.java
@@ -151,12 +151,19 @@ public abstract class QueryEngineImpl implements QueryEngine {
} else {
LOG.debug("Parsing {} statement: {}", language, statement);
}
+ QueryEngineSettings settings = context.getSettings();
+ if (statement.length() > (settings.getQueryLengthErrorLimit())){
+ LOG.error("Too large query: " + statement);
+ throw new ParseException("Query length "+ statement.length() + " is larger than max supported query length: " + settings.getQueryLengthErrorLimit(), 0);
+ }
+ if (statement.length() > (settings.getQueryLengthWarnLimit())){
+ LOG.warn("Query length {} breached queryWarnLimit {}. Query: {}", statement.length(), settings.getQueryLengthWarnLimit(), statement);
+ }
NamePathMapper mapper = new NamePathMapperImpl(
new LocalNameMapper(context.getRoot(), mappings));
NodeTypeInfoProvider nodeTypes = context.getNodeTypeInfoProvider();
- QueryEngineSettings settings = context.getSettings();
settings.getQueryValidator().checkStatement(statement);
QueryExecutionStats stats = settings.getQueryStatsReporter().getQueryExecution(statement, language);
diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryEngineSettings.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryEngineSettings.java
index 119488c..c7494ae 100644
--- a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryEngineSettings.java
+++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryEngineSettings.java
@@ -93,6 +93,22 @@ public class QueryEngineSettings implements QueryEngineSettingsMBean, QueryLimit
private String[] classNamesIgnoredInCallTrace = new String[] {};
+
+ private static final String OAK_QUERY_LENGTH_WARN_LIMIT = "oak.query.length.warn.limit";
+ private static final String OAK_QUERY_LENGTH_ERROR_LIMIT = "oak.query.length.error.limit";
+
+ private final long queryLengthWarnLimit = Long.getLong(OAK_QUERY_LENGTH_WARN_LIMIT, 1024 * 1024); // 1 MB
+ private final long queryLengthErrorLimit = Long.getLong(OAK_QUERY_LENGTH_ERROR_LIMIT, 100 * 1024 * 1024); //100MB
+
+
+ public long getQueryLengthWarnLimit() {
+ return queryLengthWarnLimit;
+ }
+
+ public long getQueryLengthErrorLimit() {
+ return queryLengthErrorLimit;
+ }
+
public QueryEngineSettings() {
statisticsProvider = StatisticsProvider.NOOP;
}
diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/stats/QueryStatsMBeanImpl.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/stats/QueryStatsMBeanImpl.java
index 781b8ff..51b3ae1 100644
--- a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/stats/QueryStatsMBeanImpl.java
+++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/stats/QueryStatsMBeanImpl.java
@@ -42,10 +42,12 @@ public class QueryStatsMBeanImpl extends AnnotatedStandardMBean
private final Logger log = LoggerFactory.getLogger(getClass());
private final int SLOW_QUERY_LIMIT_SCANNED =
Integer.getInteger("oak.query.slowScanLimit", 100000);
- private final int MAX_STATS_DATA =
+ private final int MAX_STATS_DATA =
Integer.getInteger("oak.query.stats", 5000);
private final int MAX_POPULAR_QUERIES =
Integer.getInteger("oak.query.slowLimit", 100);
+ private final int MAX_QUERY_SIZE =
+ Integer.getInteger("oak.query.maxQuerySize", 2048);
private final ConcurrentSkipListMap<String, QueryStatsData> statistics =
new ConcurrentSkipListMap<String, QueryStatsData>();
private final QueryEngineSettings settings;
@@ -134,6 +136,13 @@ public class QueryStatsMBeanImpl extends AnnotatedStandardMBean
if (statistics.size() > 2 * MAX_STATS_DATA) {
evict();
}
+ if (statement.length() > MAX_QUERY_SIZE) {
+ statement = new StringBuilder().append("Truncated query: ")
+ .append(statement.substring(0, MAX_QUERY_SIZE >> 1))
+ .append(" ...... ")
+ .append(statement.substring(statement.length() - (MAX_QUERY_SIZE >> 1)))
+ .toString();
+ }
QueryStatsData stats = new QueryStatsData(statement, language);
QueryStatsData s2 = statistics.putIfAbsent(stats.getKey(), stats);
if (s2 != null) {
diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/query/QueryLimitTest.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/query/QueryLimitTest.java
new file mode 100644
index 0000000..70349da
--- /dev/null
+++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/query/QueryLimitTest.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jackrabbit.oak.query;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.jackrabbit.oak.InitialContent;
+import org.apache.jackrabbit.oak.Oak;
+import org.apache.jackrabbit.oak.api.ContentRepository;
+import org.apache.jackrabbit.oak.api.QueryEngine;
+import org.apache.jackrabbit.oak.commons.junit.LogCustomizer;
+import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore;
+import org.apache.jackrabbit.oak.spi.security.OpenSecurityProvider;
+import org.hamcrest.core.IsCollectionContaining;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.event.Level;
+
+import java.text.ParseException;
+import java.util.Properties;
+
+import static org.junit.Assert.assertThrows;
+
+public class QueryLimitTest extends AbstractQueryTest {
+
+ MemoryNodeStore store;
+ QueryEngineSettings qeSettings;
+ private Properties systemProperties;
+ int queryLengthWarnLimit = 500;
+ int queryLengthErrorLimit = 1000;
+
+ @Override
+ protected ContentRepository createRepository() {
+ store = new MemoryNodeStore();
+ qeSettings = new QueryEngineSettings();
+
+ return new Oak(store)
+ .with(new OpenSecurityProvider())
+ .with(new InitialContent())
+ .with(qeSettings)
+ .createContentRepository();
+ }
+
+ @Before
+ @Override
+ public void before() throws Exception {
+ systemProperties = (Properties) System.getProperties().clone();
+ System.setProperty("oak.query.length.warn.limit", "" + queryLengthWarnLimit);
+ System.setProperty("oak.query.length.error.limit", "" + queryLengthErrorLimit);
+ super.before();
+ }
+
+ @After
+ public void after() throws Exception {
+ System.setProperties(systemProperties);
+ }
+
+ @Test
+ public void queryLengthErrorLimitBreachThrowsException() throws Exception {
+ String generatedString = RandomStringUtils.random(queryLengthErrorLimit, true, false);
+
+ String query = "SELECT [jcr:path] FROM [nt:base] AS a WHERE a.[x]='" + generatedString + "'";
+ assertThrows(ParseException.class,
+ () -> {
+ try {
+ qe.executeQuery(query, QueryEngineImpl.SQL2, 10, 0,
+ QueryEngine.NO_BINDINGS, QueryEngine.NO_MAPPINGS);
+ } catch (RuntimeException e) {
+ Assert.assertEquals("Query length " + query.length() + " is larger than max supported query length: " + queryLengthErrorLimit, e.getMessage());
+ throw e;
+ }
+ });
+ }
+
+ @Test
+ public void queryLengthWarnLimitBreachLogsWarning() throws Exception {
+ String generatedString = RandomStringUtils.random(queryLengthWarnLimit, true, false);
+
+ LogCustomizer customLogs = LogCustomizer.forLogger(QueryEngineImpl.class.getName()).enable(Level.WARN).create();
+
+ try {
+ customLogs.starting();
+ String query = "SELECT [jcr:path] FROM [nt:base] AS a WHERE a.[x]='" + generatedString + "'";
+
+ qe.executeQuery(query, QueryEngineImpl.SQL2, 10, 0,
+ QueryEngine.NO_BINDINGS, QueryEngine.NO_MAPPINGS);
+
+ String expectedLogMessage = "Query length " + query.length() + " breached queryWarnLimit " + queryLengthWarnLimit + ". Query: " + query;
+
+ Assert.assertThat(customLogs.getLogs(), IsCollectionContaining.hasItems(expectedLogMessage));
+ } finally {
+ customLogs.finished();
+ }
+ }
+}
diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/query/stats/QueryStatsTest.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/query/stats/QueryStatsTest.java
index df5392c..9cd2d99 100644
--- a/oak-core/src/test/java/org/apache/jackrabbit/oak/query/stats/QueryStatsTest.java
+++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/query/stats/QueryStatsTest.java
@@ -17,13 +17,33 @@
package org.apache.jackrabbit.oak.query.stats;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.jackrabbit.oak.query.QueryEngineSettings;
+import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
+import java.util.Properties;
+
public class QueryStatsTest {
+ private Properties systemProperties;
+ private int maxQuerySize = 2000;
+
+ @Before
+ public void setup() {
+ systemProperties =(Properties) System.getProperties().clone();
+ System.setProperty("oak.query.maxQuerySize", "" + maxQuerySize);
+ }
+
+ @After
+ public void teardown() {
+ System.setProperties(systemProperties);
+ }
+
@Test
public void testEviction() throws InterruptedException {
QueryStatsMBeanImpl bean = new QueryStatsMBeanImpl(new QueryEngineSettings());
@@ -56,4 +76,22 @@ public class QueryStatsTest {
}
assertTrue(json.indexOf("old") < 0);
}
+
+ @Test
+ public void testTruncation() throws InterruptedException {
+ QueryEngineSettings qes = new QueryEngineSettings();
+ QueryStatsMBeanImpl bean = new QueryStatsMBeanImpl(qes);
+ String generatedString = RandomStringUtils.random(5000, true, false);
+ bean.getQueryExecution(generatedString, "");
+
+ String data = bean.getPopularQueries().values().iterator().next().toString();
+ assertFalse(data.contains(generatedString));
+ String statLog = new StringBuilder()
+ .append("Truncated query: ")
+ .append(generatedString.substring(0, maxQuerySize >> 1))
+ .append(" ...... ")
+ .append(generatedString.substring(generatedString.length() - (maxQuerySize >> 1)))
+ .toString();
+ assertTrue(data.contains(statLog));
+ }
}