You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by ma...@apache.org on 2025/03/25 07:29:52 UTC
(druid) branch master updated: Fix resource leak for GroupBy query merge buffer when query matched result cache (#17823)
This is an automated email from the ASF dual-hosted git repository.
maytasm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 9261b921f0d Fix resource leak for GroupBy query merge buffer when query matched result cache (#17823)
9261b921f0d is described below
commit 9261b921f0dfd3fdf02e0836220a29b5becb716e
Author: Maytas Monsereenusorn <ma...@apache.org>
AuthorDate: Tue Mar 25 00:29:45 2025 -0700
Fix resource leak for GroupBy query merge buffer when query matched result cache (#17823)
* Fix resource leak for GroupBy query merge buffer when match result cache
* Fix resource leak for GroupBy query merge buffer when match result cache
* Add test
* Add test
* Add comment
* Add test
---
.../druid/query/ResultLevelCachingQueryRunner.java | 2 +
.../query/ResultLevelCachingQueryRunnerTest.java | 78 +++++++++++++++++++++-
2 files changed, 79 insertions(+), 1 deletion(-)
diff --git a/server/src/main/java/org/apache/druid/query/ResultLevelCachingQueryRunner.java b/server/src/main/java/org/apache/druid/query/ResultLevelCachingQueryRunner.java
index 8cc6348a7c8..7e32e098a13 100644
--- a/server/src/main/java/org/apache/druid/query/ResultLevelCachingQueryRunner.java
+++ b/server/src/main/java/org/apache/druid/query/ResultLevelCachingQueryRunner.java
@@ -106,6 +106,8 @@ public class ResultLevelCachingQueryRunner<T> implements QueryRunner<T>
if (useResultCache && newResultSetId != null && newResultSetId.equals(existingResultSetId)) {
log.debug("Return cached result set as there is no change in identifiers for query %s ", query.getId());
+ // Call accumulate on the sequence to ensure that all Wrapper/Closer/Baggage/etc. get called
+ resultFromClient.accumulate(null, (accumulated, in) -> accumulated);
return deserializeResults(cachedResultSet, strategy, existingResultSetId);
} else {
@Nullable
diff --git a/server/src/test/java/org/apache/druid/query/ResultLevelCachingQueryRunnerTest.java b/server/src/test/java/org/apache/druid/query/ResultLevelCachingQueryRunnerTest.java
index 3cb4ae528e6..a0c14239e49 100644
--- a/server/src/test/java/org/apache/druid/query/ResultLevelCachingQueryRunnerTest.java
+++ b/server/src/test/java/org/apache/druid/query/ResultLevelCachingQueryRunnerTest.java
@@ -23,8 +23,13 @@ import org.apache.druid.client.SimpleServerView;
import org.apache.druid.client.cache.Cache;
import org.apache.druid.client.cache.CacheConfig;
import org.apache.druid.client.cache.MapCache;
+import org.apache.druid.collections.BlockingPool;
+import org.apache.druid.collections.DefaultBlockingPool;
+import org.apache.druid.collections.ReferenceCountingResourceHolder;
import org.apache.druid.java.util.common.Intervals;
import org.apache.druid.java.util.common.guava.Sequence;
+import org.apache.druid.java.util.common.guava.Sequences;
+import org.apache.druid.java.util.common.io.Closer;
import org.apache.druid.query.timeseries.TimeseriesResultValue;
import org.apache.druid.timeline.DataSegment;
import org.joda.time.Interval;
@@ -32,10 +37,16 @@ import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.ArgumentMatchers;
+import org.mockito.Mockito;
+import org.mockito.stubbing.Answer;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.util.List;
+import static org.junit.Assert.fail;
+
public class ResultLevelCachingQueryRunnerTest extends QueryRunnerBasedOnClusteredClientTestBase
{
private Cache cache;
@@ -252,7 +263,7 @@ public class ResultLevelCachingQueryRunnerTest extends QueryRunnerBasedOnCluster
);
try {
sequence.toList();
- Assert.fail("Expected to throw an exception");
+ fail("Expected to throw an exception");
}
catch (RuntimeException e) {
Assert.assertEquals("Exception for testing", e.getMessage());
@@ -264,6 +275,71 @@ public class ResultLevelCachingQueryRunnerTest extends QueryRunnerBasedOnCluster
}
}
+ @Test
+ public void testUseCacheAndReleaseResourceFromClient()
+ {
+ final BlockingPool<ByteBuffer> mergePool = new DefaultBlockingPool<>(() -> ByteBuffer.allocate(1), 1);
+ prepareCluster(10);
+ final Query<Result<TimeseriesResultValue>> query = timeseriesQuery(BASE_SCHEMA_INFO.getDataInterval());
+ CacheConfig cacheConfig = newCacheConfig(true, true, DEFAULT_CACHE_ENTRY_MAX_SIZE);
+ final QueryRunner<Result<TimeseriesResultValue>> baseRunner = cachingClusteredClient.getQueryRunnerForIntervals(query, query.getIntervals());
+ RetryQueryRunner<Result<TimeseriesResultValue>> spyRunner = Mockito.spy(new RetryQueryRunner<>(
+ baseRunner,
+ cachingClusteredClient::getQueryRunnerForSegments,
+ new RetryQueryRunnerConfig(),
+ objectMapper
+ ));
+ Mockito.doAnswer((Answer<Object>) invocation -> {
+ List<ReferenceCountingResourceHolder<ByteBuffer>> resoruce = mergePool.takeBatch(1, 1);
+ if (resoruce.isEmpty()) {
+ fail("Resource should not be empty");
+ }
+ Sequence<Result<TimeseriesResultValue>> realSequence = (Sequence<Result<TimeseriesResultValue>>) invocation.callRealMethod();
+ Closer closer = Closer.create();
+ closer.register(() -> resoruce.forEach(ReferenceCountingResourceHolder::close));
+ return Sequences.withBaggage(realSequence, closer);
+ }).when(spyRunner).run(ArgumentMatchers.any(), ArgumentMatchers.any());
+
+ final ResultLevelCachingQueryRunner<Result<TimeseriesResultValue>> queryRunner1 = new ResultLevelCachingQueryRunner<>(
+ spyRunner,
+ conglomerate.getToolChest(query),
+ query,
+ objectMapper,
+ cache,
+ cacheConfig
+ );
+
+ final Sequence<Result<TimeseriesResultValue>> sequence1 = queryRunner1.run(
+ QueryPlus.wrap(query),
+ responseContext()
+ );
+ final List<Result<TimeseriesResultValue>> results1 = sequence1.toList();
+ Assert.assertEquals(0, cache.getStats().getNumHits());
+ Assert.assertEquals(1, cache.getStats().getNumEntries());
+ Assert.assertEquals(1, cache.getStats().getNumMisses());
+
+
+ final Sequence<Result<TimeseriesResultValue>> sequence2 = queryRunner1.run(
+ QueryPlus.wrap(query),
+ responseContext()
+ );
+ final List<Result<TimeseriesResultValue>> results2 = sequence2.toList();
+ Assert.assertEquals(results1, results2);
+ Assert.assertEquals(1, cache.getStats().getNumHits());
+ Assert.assertEquals(1, cache.getStats().getNumEntries());
+ Assert.assertEquals(1, cache.getStats().getNumMisses());
+
+ final Sequence<Result<TimeseriesResultValue>> sequence3 = queryRunner1.run(
+ QueryPlus.wrap(query),
+ responseContext()
+ );
+ final List<Result<TimeseriesResultValue>> results3 = sequence3.toList();
+ Assert.assertEquals(results1, results3);
+ Assert.assertEquals(2, cache.getStats().getNumHits());
+ Assert.assertEquals(1, cache.getStats().getNumEntries());
+ Assert.assertEquals(1, cache.getStats().getNumMisses());
+ }
+
private <T> ResultLevelCachingQueryRunner<T> createQueryRunner(
CacheConfig cacheConfig,
Query<T> query
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org