You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2021/06/07 04:46:15 UTC
[iotdb] 01/01: add logs for memory control of query
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch LogQueryMemoryControl
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 55dc66c99437d909a336d5b4bd7d41fbe9d87084
Author: JackieTien97 <Ja...@foxmail.com>
AuthorDate: Mon Jun 7 12:45:08 2021 +0800
add logs for memory control of query
---
.../db/query/control/QueryResourceManager.java | 35 ++++++++++++++++++++++
.../org/apache/iotdb/db/service/TSServiceImpl.java | 7 ++++-
2 files changed, 41 insertions(+), 1 deletion(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java b/server/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java
index e4491cd..5c96197 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java
@@ -122,6 +122,34 @@ public class QueryResourceManager {
return queryId;
}
+ /**
+ * Register a new query. When a query request is created firstly, this method must be invoked.
+ */
+ public long assignQueryId(boolean isDataQuery, int fetchSize, int deduplicatedPathNum,
+ String sql) {
+ long queryId = queryIdAtom.incrementAndGet();
+ if (isDataQuery) {
+ filePathsManager.addQueryId(queryId);
+ if (deduplicatedPathNum > 0) {
+ long estimatedMemoryUsage =
+ (long) deduplicatedPathNum * POINT_ESTIMATED_SIZE * (long) fetchSize;
+ // apply the memory successfully
+ if (totalFreeMemoryForRead.addAndGet(-estimatedMemoryUsage) >= 0) {
+ queryIdEstimatedMemoryMap.put(queryId, estimatedMemoryUsage);
+ logger.info(
+ "query id {}, sql is {}, apply the memory successfully, used {} memory, left {} memory",
+ queryId, sql, estimatedMemoryUsage, totalFreeMemoryForRead.get());
+ } else {
+ totalFreeMemoryForRead.addAndGet(estimatedMemoryUsage);
+ logger.info(
+ "query id {}, sql is {}, apply the memory failed, used {} memory, left {} memory",
+ queryId, sql, estimatedMemoryUsage, totalFreeMemoryForRead.get());
+ }
+ }
+ }
+ return queryId;
+ }
+
public Map<Long, Integer> getChunkNumMap() {
return chunkNumMap;
}
@@ -203,6 +231,13 @@ public class QueryResourceManager {
Long estimatedMemoryUsage = queryIdEstimatedMemoryMap.remove(queryId);
if (estimatedMemoryUsage != null) {
totalFreeMemoryForRead.addAndGet(estimatedMemoryUsage);
+ logger.info(
+ "query id {}, put back the memory successfully, memory size is {}, current memory is {}",
+ queryId, estimatedMemoryUsage, totalFreeMemoryForRead.get());
+ } else {
+ logger.info(
+ "query id {}, put back the memory failed, current memory is {}",
+ queryId, totalFreeMemoryForRead.get());
}
// remove usage of opened file paths of current thread
diff --git a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
index da62967..71677d5 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
@@ -696,7 +696,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
}
// generate the queryId for the operation
- queryId = generateQueryId(true, fetchSize, deduplicatedPathNum);
+ queryId = generateQueryId(true, fetchSize, deduplicatedPathNum, statement);
if (plan instanceof QueryPlan && config.isEnablePerformanceTracing()) {
if (!(plan instanceof AlignByDevicePlan)) {
TracingManager.getInstance()
@@ -1814,6 +1814,11 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
.assignQueryId(isDataQuery, fetchSize, deduplicatedPathNum);
}
+ private long generateQueryId(boolean isDataQuery, int fetchSize, int deduplicatedPathNum, String sql) {
+ return QueryResourceManager.getInstance()
+ .assignQueryId(isDataQuery, fetchSize, deduplicatedPathNum, sql);
+ }
+
protected List<TSDataType> getSeriesTypesByPaths(List<PartialPath> paths,
List<String> aggregations)
throws MetadataException {