You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2021/06/07 04:46:14 UTC
[iotdb] branch LogQueryMemoryControl created (now 55dc66c)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a change to branch LogQueryMemoryControl
in repository https://gitbox.apache.org/repos/asf/iotdb.git.
at 55dc66c add logs for memory control of query
This branch includes the following new commits:
new 55dc66c add logs for memory control of query
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: add logs for memory control of query
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch LogQueryMemoryControl
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 55dc66c99437d909a336d5b4bd7d41fbe9d87084
Author: JackieTien97 <Ja...@foxmail.com>
AuthorDate: Mon Jun 7 12:45:08 2021 +0800
add logs for memory control of query
---
.../db/query/control/QueryResourceManager.java | 35 ++++++++++++++++++++++
.../org/apache/iotdb/db/service/TSServiceImpl.java | 7 ++++-
2 files changed, 41 insertions(+), 1 deletion(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java b/server/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java
index e4491cd..5c96197 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java
@@ -122,6 +122,34 @@ public class QueryResourceManager {
return queryId;
}
+ /**
+ * Register a new query. When a query request is created firstly, this method must be invoked.
+ */
+ public long assignQueryId(boolean isDataQuery, int fetchSize, int deduplicatedPathNum,
+ String sql) {
+ long queryId = queryIdAtom.incrementAndGet();
+ if (isDataQuery) {
+ filePathsManager.addQueryId(queryId);
+ if (deduplicatedPathNum > 0) {
+ long estimatedMemoryUsage =
+ (long) deduplicatedPathNum * POINT_ESTIMATED_SIZE * (long) fetchSize;
+ // apply the memory successfully
+ if (totalFreeMemoryForRead.addAndGet(-estimatedMemoryUsage) >= 0) {
+ queryIdEstimatedMemoryMap.put(queryId, estimatedMemoryUsage);
+ logger.info(
+ "query id {}, sql is {}, apply the memory successfully, used {} memory, left {} memory",
+ queryId, sql, estimatedMemoryUsage, totalFreeMemoryForRead.get());
+ } else {
+ totalFreeMemoryForRead.addAndGet(estimatedMemoryUsage);
+ logger.info(
+ "query id {}, sql is {}, apply the memory failed, used {} memory, left {} memory",
+ queryId, sql, estimatedMemoryUsage, totalFreeMemoryForRead.get());
+ }
+ }
+ }
+ return queryId;
+ }
+
public Map<Long, Integer> getChunkNumMap() {
return chunkNumMap;
}
@@ -203,6 +231,13 @@ public class QueryResourceManager {
Long estimatedMemoryUsage = queryIdEstimatedMemoryMap.remove(queryId);
if (estimatedMemoryUsage != null) {
totalFreeMemoryForRead.addAndGet(estimatedMemoryUsage);
+ logger.info(
+ "query id {}, put back the memory successfully, memory size is {}, current memory is {}",
+ queryId, estimatedMemoryUsage, totalFreeMemoryForRead.get());
+ } else {
+ logger.info(
+ "query id {}, put back the memory failed, current memory is {}",
+ queryId, totalFreeMemoryForRead.get());
}
// remove usage of opened file paths of current thread
diff --git a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
index da62967..71677d5 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
@@ -696,7 +696,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
}
// generate the queryId for the operation
- queryId = generateQueryId(true, fetchSize, deduplicatedPathNum);
+ queryId = generateQueryId(true, fetchSize, deduplicatedPathNum, statement);
if (plan instanceof QueryPlan && config.isEnablePerformanceTracing()) {
if (!(plan instanceof AlignByDevicePlan)) {
TracingManager.getInstance()
@@ -1814,6 +1814,11 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
.assignQueryId(isDataQuery, fetchSize, deduplicatedPathNum);
}
+ private long generateQueryId(boolean isDataQuery, int fetchSize, int deduplicatedPathNum, String sql) {
+ return QueryResourceManager.getInstance()
+ .assignQueryId(isDataQuery, fetchSize, deduplicatedPathNum, sql);
+ }
+
protected List<TSDataType> getSeriesTypesByPaths(List<PartialPath> paths,
List<String> aggregations)
throws MetadataException {