You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2021/07/12 15:46:33 UTC
[iotdb] branch to0.12removeQMC created (now d27fe2c)
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a change to branch to0.12removeQMC
in repository https://gitbox.apache.org/repos/asf/iotdb.git.
at d27fe2c [To rel/0.12] remove query memory control
This branch includes the following new commits:
new d27fe2c [To rel/0.12] remove query memory control
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: [To rel/0.12] remove query memory control
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a commit to branch to0.12removeQMC
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit d27fe2c4ccabe14529f4e98c73c84e6f8a47e0d5
Author: Alima777 <wx...@gmail.com>
AuthorDate: Mon Jul 12 22:33:44 2021 +0800
[To rel/0.12] remove query memory control
---
.../apache/iotdb/cluster/query/ClusterPlanner.java | 13 +----
.../iotdb/cluster/query/LocalQueryExecutor.java | 32 +++--------
.../cluster/query/manage/ClusterQueryManager.java | 8 +--
.../org/apache/iotdb/cluster/common/IoTDBTest.java | 3 +-
.../query/ClusterAggregateExecutorTest.java | 4 +-
.../query/ClusterDataQueryExecutorTest.java | 10 ++--
.../cluster/query/ClusterFillExecutorTest.java | 4 +-
.../cluster/query/ClusterPlanExecutorTest.java | 4 +-
.../cluster/query/ClusterQueryRouterTest.java | 12 ++--
.../ClusterGroupByNoVFilterDataSetTest.java | 2 +-
.../groupby/ClusterGroupByVFilterDataSetTest.java | 2 +-
.../query/groupby/MergeGroupByExecutorTest.java | 4 +-
.../query/groupby/RemoteGroupByExecutorTest.java | 4 +-
.../query/manage/ClusterQueryManagerTest.java | 12 ++--
.../query/reader/ClusterReaderFactoryTest.java | 2 +-
.../query/reader/ClusterTimeGeneratorTest.java | 4 +-
.../cluster/server/member/DataGroupMemberTest.java | 2 +-
.../cluster/server/member/MetaGroupMemberTest.java | 4 +-
.../exception/query/PathNumOverLimitException.java | 9 +--
.../org/apache/iotdb/db/monitor/StatMonitor.java | 2 +-
.../main/java/org/apache/iotdb/db/qp/Planner.java | 31 ++---------
.../qp/strategy/optimizer/ConcatPathOptimizer.java | 18 +++---
.../qp/strategy/optimizer/ILogicalOptimizer.java | 3 +-
.../db/query/control/QueryResourceManager.java | 48 +---------------
.../org/apache/iotdb/db/service/TSServiceImpl.java | 65 ++--------------------
.../db/integration/IoTDBSequenceDataQueryIT.java | 9 +--
.../iotdb/db/integration/IoTDBSeriesReaderIT.java | 11 ++--
.../iotdb/db/qp/logical/LogicalPlanSmallTest.java | 2 +-
.../apache/iotdb/db/utils/EnvironmentUtils.java | 9 ++-
29 files changed, 77 insertions(+), 256 deletions(-)
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/query/ClusterPlanner.java b/cluster/src/main/java/org/apache/iotdb/cluster/query/ClusterPlanner.java
index ecbfb74..90fd0be 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/query/ClusterPlanner.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/query/ClusterPlanner.java
@@ -22,11 +22,9 @@ package org.apache.iotdb.cluster.query;
import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.qp.Planner;
import org.apache.iotdb.db.qp.logical.Operator;
-import org.apache.iotdb.db.qp.logical.crud.SFWOperator;
import org.apache.iotdb.db.qp.physical.PhysicalPlan;
import org.apache.iotdb.db.qp.strategy.PhysicalGenerator;
import org.apache.iotdb.db.qp.strategy.optimizer.ConcatPathOptimizer;
-import org.apache.iotdb.db.query.control.QueryResourceManager;
import java.time.ZoneId;
@@ -37,16 +35,7 @@ public class ClusterPlanner extends Planner {
public PhysicalPlan parseSQLToPhysicalPlan(String sqlStr, ZoneId zoneId, int fetchSize)
throws QueryProcessException {
Operator operator = logicalGenerator.generate(sqlStr, zoneId);
- int maxDeduplicatedPathNum =
- QueryResourceManager.getInstance().getMaxDeduplicatedPathNum(fetchSize);
- if (operator instanceof SFWOperator && ((SFWOperator) operator).isLastQuery()) {
- // Dataset of last query actually has only three columns, so we shouldn't limit the path num
- // while constructing logical plan
- // To avoid overflowing because logicalOptimize function may do maxDeduplicatedPathNum + 1, we
- // set it to Integer.MAX_VALUE - 1
- maxDeduplicatedPathNum = Integer.MAX_VALUE - 1;
- }
- operator = logicalOptimize(operator, maxDeduplicatedPathNum);
+ operator = logicalOptimize(operator);
PhysicalGenerator physicalGenerator = new ClusterPhysicalGenerator();
return physicalGenerator.transformToPhysicalPlan(operator, fetchSize);
}
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/query/LocalQueryExecutor.java b/cluster/src/main/java/org/apache/iotdb/cluster/query/LocalQueryExecutor.java
index 460180d..903167d 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/query/LocalQueryExecutor.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/query/LocalQueryExecutor.java
@@ -86,8 +86,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import static org.apache.iotdb.session.Config.DEFAULT_FETCH_SIZE;
-
public class LocalQueryExecutor {
private static final Logger logger = LoggerFactory.getLogger(LocalQueryExecutor.class);
@@ -231,11 +229,7 @@ public class LocalQueryExecutor {
// the same query from a requester correspond to a context here
RemoteQueryContext queryContext =
- queryManager.getQueryContext(
- request.getRequester(),
- request.getQueryId(),
- request.getFetchSize(),
- request.getDeduplicatedPathNum());
+ queryManager.getQueryContext(request.getRequester(), request.getQueryId());
logger.debug(
DEBUG_SHOW_QUERY_ID,
name,
@@ -325,11 +319,7 @@ public class LocalQueryExecutor {
// the same query from a requester correspond to a context here
RemoteQueryContext queryContext =
- queryManager.getQueryContext(
- request.getRequester(),
- request.getQueryId(),
- request.getFetchSize(),
- request.getDeduplicatedPathNum());
+ queryManager.getQueryContext(request.getRequester(), request.getQueryId());
logger.debug(
DEBUG_SHOW_QUERY_ID,
name,
@@ -492,11 +482,7 @@ public class LocalQueryExecutor {
Set<String> deviceMeasurements = request.getDeviceMeasurements();
RemoteQueryContext queryContext =
- queryManager.getQueryContext(
- request.getRequester(),
- request.getQueryId(),
- request.getFetchSize(),
- request.getDeduplicatedPathNum());
+ queryManager.getQueryContext(request.getRequester(), request.getQueryId());
logger.debug(
DEBUG_SHOW_QUERY_ID,
name,
@@ -584,8 +570,7 @@ public class LocalQueryExecutor {
timeFilter = FilterFactory.deserialize(request.timeFilterBytes);
}
RemoteQueryContext queryContext =
- queryManager.getQueryContext(
- request.getRequestor(), request.queryId, DEFAULT_FETCH_SIZE, -1);
+ queryManager.getQueryContext(request.getRequestor(), request.queryId);
Set<String> deviceMeasurements = request.getDeviceMeasurements();
boolean ascending = request.ascending;
@@ -780,8 +765,7 @@ public class LocalQueryExecutor {
Set<String> deviceMeasurements = request.getDeviceMeasurements();
boolean ascending = request.ascending;
- RemoteQueryContext queryContext =
- queryManager.getQueryContext(request.getRequestor(), queryId, DEFAULT_FETCH_SIZE, -1);
+ RemoteQueryContext queryContext = queryManager.getQueryContext(request.getRequestor(), queryId);
LocalGroupByExecutor executor =
getGroupByExecutor(
path,
@@ -867,8 +851,7 @@ public class LocalQueryExecutor {
long beforeRange = request.getBeforeRange();
Node requester = request.getRequester();
Set<String> deviceMeasurements = request.getDeviceMeasurements();
- RemoteQueryContext queryContext =
- queryManager.getQueryContext(requester, queryId, DEFAULT_FETCH_SIZE, -1);
+ RemoteQueryContext queryContext = queryManager.getQueryContext(requester, queryId);
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream);
@@ -933,8 +916,7 @@ public class LocalQueryExecutor {
dataGroupMember.syncLeaderWithConsistencyCheck(false);
RemoteQueryContext queryContext =
- queryManager.getQueryContext(
- request.getRequestor(), request.getQueryId(), DEFAULT_FETCH_SIZE, -1);
+ queryManager.getQueryContext(request.getRequestor(), request.getQueryId());
List<PartialPath> partialPaths = new ArrayList<>();
for (String path : request.getPaths()) {
partialPaths.add(new PartialPath(path));
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/query/manage/ClusterQueryManager.java b/cluster/src/main/java/org/apache/iotdb/cluster/query/manage/ClusterQueryManager.java
index 455e7fb..f3376ba 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/query/manage/ClusterQueryManager.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/query/manage/ClusterQueryManager.java
@@ -43,16 +43,12 @@ public class ClusterQueryManager {
private Map<Long, IAggregateReader> aggrReaderMap = new ConcurrentHashMap<>();
private Map<Long, GroupByExecutor> groupByExecutorMap = new ConcurrentHashMap<>();
- public synchronized RemoteQueryContext getQueryContext(
- Node node, long queryId, int fetchSize, int deduplicatedPathNum) {
+ public synchronized RemoteQueryContext getQueryContext(Node node, long queryId) {
Map<Long, RemoteQueryContext> nodeContextMap =
queryContextMap.computeIfAbsent(node, n -> new HashMap<>());
return nodeContextMap.computeIfAbsent(
queryId,
- qId ->
- new RemoteQueryContext(
- QueryResourceManager.getInstance()
- .assignQueryId(true, fetchSize, deduplicatedPathNum)));
+ qId -> new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true)));
}
public long registerReader(IBatchReader reader) {
diff --git a/cluster/src/test/java/org/apache/iotdb/cluster/common/IoTDBTest.java b/cluster/src/test/java/org/apache/iotdb/cluster/common/IoTDBTest.java
index e7b1b7b..6a25ce8 100644
--- a/cluster/src/test/java/org/apache/iotdb/cluster/common/IoTDBTest.java
+++ b/cluster/src/test/java/org/apache/iotdb/cluster/common/IoTDBTest.java
@@ -158,8 +158,7 @@ public abstract class IoTDBTest {
protected QueryDataSet query(List<String> pathStrs, IExpression expression)
throws QueryProcessException, QueryFilterOptimizationException, StorageEngineException,
IOException, MetadataException, InterruptedException {
- QueryContext context =
- new QueryContext(QueryResourceManager.getInstance().assignQueryId(true, 1024, -1));
+ QueryContext context = new QueryContext(QueryResourceManager.getInstance().assignQueryId(true));
RawDataQueryPlan queryPlan = new RawDataQueryPlan();
queryPlan.setExpression(expression);
List<PartialPath> paths = new ArrayList<>();
diff --git a/cluster/src/test/java/org/apache/iotdb/cluster/query/ClusterAggregateExecutorTest.java b/cluster/src/test/java/org/apache/iotdb/cluster/query/ClusterAggregateExecutorTest.java
index db4b6fe..086cdc7 100644
--- a/cluster/src/test/java/org/apache/iotdb/cluster/query/ClusterAggregateExecutorTest.java
+++ b/cluster/src/test/java/org/apache/iotdb/cluster/query/ClusterAggregateExecutorTest.java
@@ -85,7 +85,7 @@ public class ClusterAggregateExecutorTest extends BaseQueryTest {
plan.setDeduplicatedAggregations(aggregations);
QueryContext context =
- new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true, 1024, -1));
+ new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true));
try {
executor = new ClusterAggregateExecutor(plan, testMetaMember);
QueryDataSet queryDataSet = executor.executeWithoutValueFilter(context, plan);
@@ -142,7 +142,7 @@ public class ClusterAggregateExecutorTest extends BaseQueryTest {
new PartialPath(TestUtils.getTestSeries(0, 0)), TimeFilter.gtEq(3))));
QueryContext context =
- new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true, 1024, -1));
+ new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true));
try {
executor = new ClusterAggregateExecutor(plan, testMetaMember);
QueryDataSet queryDataSet = executor.executeWithValueFilter(context, plan);
diff --git a/cluster/src/test/java/org/apache/iotdb/cluster/query/ClusterDataQueryExecutorTest.java b/cluster/src/test/java/org/apache/iotdb/cluster/query/ClusterDataQueryExecutorTest.java
index c6dfb93..034a1a6 100644
--- a/cluster/src/test/java/org/apache/iotdb/cluster/query/ClusterDataQueryExecutorTest.java
+++ b/cluster/src/test/java/org/apache/iotdb/cluster/query/ClusterDataQueryExecutorTest.java
@@ -66,7 +66,7 @@ public class ClusterDataQueryExecutorTest extends BaseQueryTest {
plan.setDeduplicatedDataTypes(dataTypes);
queryExecutor = new ClusterDataQueryExecutor(plan, testMetaMember);
RemoteQueryContext context =
- new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true, 1024, -1));
+ new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true));
try {
QueryDataSet dataSet = queryExecutor.executeWithoutValueFilter(context);
checkSequentialDataset(dataSet, 0, 20);
@@ -87,7 +87,7 @@ public class ClusterDataQueryExecutorTest extends BaseQueryTest {
plan.setExpression(expression);
queryExecutor = new ClusterDataQueryExecutor(plan, testMetaMember);
RemoteQueryContext context =
- new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true, 1024, -1));
+ new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true));
try {
QueryDataSet dataSet = queryExecutor.executeWithValueFilter(context);
checkSequentialDataset(dataSet, 5, 15);
@@ -104,7 +104,7 @@ public class ClusterDataQueryExecutorTest extends BaseQueryTest {
plan.setEnableRedirect(true);
queryExecutor = new ClusterDataQueryExecutor(plan, testMetaMember);
RemoteQueryContext context =
- new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true, 1024, -1));
+ new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true));
try {
QueryDataSet dataSet = queryExecutor.executeWithoutValueFilter(context);
assertNull(dataSet.getEndPoint());
@@ -126,7 +126,7 @@ public class ClusterDataQueryExecutorTest extends BaseQueryTest {
plan.setEnableRedirect(true);
queryExecutor = new ClusterDataQueryExecutor(plan, testMetaMember);
RemoteQueryContext context =
- new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true, 1024, -1));
+ new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true));
try {
QueryDataSet dataSet = queryExecutor.executeWithValueFilter(context);
assertNull(dataSet.getEndPoint());
@@ -147,7 +147,7 @@ public class ClusterDataQueryExecutorTest extends BaseQueryTest {
plan.setEnableRedirect(true);
queryExecutor = new ClusterDataQueryExecutor(plan, testMetaMember);
RemoteQueryContext context =
- new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true, 1024, -1));
+ new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true));
try {
QueryDataSet dataSet = queryExecutor.executeWithoutValueFilter(context);
assertEquals("ip:port=0.0.0.0:6667", dataSet.getEndPoint().toString());
diff --git a/cluster/src/test/java/org/apache/iotdb/cluster/query/ClusterFillExecutorTest.java b/cluster/src/test/java/org/apache/iotdb/cluster/query/ClusterFillExecutorTest.java
index 8061851..8bbd9fa 100644
--- a/cluster/src/test/java/org/apache/iotdb/cluster/query/ClusterFillExecutorTest.java
+++ b/cluster/src/test/java/org/apache/iotdb/cluster/query/ClusterFillExecutorTest.java
@@ -60,7 +60,7 @@ public class ClusterFillExecutorTest extends BaseQueryTest {
TSDataType.DOUBLE, new PreviousFill(TSDataType.DOUBLE, 0, defaultFillInterval));
plan.setFillType(tsDataTypeIFillMap);
QueryContext context =
- new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true, 1024, -1));
+ new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true));
try {
ClusterFillExecutor fillExecutor;
@@ -107,7 +107,7 @@ public class ClusterFillExecutorTest extends BaseQueryTest {
new LinearFill(TSDataType.DOUBLE, 0, defaultFillInterval, defaultFillInterval));
plan.setFillType(tsDataTypeIFillMap);
QueryContext context =
- new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true, 1024, -1));
+ new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true));
try {
ClusterFillExecutor fillExecutor;
diff --git a/cluster/src/test/java/org/apache/iotdb/cluster/query/ClusterPlanExecutorTest.java b/cluster/src/test/java/org/apache/iotdb/cluster/query/ClusterPlanExecutorTest.java
index 5de2ce2..a6254b9 100644
--- a/cluster/src/test/java/org/apache/iotdb/cluster/query/ClusterPlanExecutorTest.java
+++ b/cluster/src/test/java/org/apache/iotdb/cluster/query/ClusterPlanExecutorTest.java
@@ -61,7 +61,7 @@ public class ClusterPlanExecutorTest extends BaseQueryTest {
queryPlan.setPaths(pathList);
queryPlan.setDataTypes(dataTypes);
QueryContext context =
- new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true, 1024, -1));
+ new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true));
try {
QueryDataSet dataSet = queryExecutor.processQuery(queryPlan, context);
@@ -96,7 +96,7 @@ public class ClusterPlanExecutorTest extends BaseQueryTest {
IOException, InterruptedException, QueryProcessException {
ShowTimeSeriesPlan showTimeSeriesPlan = new ShowTimeSeriesPlan(pathList.get(0));
QueryContext context =
- new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true, 1024, -1));
+ new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true));
try {
QueryDataSet dataSet = queryExecutor.processQuery(showTimeSeriesPlan, context);
int count = 0;
diff --git a/cluster/src/test/java/org/apache/iotdb/cluster/query/ClusterQueryRouterTest.java b/cluster/src/test/java/org/apache/iotdb/cluster/query/ClusterQueryRouterTest.java
index 9cd7690..7c963c2 100644
--- a/cluster/src/test/java/org/apache/iotdb/cluster/query/ClusterQueryRouterTest.java
+++ b/cluster/src/test/java/org/apache/iotdb/cluster/query/ClusterQueryRouterTest.java
@@ -73,7 +73,7 @@ public class ClusterQueryRouterTest extends BaseQueryTest {
queryPlan.setDeduplicatedPaths(pathList);
queryPlan.setDeduplicatedDataTypes(dataTypes);
QueryContext context =
- new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true, 1024, -1));
+ new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true));
try {
QueryDataSet dataSet = clusterQueryRouter.rawDataQuery(queryPlan, context);
@@ -117,7 +117,7 @@ public class ClusterQueryRouterTest extends BaseQueryTest {
plan.setDeduplicatedAggregations(aggregations);
QueryContext context =
- new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true, 1024, -1));
+ new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true));
try {
QueryDataSet queryDataSet = clusterQueryRouter.aggregate(plan, context);
checkDoubleDataset(queryDataSet, new Object[] {0.0, 19.0, 9.5, 20.0, 190.0});
@@ -142,7 +142,7 @@ public class ClusterQueryRouterTest extends BaseQueryTest {
TSDataType.DOUBLE, new PreviousFill(TSDataType.DOUBLE, 0, defaultFillInterval));
plan.setFillType(tsDataTypeIFillMap);
QueryContext context =
- new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true, 1024, -1));
+ new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true));
try {
QueryDataSet queryDataSet;
@@ -183,7 +183,7 @@ public class ClusterQueryRouterTest extends BaseQueryTest {
plan.setFillType(tsDataTypeIFillMap);
QueryContext context =
- new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true, 1024, -1));
+ new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true));
try {
QueryDataSet queryDataSet;
@@ -212,7 +212,7 @@ public class ClusterQueryRouterTest extends BaseQueryTest {
throws IOException, StorageEngineException, QueryFilterOptimizationException,
QueryProcessException, IllegalPathException {
QueryContext queryContext =
- new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true, 1024, -1));
+ new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true));
try {
GroupByTimePlan groupByPlan = new GroupByTimePlan();
List<PartialPath> pathList = new ArrayList<>();
@@ -265,7 +265,7 @@ public class ClusterQueryRouterTest extends BaseQueryTest {
throws StorageEngineException, IOException, QueryFilterOptimizationException,
QueryProcessException, IllegalPathException {
QueryContext queryContext =
- new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true, 1024, -1));
+ new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true));
try {
GroupByTimePlan groupByPlan = new GroupByTimePlan();
List<PartialPath> pathList = new ArrayList<>();
diff --git a/cluster/src/test/java/org/apache/iotdb/cluster/query/groupby/ClusterGroupByNoVFilterDataSetTest.java b/cluster/src/test/java/org/apache/iotdb/cluster/query/groupby/ClusterGroupByNoVFilterDataSetTest.java
index 66aa6e5..db40a5f 100644
--- a/cluster/src/test/java/org/apache/iotdb/cluster/query/groupby/ClusterGroupByNoVFilterDataSetTest.java
+++ b/cluster/src/test/java/org/apache/iotdb/cluster/query/groupby/ClusterGroupByNoVFilterDataSetTest.java
@@ -48,7 +48,7 @@ public class ClusterGroupByNoVFilterDataSetTest extends BaseQueryTest {
public void test()
throws StorageEngineException, IOException, QueryProcessException, IllegalPathException {
QueryContext queryContext =
- new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true, 1024, -1));
+ new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true));
try {
GroupByTimePlan groupByPlan = new GroupByTimePlan();
List<PartialPath> pathList = new ArrayList<>();
diff --git a/cluster/src/test/java/org/apache/iotdb/cluster/query/groupby/ClusterGroupByVFilterDataSetTest.java b/cluster/src/test/java/org/apache/iotdb/cluster/query/groupby/ClusterGroupByVFilterDataSetTest.java
index 8e39e93..fb89912 100644
--- a/cluster/src/test/java/org/apache/iotdb/cluster/query/groupby/ClusterGroupByVFilterDataSetTest.java
+++ b/cluster/src/test/java/org/apache/iotdb/cluster/query/groupby/ClusterGroupByVFilterDataSetTest.java
@@ -51,7 +51,7 @@ public class ClusterGroupByVFilterDataSetTest extends BaseQueryTest {
public void test()
throws IOException, StorageEngineException, QueryProcessException, IllegalPathException {
QueryContext queryContext =
- new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true, 1024, -1));
+ new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true));
try {
GroupByTimePlan groupByPlan = new GroupByTimePlan();
List<PartialPath> pathList = new ArrayList<>();
diff --git a/cluster/src/test/java/org/apache/iotdb/cluster/query/groupby/MergeGroupByExecutorTest.java b/cluster/src/test/java/org/apache/iotdb/cluster/query/groupby/MergeGroupByExecutorTest.java
index e04e5a6..aa5bb34 100644
--- a/cluster/src/test/java/org/apache/iotdb/cluster/query/groupby/MergeGroupByExecutorTest.java
+++ b/cluster/src/test/java/org/apache/iotdb/cluster/query/groupby/MergeGroupByExecutorTest.java
@@ -49,7 +49,7 @@ public class MergeGroupByExecutorTest extends BaseQueryTest {
PartialPath path = new PartialPath(TestUtils.getTestSeries(0, 0));
TSDataType dataType = TSDataType.DOUBLE;
QueryContext context =
- new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true, 1024, -1));
+ new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true));
try {
Filter timeFilter = null;
@@ -88,7 +88,7 @@ public class MergeGroupByExecutorTest extends BaseQueryTest {
PartialPath path = new PartialPath(TestUtils.getTestSeries(0, 0));
TSDataType dataType = TSDataType.DOUBLE;
QueryContext context =
- new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true, 1024, -1));
+ new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true));
try {
Filter timeFilter = TimeFilter.gtEq(3);
diff --git a/cluster/src/test/java/org/apache/iotdb/cluster/query/groupby/RemoteGroupByExecutorTest.java b/cluster/src/test/java/org/apache/iotdb/cluster/query/groupby/RemoteGroupByExecutorTest.java
index dac449a..a89e811 100644
--- a/cluster/src/test/java/org/apache/iotdb/cluster/query/groupby/RemoteGroupByExecutorTest.java
+++ b/cluster/src/test/java/org/apache/iotdb/cluster/query/groupby/RemoteGroupByExecutorTest.java
@@ -54,7 +54,7 @@ public class RemoteGroupByExecutorTest extends BaseQueryTest {
PartialPath path = new PartialPath(TestUtils.getTestSeries(0, 0));
TSDataType dataType = TSDataType.DOUBLE;
QueryContext context =
- new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true, 1024, -1));
+ new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true));
try {
Filter timeFilter = null;
List<Integer> aggregationTypes = new ArrayList<>();
@@ -116,7 +116,7 @@ public class RemoteGroupByExecutorTest extends BaseQueryTest {
PartialPath path = new PartialPath(TestUtils.getTestSeries(0, 0));
TSDataType dataType = TSDataType.DOUBLE;
QueryContext context =
- new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true, 1024, -1));
+ new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true));
try {
Filter timeFilter = TimeFilter.gtEq(3);
List<Integer> aggregationTypes = new ArrayList<>();
diff --git a/cluster/src/test/java/org/apache/iotdb/cluster/query/manage/ClusterQueryManagerTest.java b/cluster/src/test/java/org/apache/iotdb/cluster/query/manage/ClusterQueryManagerTest.java
index 35a404e..a50b037 100644
--- a/cluster/src/test/java/org/apache/iotdb/cluster/query/manage/ClusterQueryManagerTest.java
+++ b/cluster/src/test/java/org/apache/iotdb/cluster/query/manage/ClusterQueryManagerTest.java
@@ -46,12 +46,9 @@ public class ClusterQueryManagerTest {
@Test
public void testContext() {
- RemoteQueryContext queryContext1 =
- queryManager.getQueryContext(TestUtils.getNode(0), 1, 1024, -1);
- RemoteQueryContext queryContext2 =
- queryManager.getQueryContext(TestUtils.getNode(0), 1, 1024, -1);
- RemoteQueryContext queryContext3 =
- queryManager.getQueryContext(TestUtils.getNode(1), 1, 1024, -1);
+ RemoteQueryContext queryContext1 = queryManager.getQueryContext(TestUtils.getNode(0), 1);
+ RemoteQueryContext queryContext2 = queryManager.getQueryContext(TestUtils.getNode(0), 1);
+ RemoteQueryContext queryContext3 = queryManager.getQueryContext(TestUtils.getNode(1), 1);
assertSame(queryContext1, queryContext2);
assertNotEquals(queryContext2, queryContext3);
}
@@ -158,8 +155,7 @@ public class ClusterQueryManagerTest {
@Test
public void testEndQuery() throws StorageEngineException {
- RemoteQueryContext queryContext =
- queryManager.getQueryContext(TestUtils.getNode(0), 1, 1024, -1);
+ RemoteQueryContext queryContext = queryManager.getQueryContext(TestUtils.getNode(0), 1);
for (int i = 0; i < 10; i++) {
IBatchReader reader =
new IBatchReader() {
diff --git a/cluster/src/test/java/org/apache/iotdb/cluster/query/reader/ClusterReaderFactoryTest.java b/cluster/src/test/java/org/apache/iotdb/cluster/query/reader/ClusterReaderFactoryTest.java
index e40be6a..dd3ccc4 100644
--- a/cluster/src/test/java/org/apache/iotdb/cluster/query/reader/ClusterReaderFactoryTest.java
+++ b/cluster/src/test/java/org/apache/iotdb/cluster/query/reader/ClusterReaderFactoryTest.java
@@ -46,7 +46,7 @@ public class ClusterReaderFactoryTest extends BaseQueryTest {
ClusterReaderFactory readerFactory = new ClusterReaderFactory(testMetaMember);
RemoteQueryContext context =
- new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true, 1024, -1));
+ new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true));
try {
SeriesRawDataBatchReader seriesReader =
diff --git a/cluster/src/test/java/org/apache/iotdb/cluster/query/reader/ClusterTimeGeneratorTest.java b/cluster/src/test/java/org/apache/iotdb/cluster/query/reader/ClusterTimeGeneratorTest.java
index 17261b2..fbbee45 100644
--- a/cluster/src/test/java/org/apache/iotdb/cluster/query/reader/ClusterTimeGeneratorTest.java
+++ b/cluster/src/test/java/org/apache/iotdb/cluster/query/reader/ClusterTimeGeneratorTest.java
@@ -57,7 +57,7 @@ public class ClusterTimeGeneratorTest extends BaseQueryTest {
throws StorageEngineException, IOException, IllegalPathException, QueryProcessException {
RawDataQueryPlan dataQueryPlan = new RawDataQueryPlan();
QueryContext context =
- new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true, 1024, -1));
+ new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true));
try {
IExpression expression =
BinaryExpression.and(
@@ -86,7 +86,7 @@ public class ClusterTimeGeneratorTest extends BaseQueryTest {
throws StorageEngineException, IOException, IllegalPathException, QueryProcessException {
RawDataQueryPlan dataQueryPlan = new RawDataQueryPlan();
QueryContext context =
- new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true, 1024, -1));
+ new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true));
Filter valueFilter = ValueFilter.gtEq(3.0);
Filter timeFilter = TimeFilter.ltEq(8);
try {
diff --git a/cluster/src/test/java/org/apache/iotdb/cluster/server/member/DataGroupMemberTest.java b/cluster/src/test/java/org/apache/iotdb/cluster/server/member/DataGroupMemberTest.java
index 05f4c60..5d138fa 100644
--- a/cluster/src/test/java/org/apache/iotdb/cluster/server/member/DataGroupMemberTest.java
+++ b/cluster/src/test/java/org/apache/iotdb/cluster/server/member/DataGroupMemberTest.java
@@ -1108,7 +1108,7 @@ public class DataGroupMemberTest extends BaseMember {
Filter timeFilter = TimeFilter.gtEq(5);
request.setTimeFilterBytes(SerializeUtils.serializeFilter(timeFilter));
QueryContext queryContext =
- new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true, 1024, -1));
+ new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true));
try {
request.setQueryId(queryContext.getQueryId());
request.setRequestor(TestUtils.getNode(0));
diff --git a/cluster/src/test/java/org/apache/iotdb/cluster/server/member/MetaGroupMemberTest.java b/cluster/src/test/java/org/apache/iotdb/cluster/server/member/MetaGroupMemberTest.java
index f98bc22..0e1e677 100644
--- a/cluster/src/test/java/org/apache/iotdb/cluster/server/member/MetaGroupMemberTest.java
+++ b/cluster/src/test/java/org/apache/iotdb/cluster/server/member/MetaGroupMemberTest.java
@@ -878,7 +878,7 @@ public class MetaGroupMemberTest extends BaseMember {
}
QueryContext context =
- new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true, 1024, -1));
+ new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true));
try {
ClusterReaderFactory readerFactory = new ClusterReaderFactory(testMetaMember);
@@ -940,7 +940,7 @@ public class MetaGroupMemberTest extends BaseMember {
}
QueryContext context =
- new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true, 1024, -1));
+ new RemoteQueryContext(QueryResourceManager.getInstance().assignQueryId(true));
try {
ClusterReaderFactory readerFactory = new ClusterReaderFactory(testMetaMember);
diff --git a/server/src/main/java/org/apache/iotdb/db/exception/query/PathNumOverLimitException.java b/server/src/main/java/org/apache/iotdb/db/exception/query/PathNumOverLimitException.java
index 7714fc8..c4eb10f 100644
--- a/server/src/main/java/org/apache/iotdb/db/exception/query/PathNumOverLimitException.java
+++ b/server/src/main/java/org/apache/iotdb/db/exception/query/PathNumOverLimitException.java
@@ -21,17 +21,10 @@ package org.apache.iotdb.db.exception.query;
public class PathNumOverLimitException extends QueryProcessException {
- public PathNumOverLimitException(long maxDeduplicatedPathNum, long deduplicatedPathNum) {
- super(
- String.format(
- "Too many paths in one query! Currently allowed max deduplicated path number is %d, this query contains %d deduplicated path. Please use slimit to choose what you real want or adjust max_deduplicated_path_num in iotdb-engine.properties.",
- maxDeduplicatedPathNum, deduplicatedPathNum));
- }
-
public PathNumOverLimitException(long maxDeduplicatedPathNum) {
super(
String.format(
- "Too many paths in one query! Currently allowed max deduplicated path number is %d, this query contains unknown deduplicated path. Please use slimit to choose what you real want or adjust max_deduplicated_path_num in iotdb-engine.properties.",
+ "Too many paths in one query! Currently allowed max deduplicated path number is %d. Please use slimit or adjust max_deduplicated_path_num in iotdb-engine.properties.",
maxDeduplicatedPathNum));
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/monitor/StatMonitor.java b/server/src/main/java/org/apache/iotdb/db/monitor/StatMonitor.java
index c5a7848..e97d17a 100644
--- a/server/src/main/java/org/apache/iotdb/db/monitor/StatMonitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/monitor/StatMonitor.java
@@ -164,7 +164,7 @@ public class StatMonitor implements StatMonitorMBean, IService {
LastQueryExecutor.calculateLastPairForSeriesLocally(
Collections.singletonList(monitorSeries),
Collections.singletonList(TSDataType.INT64),
- new QueryContext(QueryResourceManager.getInstance().assignQueryId(true, 1024, 1)),
+ new QueryContext(QueryResourceManager.getInstance().assignQueryId(true)),
null,
Collections.singletonMap(
monitorSeries.getDevice(),
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/Planner.java b/server/src/main/java/org/apache/iotdb/db/qp/Planner.java
index 355b7ef..dd9abd1 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/Planner.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/Planner.java
@@ -39,7 +39,6 @@ import org.apache.iotdb.db.qp.strategy.optimizer.ConcatPathOptimizer;
import org.apache.iotdb.db.qp.strategy.optimizer.DnfFilterOptimizer;
import org.apache.iotdb.db.qp.strategy.optimizer.MergeSingleFilterOptimizer;
import org.apache.iotdb.db.qp.strategy.optimizer.RemoveNotOptimizer;
-import org.apache.iotdb.db.query.control.QueryResourceManager;
import org.apache.iotdb.db.utils.TestOnly;
import org.apache.iotdb.service.rpc.thrift.TSRawDataQueryReq;
@@ -68,16 +67,7 @@ public class Planner {
public PhysicalPlan parseSQLToPhysicalPlan(String sqlStr, ZoneId zoneId, int fetchSize)
throws QueryProcessException {
Operator operator = logicalGenerator.generate(sqlStr, zoneId);
- int maxDeduplicatedPathNum =
- QueryResourceManager.getInstance().getMaxDeduplicatedPathNum(fetchSize);
- if (operator instanceof SFWOperator && ((SFWOperator) operator).isLastQuery()) {
- // Dataset of last query actually has only three columns, so we shouldn't limit the path num
- // while constructing logical plan
- // To avoid overflowing because logicalOptimize function may do maxDeduplicatedPathNum + 1, we
- // set it to Integer.MAX_VALUE - 1
- maxDeduplicatedPathNum = Integer.MAX_VALUE - 1;
- }
- operator = logicalOptimize(operator, maxDeduplicatedPathNum);
+ operator = logicalOptimize(operator);
PhysicalGenerator physicalGenerator = new PhysicalGenerator();
return physicalGenerator.transformToPhysicalPlan(operator, fetchSize);
}
@@ -124,16 +114,7 @@ public class Planner {
queryOp.setFilterOperator(filterOp);
- int maxDeduplicatedPathNum =
- QueryResourceManager.getInstance().getMaxDeduplicatedPathNum(rawDataQueryReq.fetchSize);
- if (queryOp.isLastQuery()) {
- // Dataset of last query actually has only three columns, so we shouldn't limit the path num
- // while constructing logical plan
- // To avoid overflowing because logicalOptimize function may do maxDeduplicatedPathNum + 1, we
- // set it to Integer.MAX_VALUE - 1
- maxDeduplicatedPathNum = Integer.MAX_VALUE - 1;
- }
- SFWOperator op = (SFWOperator) logicalOptimize(queryOp, maxDeduplicatedPathNum);
+ SFWOperator op = (SFWOperator) logicalOptimize(queryOp);
PhysicalGenerator physicalGenerator = new PhysicalGenerator();
return physicalGenerator.transformToPhysicalPlan(op, rawDataQueryReq.fetchSize);
@@ -146,7 +127,7 @@ public class Planner {
* @return optimized logical operator
* @throws LogicalOptimizeException exception in logical optimizing
*/
- protected Operator logicalOptimize(Operator operator, int maxDeduplicatedPathNum)
+ protected Operator logicalOptimize(Operator operator)
throws LogicalOperatorException, PathNumOverLimitException {
switch (operator.getType()) {
case AUTHOR:
@@ -188,7 +169,7 @@ public class Planner {
case DROP_INDEX:
case QUERY_INDEX:
SFWOperator root = (SFWOperator) operator;
- return optimizeSFWOperator(root, maxDeduplicatedPathNum);
+ return optimizeSFWOperator(root);
default:
throw new LogicalOperatorException(operator.getType().toString(), "");
}
@@ -201,10 +182,10 @@ public class Planner {
* @return optimized select-from-where operator
* @throws LogicalOptimizeException exception in SFW optimizing
*/
- private SFWOperator optimizeSFWOperator(SFWOperator root, int maxDeduplicatedPathNum)
+ private SFWOperator optimizeSFWOperator(SFWOperator root)
throws LogicalOperatorException, PathNumOverLimitException {
ConcatPathOptimizer concatPathOptimizer = getConcatPathOptimizer();
- root = (SFWOperator) concatPathOptimizer.transform(root, maxDeduplicatedPathNum);
+ root = (SFWOperator) concatPathOptimizer.transform(root);
FilterOperator filter = root.getFilterOperator();
if (filter == null) {
return root;
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java b/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java
index 0991328..4752eee 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java
@@ -18,6 +18,7 @@
*/
package org.apache.iotdb.db.qp.strategy.optimizer;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.exception.query.LogicalOptimizeException;
import org.apache.iotdb.db.exception.query.PathNumOverLimitException;
@@ -55,7 +56,7 @@ public class ConcatPathOptimizer implements ILogicalOptimizer {
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
@Override
- public Operator transform(Operator operator, int maxDeduplicatedPathNum)
+ public Operator transform(Operator operator)
throws LogicalOptimizeException, PathNumOverLimitException {
if (!(operator instanceof SFWOperator)) {
logger.warn("given operator isn't SFWOperator, cannot concat seriesPath");
@@ -101,7 +102,6 @@ public class ConcatPathOptimizer implements ILogicalOptimizer {
select,
seriesLimit,
seriesOffset,
- maxDeduplicatedPathNum,
((QueryOperator) operator).getIndexType() == null);
} else {
isAlignByDevice = true;
@@ -178,7 +178,6 @@ public class ConcatPathOptimizer implements ILogicalOptimizer {
SelectOperator selectOperator,
int limit,
int offset,
- int maxDeduplicatedPathNum,
boolean needRemoveStar)
throws LogicalOptimizeException, PathNumOverLimitException {
List<PartialPath> suffixPaths = judgeSelectOperator(selectOperator);
@@ -238,8 +237,7 @@ public class ConcatPathOptimizer implements ILogicalOptimizer {
afterConcatUdfList,
selectOperator,
limit,
- offset,
- maxDeduplicatedPathNum);
+ offset);
} else {
selectOperator.setSuffixPathList(afterConcatPaths);
}
@@ -343,14 +341,10 @@ public class ConcatPathOptimizer implements ILogicalOptimizer {
List<UDFContext> afterConcatUdfList,
SelectOperator selectOperator,
int finalLimit,
- int finalOffset,
- int maxDeduplicatedPathNum)
+ int finalOffset)
throws LogicalOptimizeException, PathNumOverLimitException {
int offset = finalOffset;
- int limit =
- finalLimit == 0 || maxDeduplicatedPathNum < finalLimit
- ? maxDeduplicatedPathNum + 1
- : finalLimit;
+ int limit = finalLimit == 0 ? Integer.MAX_VALUE : finalLimit;
int consumed = 0;
List<PartialPath> newSuffixPathList = new ArrayList<>();
@@ -424,6 +418,8 @@ public class ConcatPathOptimizer implements ILogicalOptimizer {
}
if (limit == 0) {
+ int maxDeduplicatedPathNum =
+ IoTDBDescriptor.getInstance().getConfig().getMaxQueryDeduplicatedPathNum();
if (maxDeduplicatedPathNum < newSuffixPathList.size()) {
throw new PathNumOverLimitException(maxDeduplicatedPathNum);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ILogicalOptimizer.java b/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ILogicalOptimizer.java
index 26ab0a5..e96019d 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ILogicalOptimizer.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ILogicalOptimizer.java
@@ -26,6 +26,5 @@ import org.apache.iotdb.db.qp.logical.Operator;
@FunctionalInterface
public interface ILogicalOptimizer {
- Operator transform(Operator operator, int maxDeduplicatedPathNum)
- throws LogicalOptimizeException, PathNumOverLimitException;
+ Operator transform(Operator operator) throws LogicalOptimizeException, PathNumOverLimitException;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java b/server/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java
index d0042a1..feec5e0 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java
@@ -25,7 +25,6 @@ import org.apache.iotdb.db.engine.StorageEngine;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.exception.StorageEngineException;
-import org.apache.iotdb.db.exception.query.PathNumOverLimitException;
import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.metadata.PartialPath;
import org.apache.iotdb.db.query.context.QueryContext;
@@ -74,59 +73,20 @@ public class QueryResourceManager {
*/
private final Map<Long, List<IExternalSortFileDeserializer>> externalSortFileMap;
- private final Map<Long, Long> queryIdEstimatedMemoryMap;
-
- // current total free memory for reading process(not including the cache memory)
- private final AtomicLong totalFreeMemoryForRead;
-
- // estimated size for one point memory size, the unit is byte
- private static final long POINT_ESTIMATED_SIZE = 16L;
-
- private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig();
-
private QueryResourceManager() {
filePathsManager = new QueryFileManager();
externalSortFileMap = new ConcurrentHashMap<>();
- queryIdEstimatedMemoryMap = new ConcurrentHashMap<>();
- totalFreeMemoryForRead =
- new AtomicLong(
- IoTDBDescriptor.getInstance().getConfig().getAllocateMemoryForReadWithoutCache());
}
public static QueryResourceManager getInstance() {
return QueryTokenManagerHelper.INSTANCE;
}
- public int getMaxDeduplicatedPathNum(int fetchSize) {
- if (fetchSize == 0) {
- return CONFIG.getMaxQueryDeduplicatedPathNum();
- }
- return (int)
- Math.min(
- ((totalFreeMemoryForRead.get() / fetchSize) / POINT_ESTIMATED_SIZE),
- CONFIG.getMaxQueryDeduplicatedPathNum());
- }
-
/** Register a new query. When a query request is created firstly, this method must be invoked. */
- public long assignQueryId(boolean isDataQuery, int fetchSize, int deduplicatedPathNum) {
- int maxDeduplicatedPathNum = getMaxDeduplicatedPathNum(fetchSize);
- if (deduplicatedPathNum > maxDeduplicatedPathNum) {
- throw new RuntimeException(
- new PathNumOverLimitException(maxDeduplicatedPathNum, deduplicatedPathNum));
- }
+ public long assignQueryId(boolean isDataQuery) {
long queryId = queryIdAtom.incrementAndGet();
if (isDataQuery) {
filePathsManager.addQueryId(queryId);
- if (deduplicatedPathNum > 0) {
- long estimatedMemoryUsage =
- (long) deduplicatedPathNum * POINT_ESTIMATED_SIZE * (long) fetchSize;
- // apply the memory successfully
- if (totalFreeMemoryForRead.addAndGet(-estimatedMemoryUsage) >= 0) {
- queryIdEstimatedMemoryMap.put(queryId, estimatedMemoryUsage);
- } else {
- totalFreeMemoryForRead.addAndGet(estimatedMemoryUsage);
- }
- }
}
return queryId;
}
@@ -212,12 +172,6 @@ public class QueryResourceManager {
externalSortFileMap.remove(queryId);
}
- // put back the memory usage
- Long estimatedMemoryUsage = queryIdEstimatedMemoryMap.remove(queryId);
- if (estimatedMemoryUsage != null) {
- totalFreeMemoryForRead.addAndGet(estimatedMemoryUsage);
- }
-
// remove usage of opened file paths of current thread
filePathsManager.removeUsedFilesForQuery(queryId);
diff --git a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
index 67f0467..a61aedd 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
@@ -50,7 +50,6 @@ import org.apache.iotdb.db.qp.physical.crud.AlignByDevicePlan;
import org.apache.iotdb.db.qp.physical.crud.AlignByDevicePlan.MeasurementType;
import org.apache.iotdb.db.qp.physical.crud.CreateTemplatePlan;
import org.apache.iotdb.db.qp.physical.crud.DeletePlan;
-import org.apache.iotdb.db.qp.physical.crud.GroupByTimePlan;
import org.apache.iotdb.db.qp.physical.crud.InsertMultiTabletPlan;
import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan;
import org.apache.iotdb.db.qp.physical.crud.InsertRowsOfOneDevicePlan;
@@ -58,7 +57,6 @@ import org.apache.iotdb.db.qp.physical.crud.InsertRowsPlan;
import org.apache.iotdb.db.qp.physical.crud.InsertTabletPlan;
import org.apache.iotdb.db.qp.physical.crud.LastQueryPlan;
import org.apache.iotdb.db.qp.physical.crud.QueryPlan;
-import org.apache.iotdb.db.qp.physical.crud.RawDataQueryPlan;
import org.apache.iotdb.db.qp.physical.crud.SetDeviceTemplatePlan;
import org.apache.iotdb.db.qp.physical.crud.UDFPlan;
import org.apache.iotdb.db.qp.physical.crud.UDTFPlan;
@@ -127,7 +125,6 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet;
-import org.apache.iotdb.tsfile.utils.Pair;
import org.antlr.v4.runtime.misc.ParseCancellationException;
import org.apache.thrift.TException;
@@ -140,7 +137,6 @@ import java.sql.SQLException;
import java.time.ZoneId;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Calendar;
import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.List;
@@ -756,13 +752,8 @@ public class TSServiceImpl implements TSIService.Iface {
long startTime = System.currentTimeMillis();
long queryId = -1;
try {
-
- // pair.left = fetchSize, pair.right = deduplicatedNum
- Pair<Integer, Integer> p = getMemoryParametersFromPhysicalPlan(plan, fetchSize);
- fetchSize = p.left;
-
// generate the queryId for the operation
- queryId = generateQueryId(true, fetchSize, p.right);
+ queryId = generateQueryId(true);
// register query info to queryTimeManager
if (!(plan instanceof ShowQueryProcesslistPlan)) {
queryTimeManager.registerQuery(queryId, startTime, statement, timeout);
@@ -881,53 +872,6 @@ public class TSServiceImpl implements TSIService.Iface {
}
}
- /**
- * get fetchSize and deduplicatedPathNum that are used for memory estimation
- *
- * @return Pair - fetchSize, deduplicatedPathNum
- */
- private Pair<Integer, Integer> getMemoryParametersFromPhysicalPlan(
- PhysicalPlan plan, int fetchSizeBefore) {
- // In case users forget to set this field in query, use the default value
- int fetchSize = fetchSizeBefore == 0 ? DEFAULT_FETCH_SIZE : fetchSizeBefore;
- int deduplicatedPathNum = -1;
- if (plan instanceof GroupByTimePlan) {
- fetchSize = Math.min(getFetchSizeForGroupByTimePlan((GroupByTimePlan) plan), fetchSize);
- } else if (plan.getOperatorType() == OperatorType.AGGREGATION) {
- // the actual row number of aggregation query is 1
- fetchSize = 1;
- }
- if (plan instanceof AlignByDevicePlan) {
- deduplicatedPathNum = ((AlignByDevicePlan) plan).getMeasurements().size();
- } else if (plan instanceof LastQueryPlan) {
- // dataset of last query consists of three column: time column + value column = 1
- // deduplicatedPathNum
- // and we assume that the memory which sensor name takes equals to 1 deduplicatedPathNum
- deduplicatedPathNum = 2;
- // last query's actual row number should be the minimum between the number of series and
- // fetchSize
- fetchSize = Math.min(((LastQueryPlan) plan).getDeduplicatedPaths().size(), fetchSize);
- } else if (plan instanceof RawDataQueryPlan) {
- deduplicatedPathNum = ((RawDataQueryPlan) plan).getDeduplicatedPaths().size();
- }
- return new Pair<>(fetchSize, deduplicatedPathNum);
- }
-
- /*
- calculate fetch size for group by time plan
- */
- private int getFetchSizeForGroupByTimePlan(GroupByTimePlan plan) {
- int rows = (int) ((plan.getEndTime() - plan.getStartTime()) / plan.getInterval());
- // rows gets 0 is caused by: the end time - the start time < the time interval.
- if (rows == 0 && plan.isIntervalByMonth()) {
- Calendar calendar = Calendar.getInstance();
- calendar.setTimeInMillis(plan.getStartTime());
- calendar.add(Calendar.MONTH, (int) (plan.getInterval() / MS_TO_MONTH));
- rows = calendar.getTimeInMillis() <= plan.getEndTime() ? 1 : 0;
- }
- return rows;
- }
-
private TSExecuteStatementResp getListDataSetHeaders(QueryDataSet dataSet) {
return StaticResps.getNoTimeExecuteResp(
dataSet.getPaths().stream().map(Path::getFullPath).collect(Collectors.toList()),
@@ -1239,7 +1183,7 @@ public class TSServiceImpl implements TSIService.Iface {
status = executeNonQueryPlan(plan);
TSExecuteStatementResp resp = RpcUtils.getTSExecuteStatementResp(status);
- long queryId = generateQueryId(false, DEFAULT_FETCH_SIZE, -1);
+ long queryId = generateQueryId(false);
return resp.setQueryId(queryId);
}
@@ -2000,9 +1944,8 @@ public class TSServiceImpl implements TSIService.Iface {
: RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR);
}
- private long generateQueryId(boolean isDataQuery, int fetchSize, int deduplicatedPathNum) {
- return QueryResourceManager.getInstance()
- .assignQueryId(isDataQuery, fetchSize, deduplicatedPathNum);
+ private long generateQueryId(boolean isDataQuery) {
+ return QueryResourceManager.getInstance().assignQueryId(isDataQuery);
}
protected List<TSDataType> getSeriesTypesByPaths(
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSequenceDataQueryIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSequenceDataQueryIT.java
index 540bf53..a822949 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSequenceDataQueryIT.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSequenceDataQueryIT.java
@@ -205,8 +205,7 @@ public class IoTDBSequenceDataQueryIT {
new PartialPath(TestConstant.d1 + TsFileConstant.PATH_SEPARATOR + TestConstant.s1));
dataTypes.add(TSDataType.INT64);
- TEST_QUERY_JOB_ID =
- QueryResourceManager.getInstance().assignQueryId(true, 1024, pathList.size());
+ TEST_QUERY_JOB_ID = QueryResourceManager.getInstance().assignQueryId(true);
TEST_QUERY_CONTEXT = new QueryContext(TEST_QUERY_JOB_ID);
RawDataQueryPlan queryPlan = new RawDataQueryPlan();
queryPlan.setDeduplicatedDataTypes(dataTypes);
@@ -240,8 +239,7 @@ public class IoTDBSequenceDataQueryIT {
dataTypes.add(TSDataType.INT64);
GlobalTimeExpression globalTimeExpression = new GlobalTimeExpression(TimeFilter.gtEq(800L));
- TEST_QUERY_JOB_ID =
- QueryResourceManager.getInstance().assignQueryId(true, 1024, pathList.size());
+ TEST_QUERY_JOB_ID = QueryResourceManager.getInstance().assignQueryId(true);
TEST_QUERY_CONTEXT = new QueryContext(TEST_QUERY_JOB_ID);
RawDataQueryPlan queryPlan = new RawDataQueryPlan();
@@ -297,8 +295,7 @@ public class IoTDBSequenceDataQueryIT {
SingleSeriesExpression singleSeriesExpression =
new SingleSeriesExpression(queryPath, ValueFilter.gtEq(14));
- TEST_QUERY_JOB_ID =
- QueryResourceManager.getInstance().assignQueryId(true, 1024, pathList.size());
+ TEST_QUERY_JOB_ID = QueryResourceManager.getInstance().assignQueryId(true);
TEST_QUERY_CONTEXT = new QueryContext(TEST_QUERY_JOB_ID);
RawDataQueryPlan queryPlan = new RawDataQueryPlan();
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSeriesReaderIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSeriesReaderIT.java
index 1710979..55ec4c8 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSeriesReaderIT.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSeriesReaderIT.java
@@ -313,8 +313,7 @@ public class IoTDBSeriesReaderIT {
new PartialPath(TestConstant.d1 + TsFileConstant.PATH_SEPARATOR + TestConstant.s1));
dataTypes.add(TSDataType.INT64);
- TEST_QUERY_JOB_ID =
- QueryResourceManager.getInstance().assignQueryId(true, 1024, pathList.size());
+ TEST_QUERY_JOB_ID = QueryResourceManager.getInstance().assignQueryId(true);
TEST_QUERY_CONTEXT = new QueryContext(TEST_QUERY_JOB_ID);
RawDataQueryPlan queryPlan = new RawDataQueryPlan();
@@ -345,8 +344,7 @@ public class IoTDBSeriesReaderIT {
SingleSeriesExpression singleSeriesExpression =
new SingleSeriesExpression(p, ValueFilter.gtEq(20));
- TEST_QUERY_JOB_ID =
- QueryResourceManager.getInstance().assignQueryId(true, 1024, pathList.size());
+ TEST_QUERY_JOB_ID = QueryResourceManager.getInstance().assignQueryId(true);
TEST_QUERY_CONTEXT = new QueryContext(TEST_QUERY_JOB_ID);
RawDataQueryPlan queryPlan = new RawDataQueryPlan();
@@ -374,7 +372,7 @@ public class IoTDBSeriesReaderIT {
List<TSDataType> dataTypes = Collections.singletonList(TSDataType.INT32);
SingleSeriesExpression expression = new SingleSeriesExpression(path, TimeFilter.gt(22987L));
- TEST_QUERY_JOB_ID = QueryResourceManager.getInstance().assignQueryId(true, 1024, 1);
+ TEST_QUERY_JOB_ID = QueryResourceManager.getInstance().assignQueryId(true);
TEST_QUERY_CONTEXT = new QueryContext(TEST_QUERY_JOB_ID);
RawDataQueryPlan queryPlan = new RawDataQueryPlan();
@@ -414,8 +412,7 @@ public class IoTDBSeriesReaderIT {
dataTypes.add(TSDataType.INT64);
queryPlan.setDeduplicatedDataTypes(dataTypes);
- TEST_QUERY_JOB_ID =
- QueryResourceManager.getInstance().assignQueryId(true, 1024, pathList.size());
+ TEST_QUERY_JOB_ID = QueryResourceManager.getInstance().assignQueryId(true);
TEST_QUERY_CONTEXT = new QueryContext(TEST_QUERY_JOB_ID);
SingleSeriesExpression singleSeriesExpression =
diff --git a/server/src/test/java/org/apache/iotdb/db/qp/logical/LogicalPlanSmallTest.java b/server/src/test/java/org/apache/iotdb/db/qp/logical/LogicalPlanSmallTest.java
index e384edd..1ea618f 100644
--- a/server/src/test/java/org/apache/iotdb/db/qp/logical/LogicalPlanSmallTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/qp/logical/LogicalPlanSmallTest.java
@@ -193,7 +193,7 @@ public class LogicalPlanSmallTest {
(RootOperator) logicalGenerator.generate(sqlStr, ZoneId.systemDefault());
IoTDB.metaManager.init();
ConcatPathOptimizer concatPathOptimizer = new ConcatPathOptimizer();
- concatPathOptimizer.transform(operator, 1000);
+ concatPathOptimizer.transform(operator);
IoTDB.metaManager.clear();
// expected to throw LogicalOptimizeException: The value of SOFFSET (%d) is equal to or exceeds
// the number of sequences (%d) that can actually be returned.
diff --git a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
index 7245c19..6a523b5 100644
--- a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
+++ b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
@@ -41,6 +41,9 @@ import org.apache.iotdb.db.service.IoTDB;
import org.apache.iotdb.rpc.TConfigurationConst;
import org.apache.iotdb.rpc.TSocketWrapper;
+import javax.management.remote.JMXConnector;
+import javax.management.remote.JMXConnectorFactory;
+import javax.management.remote.JMXServiceURL;
import org.apache.commons.io.FileUtils;
import org.apache.thrift.TConfiguration;
import org.apache.thrift.transport.TTransport;
@@ -48,10 +51,6 @@ import org.apache.thrift.transport.TTransportException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.management.remote.JMXConnector;
-import javax.management.remote.JMXConnectorFactory;
-import javax.management.remote.JMXServiceURL;
-
import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
@@ -247,7 +246,7 @@ public class EnvironmentUtils {
}
createAllDir();
- TEST_QUERY_JOB_ID = QueryResourceManager.getInstance().assignQueryId(true, 1024, 0);
+ TEST_QUERY_JOB_ID = QueryResourceManager.getInstance().assignQueryId(true);
TEST_QUERY_CONTEXT = new QueryContext(TEST_QUERY_JOB_ID);
}