You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2023/07/16 10:34:41 UTC
[iotdb] branch master updated: [IOTDB-6019] Fix concurrent update of last query
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 5a50f0df22f [IOTDB-6019] Fix concurrent update of last query
5a50f0df22f is described below
commit 5a50f0df22ff8649a23a5139474cdede1736d95e
Author: Weihao Li <60...@users.noreply.github.com>
AuthorDate: Sun Jul 16 18:34:34 2023 +0800
[IOTDB-6019] Fix concurrent update of last query
---
.../execution/fragment/DataNodeQueryContext.java | 72 ++++++++++++++++++++++
.../fragment/FragmentInstanceContext.java | 41 +++++++++++-
.../fragment/FragmentInstanceManager.java | 18 +++++-
.../last/AbstractUpdateLastCacheOperator.java | 50 ++++++++++++++-
.../last/AlignedUpdateLastCacheOperator.java | 40 ++++++------
.../AlignedUpdateViewPathLastCacheOperator.java | 17 ++---
.../operator/process/last/LastQueryUtil.java | 4 ++
.../process/last/UpdateLastCacheOperator.java | 16 ++---
.../last/UpdateViewPathLastCacheOperator.java | 10 ++-
.../cache/schema/lastcache/LastCacheContainer.java | 11 +++-
.../plan/planner/LocalExecutionPlanContext.java | 19 +++++-
.../plan/planner/LocalExecutionPlanner.java | 12 ++--
.../plan/planner/OperatorTreeGenerator.java | 42 +++++++++++--
.../SimpleFragmentParallelPlanner.java | 63 +++++++++++++++++--
.../plan/planner/plan/FragmentInstance.java | 28 ++++++++-
.../plan/node/source/AlignedLastQueryScanNode.java | 26 ++++++--
.../plan/node/source/LastQueryScanNode.java | 27 ++++++--
.../plan/node/source/LastSeriesSourceNode.java | 63 +++++++++++++++++++
.../execution/operator/LastQueryOperatorTest.java | 4 ++
.../operator/LastQuerySortOperatorTest.java | 4 ++
.../execution/operator/OperatorMemoryTest.java | 13 +++-
.../operator/UpdateLastCacheOperatorTest.java | 1 +
.../queryengine/plan/plan/PipelineBuilderTest.java | 4 +-
.../queryengine/plan/plan/distribution/Util.java | 6 ++
.../builder/EvaluationDAGBuilderTest.java | 4 +-
25 files changed, 522 insertions(+), 73 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/DataNodeQueryContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/DataNodeQueryContext.java
new file mode 100644
index 00000000000..9331f8e4749
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/DataNodeQueryContext.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.execution.fragment;
+
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.tsfile.read.TimeValuePair;
+import org.apache.iotdb.tsfile.utils.Pair;
+
+import javax.annotation.concurrent.GuardedBy;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.ReentrantLock;
+
+public class DataNodeQueryContext {
+ // left of Pair is DataNodeSeriesScanNum, right of Pair is the last value waiting to be updated
+ @GuardedBy("lock")
+ private final Map<PartialPath, Pair<AtomicInteger, TimeValuePair>> uncachedPathToSeriesScanInfo;
+
+ private final AtomicInteger dataNodeFINum;
+
+ // TODO consider more fine-grained locks, now the AtomicInteger in uncachedPathToSeriesScanInfo is
+ // unnecessary
+ private final ReentrantLock lock = new ReentrantLock();
+
+ public DataNodeQueryContext(int dataNodeFINum) {
+ this.uncachedPathToSeriesScanInfo = new HashMap<>();
+ this.dataNodeFINum = new AtomicInteger(dataNodeFINum);
+ }
+
+ public boolean unCached(PartialPath path) {
+ return uncachedPathToSeriesScanInfo.containsKey(path);
+ }
+
+ public void addUnCachePath(PartialPath path, AtomicInteger dataNodeSeriesScanNum) {
+ uncachedPathToSeriesScanInfo.put(path, new Pair<>(dataNodeSeriesScanNum, null));
+ }
+
+ public Pair<AtomicInteger, TimeValuePair> getSeriesScanInfo(PartialPath path) {
+ return uncachedPathToSeriesScanInfo.get(path);
+ }
+
+ public int decreaseDataNodeFINum() {
+ return dataNodeFINum.decrementAndGet();
+ }
+
+ public void lock() {
+ lock.lock();
+ }
+
+ public void unLock() {
+ lock.unlock();
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java
index 648029e7181..ae73a672f72 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java
@@ -23,6 +23,7 @@ import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.queryengine.common.FragmentInstanceId;
+import org.apache.iotdb.db.queryengine.common.QueryId;
import org.apache.iotdb.db.queryengine.common.SessionInfo;
import org.apache.iotdb.db.storageengine.dataregion.IDataRegionForQuery;
import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource;
@@ -38,6 +39,7 @@ import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
@@ -77,6 +79,9 @@ public class FragmentInstanceContext extends QueryContext {
// session info
private SessionInfo sessionInfo;
+ private final Map<QueryId, DataNodeQueryContext> dataNodeQueryContextMap;
+ private DataNodeQueryContext dataNodeQueryContext;
+
public static FragmentInstanceContext createFragmentInstanceContext(
FragmentInstanceId id, FragmentInstanceStateMachine stateMachine, SessionInfo sessionInfo) {
FragmentInstanceContext instanceContext =
@@ -91,9 +96,11 @@ public class FragmentInstanceContext extends QueryContext {
FragmentInstanceStateMachine stateMachine,
SessionInfo sessionInfo,
IDataRegionForQuery dataRegion,
- Filter timeFilter) {
+ Filter timeFilter,
+ Map<QueryId, DataNodeQueryContext> dataNodeQueryContextMap) {
FragmentInstanceContext instanceContext =
- new FragmentInstanceContext(id, stateMachine, sessionInfo, dataRegion, timeFilter);
+ new FragmentInstanceContext(
+ id, stateMachine, sessionInfo, dataRegion, timeFilter, dataNodeQueryContextMap);
instanceContext.initialize();
instanceContext.start();
return instanceContext;
@@ -119,13 +126,16 @@ public class FragmentInstanceContext extends QueryContext {
FragmentInstanceStateMachine stateMachine,
SessionInfo sessionInfo,
IDataRegionForQuery dataRegion,
- Filter timeFilter) {
+ Filter timeFilter,
+ Map<QueryId, DataNodeQueryContext> dataNodeQueryContextMap) {
this.id = id;
this.stateMachine = stateMachine;
this.executionEndTime.set(END_TIME_INITIAL_VALUE);
this.sessionInfo = sessionInfo;
this.dataRegion = dataRegion;
this.timeFilter = timeFilter;
+ this.dataNodeQueryContextMap = dataNodeQueryContextMap;
+ this.dataNodeQueryContext = dataNodeQueryContextMap.get(id.getQueryId());
}
private FragmentInstanceContext(
@@ -134,6 +144,8 @@ public class FragmentInstanceContext extends QueryContext {
this.stateMachine = stateMachine;
this.executionEndTime.set(END_TIME_INITIAL_VALUE);
this.sessionInfo = sessionInfo;
+ this.dataNodeQueryContextMap = null;
+ this.dataNodeQueryContext = null;
}
@TestOnly
@@ -146,6 +158,8 @@ public class FragmentInstanceContext extends QueryContext {
this.queryId = queryId;
this.id = null;
this.stateMachine = null;
+ this.dataNodeQueryContextMap = null;
+ this.dataNodeQueryContext = null;
}
public void start() {
@@ -230,6 +244,14 @@ public class FragmentInstanceContext extends QueryContext {
return executionStartTime.get();
}
+ public DataNodeQueryContext getDataNodeQueryContext() {
+ return dataNodeQueryContext;
+ }
+
+ public void setDataNodeQueryContext(DataNodeQueryContext dataNodeQueryContext) {
+ this.dataNodeQueryContext = dataNodeQueryContext;
+ }
+
public FragmentInstanceInfo getInstanceInfo() {
return new FragmentInstanceInfo(
stateMachine.getState(), getEndTime(), getFailedCause(), getFailureInfoList());
@@ -390,6 +412,19 @@ public class FragmentInstanceContext extends QueryContext {
timeFilter = null;
sourcePaths = null;
sharedQueryDataSource = null;
+ releaseDataNodeQueryContext();
+ }
+
+ private void releaseDataNodeQueryContext() {
+ if (dataNodeQueryContextMap == null) {
+ // this process is in fetch schema, nothing need to release
+ return;
+ }
+
+ if (dataNodeQueryContext.decreaseDataNodeFINum() == 0) {
+ dataNodeQueryContext = null;
+ dataNodeQueryContextMap.remove(id.getQueryId());
+ }
}
public void setMayHaveTmpFile(boolean mayHaveTmpFile) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceManager.java
index 55d157ad1ea..7cb67575246 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceManager.java
@@ -24,6 +24,7 @@ import org.apache.iotdb.commons.concurrent.ThreadName;
import org.apache.iotdb.commons.concurrent.threadpool.ScheduledExecutorUtil;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.queryengine.common.FragmentInstanceId;
+import org.apache.iotdb.db.queryengine.common.QueryId;
import org.apache.iotdb.db.queryengine.execution.driver.IDriver;
import org.apache.iotdb.db.queryengine.execution.exchange.MPPDataExchangeManager;
import org.apache.iotdb.db.queryengine.execution.exchange.MPPDataExchangeService;
@@ -64,6 +65,7 @@ public class FragmentInstanceManager {
private final Map<FragmentInstanceId, FragmentInstanceContext> instanceContext;
private final Map<FragmentInstanceId, FragmentInstanceExecution> instanceExecution;
+ private final Map<QueryId, DataNodeQueryContext> dataNodeQueryContextMap;
private final LocalExecutionPlanner planner = LocalExecutionPlanner.getInstance();
private final IDriverScheduler scheduler = DriverScheduler.getInstance();
@@ -90,6 +92,7 @@ public class FragmentInstanceManager {
private FragmentInstanceManager() {
this.instanceContext = new ConcurrentHashMap<>();
this.instanceExecution = new ConcurrentHashMap<>();
+ this.dataNodeQueryContextMap = new ConcurrentHashMap<>();
this.instanceManagementExecutor =
IoTDBThreadPoolFactory.newScheduledThreadPool(
1, ThreadName.FRAGMENT_INSTANCE_MANAGEMENT.getName());
@@ -135,6 +138,10 @@ public class FragmentInstanceManager {
FragmentInstanceStateMachine stateMachine =
new FragmentInstanceStateMachine(instanceId, instanceNotificationExecutor);
+ int dataNodeFINum = instance.getDataNodeFINum();
+ DataNodeQueryContext dataNodeQueryContext =
+ getOrCreateDataNodeQueryContext(instanceId.getQueryId(), dataNodeFINum);
+
FragmentInstanceContext context =
instanceContext.computeIfAbsent(
instanceId,
@@ -144,14 +151,16 @@ public class FragmentInstanceManager {
stateMachine,
instance.getSessionInfo(),
dataRegion,
- instance.getTimeFilter()));
+ instance.getTimeFilter(),
+ dataNodeQueryContextMap));
try {
List<PipelineDriverFactory> driverFactories =
planner.plan(
instance.getFragment().getPlanNodeTree(),
instance.getFragment().getTypeProvider(),
- context);
+ context,
+ dataNodeQueryContext);
List<IDriver> drivers = new ArrayList<>();
driverFactories.forEach(factory -> drivers.add(factory.createDriver()));
@@ -195,6 +204,11 @@ public class FragmentInstanceManager {
}
}
+ private DataNodeQueryContext getOrCreateDataNodeQueryContext(QueryId queryId, int dataNodeFINum) {
+ return dataNodeQueryContextMap.computeIfAbsent(
+ queryId, queryId1 -> new DataNodeQueryContext(dataNodeFINum));
+ }
+
@SuppressWarnings("squid:S1181")
public FragmentInstanceInfo execSchemaQueryFragmentInstance(
FragmentInstance instance, ISchemaRegion schemaRegion) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/AbstractUpdateLastCacheOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/AbstractUpdateLastCacheOperator.java
index a5c13d3b99c..a979271cb72 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/AbstractUpdateLastCacheOperator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/AbstractUpdateLastCacheOperator.java
@@ -19,18 +19,27 @@
package org.apache.iotdb.db.queryengine.execution.operator.process.last;
+import org.apache.iotdb.commons.path.MeasurementPath;
import org.apache.iotdb.db.queryengine.execution.driver.DataDriverContext;
+import org.apache.iotdb.db.queryengine.execution.fragment.DataNodeQueryContext;
import org.apache.iotdb.db.queryengine.execution.operator.Operator;
import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext;
import org.apache.iotdb.db.queryengine.execution.operator.process.ProcessOperator;
import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeSchemaCache;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.TimeValuePair;
import org.apache.iotdb.tsfile.read.common.block.TsBlock;
import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
+import org.apache.iotdb.tsfile.utils.Pair;
+import org.apache.iotdb.tsfile.utils.TsPrimitiveType;
import com.google.common.collect.ImmutableList;
import com.google.common.util.concurrent.ListenableFuture;
+import javax.annotation.Nullable;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
public abstract class AbstractUpdateLastCacheOperator implements ProcessOperator {
protected static final TsBlock LAST_QUERY_EMPTY_TSBLOCK =
new TsBlockBuilder(ImmutableList.of(TSDataType.TEXT, TSDataType.TEXT, TSDataType.TEXT))
@@ -38,11 +47,15 @@ public abstract class AbstractUpdateLastCacheOperator implements ProcessOperator
protected OperatorContext operatorContext;
+ protected final DataNodeQueryContext dataNodeQueryContext;
+
protected Operator child;
protected DataNodeSchemaCache lastCache;
- protected boolean needUpdateCache;
+ protected final boolean needUpdateCache;
+
+ protected final boolean needUpdateNullEntry;
protected TsBlockBuilder tsBlockBuilder;
@@ -52,12 +65,16 @@ public abstract class AbstractUpdateLastCacheOperator implements ProcessOperator
OperatorContext operatorContext,
Operator child,
DataNodeSchemaCache dataNodeSchemaCache,
- boolean needUpdateCache) {
+ boolean needUpdateCache,
+ boolean needUpdateNullEntry) {
this.operatorContext = operatorContext;
this.child = child;
this.lastCache = dataNodeSchemaCache;
this.needUpdateCache = needUpdateCache;
+ this.needUpdateNullEntry = needUpdateNullEntry;
this.tsBlockBuilder = LastQueryUtil.createTsBlockBuilder(1);
+ this.dataNodeQueryContext =
+ operatorContext.getDriverContext().getFragmentInstanceContext().getDataNodeQueryContext();
}
@Override
@@ -80,6 +97,35 @@ public abstract class AbstractUpdateLastCacheOperator implements ProcessOperator
return databaseName;
}
+ protected void mayUpdateLastCache(
+ long time, @Nullable TsPrimitiveType value, MeasurementPath fullPath) {
+ if (!needUpdateCache) {
+ return;
+ }
+ try {
+ dataNodeQueryContext.lock();
+ Pair<AtomicInteger, TimeValuePair> seriesScanInfo =
+ dataNodeQueryContext.getSeriesScanInfo(fullPath);
+
+ // may enter this case when use TTL
+ if (seriesScanInfo == null) {
+ return;
+ }
+
+ // update cache in DataNodeQueryContext
+ if (seriesScanInfo.right == null || time > seriesScanInfo.right.getTimestamp()) {
+ seriesScanInfo.right = new TimeValuePair(time, value);
+ }
+
+ if (seriesScanInfo.left.decrementAndGet() == 0) {
+ lastCache.updateLastCache(
+ getDatabaseName(), fullPath, seriesScanInfo.right, false, Long.MIN_VALUE);
+ }
+ } finally {
+ dataNodeQueryContext.unLock();
+ }
+ }
+
@Override
public boolean hasNext() throws Exception {
return child.hasNextWithTimer();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/AlignedUpdateLastCacheOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/AlignedUpdateLastCacheOperator.java
index 4039cd1102e..eabfa98a2a7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/AlignedUpdateLastCacheOperator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/AlignedUpdateLastCacheOperator.java
@@ -25,7 +25,6 @@ import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.db.queryengine.execution.operator.Operator;
import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext;
import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeSchemaCache;
-import org.apache.iotdb.tsfile.read.TimeValuePair;
import org.apache.iotdb.tsfile.read.common.block.TsBlock;
import org.apache.iotdb.tsfile.utils.TsPrimitiveType;
@@ -43,8 +42,9 @@ public class AlignedUpdateLastCacheOperator extends AbstractUpdateLastCacheOpera
Operator child,
AlignedPath seriesPath,
DataNodeSchemaCache dataNodeSchemaCache,
- boolean needUpdateCache) {
- super(operatorContext, child, dataNodeSchemaCache, needUpdateCache);
+ boolean needUpdateCache,
+ boolean needUpdateNullEntry) {
+ super(operatorContext, child, dataNodeSchemaCache, needUpdateCache, needUpdateNullEntry);
this.seriesPath = seriesPath;
this.devicePath = seriesPath.getDevicePath();
}
@@ -63,32 +63,34 @@ public class AlignedUpdateLastCacheOperator extends AbstractUpdateLastCacheOpera
tsBlockBuilder.reset();
for (int i = 0; i + 1 < res.getValueColumnCount(); i += 2) {
+ MeasurementPath measurementPath =
+ new MeasurementPath(
+ devicePath.concatNode(seriesPath.getMeasurementList().get(i / 2)),
+ seriesPath.getSchemaList().get(i / 2),
+ true);
if (!res.getColumn(i).isNull(0)) {
long lastTime = res.getColumn(i).getLong(0);
TsPrimitiveType lastValue = res.getColumn(i + 1).getTsPrimitiveType(0);
- MeasurementPath measurementPath =
- new MeasurementPath(
- devicePath.concatNode(seriesPath.getMeasurementList().get(i / 2)),
- seriesPath.getSchemaList().get(i / 2),
- true);
- if (needUpdateCache) {
- TimeValuePair timeValuePair = new TimeValuePair(lastTime, lastValue);
- lastCache.updateLastCache(
- getDatabaseName(), measurementPath, timeValuePair, false, Long.MIN_VALUE);
+ mayUpdateLastCache(lastTime, lastValue, measurementPath);
+ appendLastValueToTsBlockBuilder(
+ lastTime,
+ lastValue,
+ measurementPath,
+ seriesPath.getSchemaList().get(i / 2).getType().name());
+ } else {
+ // we still need to update last cache if there is no data for this time series to avoid
+ // scanning all files each time
+ if (needUpdateNullEntry) {
+ mayUpdateLastCache(Long.MIN_VALUE, null, measurementPath);
}
- appendLastValueToTsBlockBuilder(lastTime, lastValue, measurementPath);
}
}
return !tsBlockBuilder.isEmpty() ? tsBlockBuilder.build() : LAST_QUERY_EMPTY_TSBLOCK;
}
protected void appendLastValueToTsBlockBuilder(
- long lastTime, TsPrimitiveType lastValue, MeasurementPath measurementPath) {
+ long lastTime, TsPrimitiveType lastValue, MeasurementPath measurementPath, String type) {
LastQueryUtil.appendLastValue(
- tsBlockBuilder,
- lastTime,
- measurementPath.getFullPath(),
- lastValue.getStringValue(),
- measurementPath.getSeriesType().name());
+ tsBlockBuilder, lastTime, measurementPath.getFullPath(), lastValue.getStringValue(), type);
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/AlignedUpdateViewPathLastCacheOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/AlignedUpdateViewPathLastCacheOperator.java
index 2275617ef8d..d72a9cc73b8 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/AlignedUpdateViewPathLastCacheOperator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/AlignedUpdateViewPathLastCacheOperator.java
@@ -38,20 +38,23 @@ public class AlignedUpdateViewPathLastCacheOperator extends AlignedUpdateLastCac
AlignedPath seriesPath,
DataNodeSchemaCache dataNodeSchemaCache,
boolean needUpdateCache,
+ boolean needUpdateNullEntry,
String outputViewPath) {
- super(operatorContext, child, seriesPath, dataNodeSchemaCache, needUpdateCache);
+ super(
+ operatorContext,
+ child,
+ seriesPath,
+ dataNodeSchemaCache,
+ needUpdateCache,
+ needUpdateNullEntry);
checkArgument(seriesPath.getMeasurementList().size() == 1);
this.outputViewPath = outputViewPath;
}
@Override
protected void appendLastValueToTsBlockBuilder(
- long lastTime, TsPrimitiveType lastValue, MeasurementPath measurementPath) {
+ long lastTime, TsPrimitiveType lastValue, MeasurementPath measurementPath, String type) {
LastQueryUtil.appendLastValue(
- tsBlockBuilder,
- lastTime,
- outputViewPath,
- lastValue.getStringValue(),
- measurementPath.getSeriesType().name());
+ tsBlockBuilder, lastTime, outputViewPath, lastValue.getStringValue(), type);
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/LastQueryUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/LastQueryUtil.java
index d1b81c58ecc..2e62f2d06c7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/LastQueryUtil.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/LastQueryUtil.java
@@ -169,6 +169,10 @@ public class LastQueryUtil {
|| (timeFilter instanceof Gt);
}
+ public static boolean needUpdateNullEntry(Filter timeFilter) {
+ return CACHE_ENABLED && timeFilter == null;
+ }
+
public static class LastEntry {
private final long time;
private final Binary timeSeries;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/UpdateLastCacheOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/UpdateLastCacheOperator.java
index e44ad7fd000..ae4e58f09fe 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/UpdateLastCacheOperator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/UpdateLastCacheOperator.java
@@ -24,7 +24,6 @@ import org.apache.iotdb.db.queryengine.execution.operator.Operator;
import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext;
import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeSchemaCache;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.read.TimeValuePair;
import org.apache.iotdb.tsfile.read.common.block.TsBlock;
import org.apache.iotdb.tsfile.utils.TsPrimitiveType;
@@ -46,8 +45,9 @@ public class UpdateLastCacheOperator extends AbstractUpdateLastCacheOperator {
MeasurementPath fullPath,
TSDataType dataType,
DataNodeSchemaCache dataNodeSchemaCache,
- boolean needUpdateCache) {
- super(operatorContext, child, dataNodeSchemaCache, needUpdateCache);
+ boolean needUpdateCache,
+ boolean isNeedUpdateNullEntry) {
+ super(operatorContext, child, dataNodeSchemaCache, needUpdateCache, isNeedUpdateNullEntry);
this.fullPath = fullPath;
this.dataType = dataType.name();
}
@@ -66,16 +66,18 @@ public class UpdateLastCacheOperator extends AbstractUpdateLastCacheOperator {
// last value is null
if (res.getColumn(0).isNull(0)) {
+ // we still need to update last cache if there is no data for this time series to avoid
+ // scanning all files each time
+ if (needUpdateNullEntry) {
+ mayUpdateLastCache(Long.MIN_VALUE, null, fullPath);
+ }
return LAST_QUERY_EMPTY_TSBLOCK;
}
long lastTime = res.getColumn(0).getLong(0);
TsPrimitiveType lastValue = res.getColumn(1).getTsPrimitiveType(0);
- if (needUpdateCache) {
- TimeValuePair timeValuePair = new TimeValuePair(lastTime, lastValue);
- lastCache.updateLastCache(getDatabaseName(), fullPath, timeValuePair, false, Long.MIN_VALUE);
- }
+ mayUpdateLastCache(lastTime, lastValue, fullPath);
tsBlockBuilder.reset();
appendLastValueToTsBlockBuilder(lastTime, lastValue);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/UpdateViewPathLastCacheOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/UpdateViewPathLastCacheOperator.java
index 0ec603f0f94..02d0344ec4e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/UpdateViewPathLastCacheOperator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/UpdateViewPathLastCacheOperator.java
@@ -37,8 +37,16 @@ public class UpdateViewPathLastCacheOperator extends UpdateLastCacheOperator {
TSDataType dataType,
DataNodeSchemaCache dataNodeSchemaCache,
boolean needUpdateCache,
+ boolean needUpdateNullEntry,
String outputViewPath) {
- super(operatorContext, child, fullPath, dataType, dataNodeSchemaCache, needUpdateCache);
+ super(
+ operatorContext,
+ child,
+ fullPath,
+ dataType,
+ dataNodeSchemaCache,
+ needUpdateCache,
+ needUpdateNullEntry);
this.outputViewPath = outputViewPath;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/LastCacheContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/LastCacheContainer.java
index c3d2c73d207..5d4299df38e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/LastCacheContainer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/LastCacheContainer.java
@@ -41,8 +41,15 @@ public class LastCacheContainer implements ILastCacheContainer {
@Override
public synchronized void updateCachedLast(
TimeValuePair timeValuePair, boolean highPriorityUpdate, Long latestFlushedTime) {
- if (timeValuePair == null || timeValuePair.getValue() == null) {
- return;
+ if (highPriorityUpdate) { // for write, we won't cache null value
+ if (timeValuePair == null || timeValuePair.getValue() == null) {
+ return;
+ }
+ } else { // for read, we need to cache null value, because it means that there is no data for
+ // this time series
+ if (timeValuePair == null) {
+ return;
+ }
}
if (lastCacheValue == null) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java
index 077b7aeaab5..15e2f8582b3 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java
@@ -24,6 +24,7 @@ import org.apache.iotdb.db.queryengine.execution.driver.DataDriverContext;
import org.apache.iotdb.db.queryengine.execution.driver.DriverContext;
import org.apache.iotdb.db.queryengine.execution.driver.SchemaDriverContext;
import org.apache.iotdb.db.queryengine.execution.exchange.sink.ISink;
+import org.apache.iotdb.db.queryengine.execution.fragment.DataNodeQueryContext;
import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext;
import org.apache.iotdb.db.queryengine.execution.operator.Operator;
import org.apache.iotdb.db.queryengine.execution.operator.source.ExchangeOperator;
@@ -78,10 +79,15 @@ public class LocalExecutionPlanContext {
private Filter lastQueryTimeFilter;
// whether we need to update last cache
private boolean needUpdateLastCache;
+ private boolean needUpdateNullEntry;
+
+ public final DataNodeQueryContext dataNodeQueryContext;
// for data region
public LocalExecutionPlanContext(
- TypeProvider typeProvider, FragmentInstanceContext instanceContext) {
+ TypeProvider typeProvider,
+ FragmentInstanceContext instanceContext,
+ DataNodeQueryContext dataNodeQueryContext) {
this.typeProvider = typeProvider;
this.allSensorsMap = new ConcurrentHashMap<>();
this.dataRegionTTL = instanceContext.getDataRegion().getDataTTL();
@@ -89,6 +95,7 @@ public class LocalExecutionPlanContext {
this.nextPipelineId = new AtomicInteger(0);
this.driverContext = new DataDriverContext(instanceContext, getNextPipelineId());
this.pipelineDriverFactories = new ArrayList<>();
+ this.dataNodeQueryContext = dataNodeQueryContext;
}
// For creating subContext, differ from parent context mainly in driver context
@@ -105,6 +112,7 @@ public class LocalExecutionPlanContext {
this.cachedDataTypes = parentContext.cachedDataTypes;
this.driverContext =
parentContext.getDriverContext().createSubDriverContext(getNextPipelineId());
+ this.dataNodeQueryContext = parentContext.dataNodeQueryContext;
}
// for schema region
@@ -120,6 +128,7 @@ public class LocalExecutionPlanContext {
this.driverContext =
new SchemaDriverContext(instanceContext, schemaRegion, getNextPipelineId());
this.pipelineDriverFactories = new ArrayList<>();
+ this.dataNodeQueryContext = null;
}
public void addPipelineDriverFactory(
@@ -269,6 +278,14 @@ public class LocalExecutionPlanContext {
return needUpdateLastCache;
}
+ public boolean isNeedUpdateNullEntry() {
+ return needUpdateNullEntry;
+ }
+
+ public void setNeedUpdateNullEntry(boolean needUpdateNullEntry) {
+ this.needUpdateNullEntry = needUpdateNullEntry;
+ }
+
public long getDataRegionTTL() {
return dataRegionTTL;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java
index 750c8191d29..b875ee70905 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java
@@ -20,9 +20,9 @@ package org.apache.iotdb.db.queryengine.plan.planner;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.queryengine.exception.MemoryNotEnoughException;
import org.apache.iotdb.db.queryengine.execution.driver.DataDriverContext;
+import org.apache.iotdb.db.queryengine.execution.fragment.DataNodeQueryContext;
import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext;
import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceStateMachine;
import org.apache.iotdb.db.queryengine.execution.operator.Operator;
@@ -59,9 +59,13 @@ public class LocalExecutionPlanner {
}
public List<PipelineDriverFactory> plan(
- PlanNode plan, TypeProvider types, FragmentInstanceContext instanceContext)
- throws MemoryNotEnoughException, QueryProcessException {
- LocalExecutionPlanContext context = new LocalExecutionPlanContext(types, instanceContext);
+ PlanNode plan,
+ TypeProvider types,
+ FragmentInstanceContext instanceContext,
+ DataNodeQueryContext dataNodeQueryContext)
+ throws MemoryNotEnoughException {
+ LocalExecutionPlanContext context =
+ new LocalExecutionPlanContext(types, instanceContext, dataNodeQueryContext);
// Generate pipelines, return the last pipeline data structure
// TODO Replace operator with operatorFactory to build multiple driver for one pipeline
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java
index 063a03358dd..66747e40685 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java
@@ -1999,9 +1999,23 @@ public class OperatorTreeGenerator extends PlanVisitor<Operator, LocalExecutionP
@Override
public Operator visitLastQueryScan(LastQueryScanNode node, LocalExecutionPlanContext context) {
PartialPath seriesPath = node.getSeriesPath().transformToPartialPath();
- TimeValuePair timeValuePair = DATA_NODE_SCHEMA_CACHE.getLastCache(seriesPath);
+ TimeValuePair timeValuePair = null;
+ try {
+ context.dataNodeQueryContext.lock();
+ if (!context.dataNodeQueryContext.unCached(seriesPath)) {
+ timeValuePair = DATA_NODE_SCHEMA_CACHE.getLastCache(seriesPath);
+ if (timeValuePair == null) {
+ context.dataNodeQueryContext.addUnCachePath(seriesPath, node.getDataNodeSeriesScanNum());
+ }
+ }
+ } finally {
+ context.dataNodeQueryContext.unLock();
+ }
+
if (timeValuePair == null) { // last value is not cached
return createUpdateLastCacheOperator(node, context, node.getSeriesPath());
+ } else if (timeValuePair.getValue() == null) { // there is no data for this time series
+ return null;
} else if (!LastQueryUtil.satisfyFilter(
updateFilterUsingTTL(context.getLastQueryTimeFilter(), context.getDataRegionTTL()),
timeValuePair)) { // cached last value is not satisfied
@@ -2039,7 +2053,8 @@ public class OperatorTreeGenerator extends PlanVisitor<Operator, LocalExecutionP
fullPath,
node.getSeriesPath().getSeriesType(),
DATA_NODE_SCHEMA_CACHE,
- context.isNeedUpdateLastCache());
+ context.isNeedUpdateLastCache(),
+ context.isNeedUpdateNullEntry());
} else {
OperatorContext operatorContext =
context
@@ -2056,6 +2071,7 @@ public class OperatorTreeGenerator extends PlanVisitor<Operator, LocalExecutionP
node.getSeriesPath().getSeriesType(),
DATA_NODE_SCHEMA_CACHE,
context.isNeedUpdateLastCache(),
+ context.isNeedUpdateNullEntry(),
node.getOutputViewPath());
}
}
@@ -2079,7 +2095,8 @@ public class OperatorTreeGenerator extends PlanVisitor<Operator, LocalExecutionP
lastQueryScan,
unCachedPath,
DATA_NODE_SCHEMA_CACHE,
- context.isNeedUpdateLastCache());
+ context.isNeedUpdateLastCache(),
+ context.isNeedUpdateNullEntry());
} else {
OperatorContext operatorContext =
context
@@ -2095,6 +2112,7 @@ public class OperatorTreeGenerator extends PlanVisitor<Operator, LocalExecutionP
unCachedPath,
DATA_NODE_SCHEMA_CACHE,
context.isNeedUpdateLastCache(),
+ context.isNeedUpdateNullEntry(),
node.getOutputViewPath());
}
}
@@ -2191,9 +2209,24 @@ public class OperatorTreeGenerator extends PlanVisitor<Operator, LocalExecutionP
List<String> measurementList = alignedPath.getMeasurementList();
for (int i = 0; i < measurementList.size(); i++) {
PartialPath measurementPath = devicePath.concatNode(measurementList.get(i));
- TimeValuePair timeValuePair = DATA_NODE_SCHEMA_CACHE.getLastCache(measurementPath);
+ TimeValuePair timeValuePair = null;
+ try {
+ context.dataNodeQueryContext.lock();
+ if (!context.dataNodeQueryContext.unCached(measurementPath)) {
+ timeValuePair = DATA_NODE_SCHEMA_CACHE.getLastCache(measurementPath);
+ if (timeValuePair == null) {
+ context.dataNodeQueryContext.addUnCachePath(
+ measurementPath, node.getDataNodeSeriesScanNum());
+ }
+ }
+ } finally {
+ context.dataNodeQueryContext.unLock();
+ }
+
if (timeValuePair == null) { // last value is not cached
unCachedMeasurementIndexes.add(i);
+ } else if (timeValuePair.getValue() == null) {
+ // there is no data for this time series, just ignore
} else if (!LastQueryUtil.satisfyFilter(
updateFilterUsingTTL(context.getLastQueryTimeFilter(), context.getDataRegionTTL()),
timeValuePair)) { // cached last value is not satisfied
@@ -2229,6 +2262,7 @@ public class OperatorTreeGenerator extends PlanVisitor<Operator, LocalExecutionP
context.setLastQueryTimeFilter(node.getTimeFilter());
context.setNeedUpdateLastCache(LastQueryUtil.needUpdateCache(node.getTimeFilter()));
+ context.setNeedUpdateNullEntry(LastQueryUtil.needUpdateNullEntry(node.getTimeFilter()));
List<AbstractUpdateLastCacheOperator> operatorList =
node.getChildren().stream()
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/SimpleFragmentParallelPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/SimpleFragmentParallelPlanner.java
index ef7842a892d..2730d858d15 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/SimpleFragmentParallelPlanner.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/SimpleFragmentParallelPlanner.java
@@ -36,8 +36,10 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.ExchangeNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.sink.MultiChildrenSinkNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.LastSeriesSourceNode;
import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement;
import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement;
+import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.utils.Pair;
@@ -49,6 +51,7 @@ import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
/**
* A simple implementation of IFragmentParallelPlaner. This planner will transform one PlanFragment
@@ -57,15 +60,18 @@ import java.util.Map;
public class SimpleFragmentParallelPlanner implements IFragmentParallelPlaner {
private static final Logger logger = LoggerFactory.getLogger(SimpleFragmentParallelPlanner.class);
- private SubPlan subPlan;
- private Analysis analysis;
- private MPPQueryContext queryContext;
+ private final SubPlan subPlan;
+ private final Analysis analysis;
+ private final MPPQueryContext queryContext;
// Record all the FragmentInstances belonged to same PlanFragment
- Map<PlanFragmentId, FragmentInstance> instanceMap;
+ private final Map<PlanFragmentId, FragmentInstance> instanceMap;
// Record which PlanFragment the PlanNode belongs
- Map<PlanNodeId, Pair<PlanFragmentId, PlanNode>> planNodeMap;
- List<FragmentInstance> fragmentInstanceList;
+ private final Map<PlanNodeId, Pair<PlanFragmentId, PlanNode>> planNodeMap;
+ private final List<FragmentInstance> fragmentInstanceList;
+
+ // Record FragmentInstances dispatched to same DataNode
+ private final Map<TDataNodeLocation, List<FragmentInstance>> dataNodeFIMap;
public SimpleFragmentParallelPlanner(
SubPlan subPlan, Analysis analysis, MPPQueryContext context) {
@@ -75,6 +81,7 @@ public class SimpleFragmentParallelPlanner implements IFragmentParallelPlaner {
this.instanceMap = new HashMap<>();
this.planNodeMap = new HashMap<>();
this.fragmentInstanceList = new ArrayList<>();
+ this.dataNodeFIMap = new HashMap<>();
}
@Override
@@ -90,6 +97,40 @@ public class SimpleFragmentParallelPlanner implements IFragmentParallelPlaner {
recordPlanNodeRelation(fragment.getPlanNodeTree(), fragment.getId());
produceFragmentInstance(fragment);
}
+ fragmentInstanceList.forEach(
+ fragmentInstance ->
+ fragmentInstance.setDataNodeFINum(
+ dataNodeFIMap.get(fragmentInstance.getHostDataNode()).size()));
+
+ // compute dataNodeSeriesScanNum in LastQueryScanNode
+ if (analysis.getStatement().isQuery()
+ && ((QueryStatement) analysis.getStatement()).isLastQuery()) {
+ final Map<Path, AtomicInteger> pathSumMap = new HashMap<>();
+ dataNodeFIMap
+ .values()
+ .forEach(
+ fragmentInstances -> {
+ fragmentInstances.forEach(
+ fragmentInstance ->
+ updateScanNum(
+ fragmentInstance.getFragment().getPlanNodeTree(), pathSumMap));
+ pathSumMap.clear();
+ });
+ }
+ }
+
+ private void updateScanNum(PlanNode planNode, Map<Path, AtomicInteger> pathSumMap) {
+ if (planNode instanceof LastSeriesSourceNode) {
+ LastSeriesSourceNode lastSeriesSourceNode = (LastSeriesSourceNode) planNode;
+ pathSumMap.merge(
+ lastSeriesSourceNode.getSeriesPath(),
+ lastSeriesSourceNode.getDataNodeSeriesScanNum(),
+ (k, v) -> {
+ v.incrementAndGet();
+ return v;
+ });
+ }
+ planNode.getChildren().forEach(node -> updateScanNum(node, pathSumMap));
}
private void produceFragmentInstance(PlanFragment fragment) {
@@ -131,6 +172,16 @@ public class SimpleFragmentParallelPlanner implements IFragmentParallelPlaner {
fragmentInstance.setHostDataNode(selectTargetDataNode(regionReplicaSet));
}
+ dataNodeFIMap.compute(
+ fragmentInstance.getHostDataNode(),
+ (k, v) -> {
+ if (v == null) {
+ v = new ArrayList<>();
+ }
+ v.add(fragmentInstance);
+ return v;
+ });
+
if (analysis.getStatement() instanceof QueryStatement
|| analysis.getStatement() instanceof ShowQueriesStatement) {
fragmentInstance.getFragment().generateTypeProvider(queryContext.getTypeProvider());
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/FragmentInstance.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/FragmentInstance.java
index 41da475b13a..2a59e9f0158 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/FragmentInstance.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/FragmentInstance.java
@@ -69,6 +69,9 @@ public class FragmentInstance implements IConsensusRequest {
private final SessionInfo sessionInfo;
+ // The num of all FI on the dispatched DataNode in this query
+ private int dataNodeFINum;
+
// We can add some more params for a specific FragmentInstance
// So that we can make different FragmentInstance owns different data range.
@@ -100,6 +103,18 @@ public class FragmentInstance implements IConsensusRequest {
this.isRoot = isRoot;
}
+ public FragmentInstance(
+ PlanFragment fragment,
+ FragmentInstanceId id,
+ Filter timeFilter,
+ QueryType type,
+ long timeOut,
+ SessionInfo sessionInfo,
+ int dataNodeFINum) {
+ this(fragment, id, timeFilter, type, timeOut, sessionInfo);
+ this.dataNodeFINum = dataNodeFINum;
+ }
+
public void setExecutorAndHost(ExecutorType executorType) {
if (executorType == null) {
return;
@@ -151,6 +166,14 @@ public class FragmentInstance implements IConsensusRequest {
return type;
}
+ public int getDataNodeFINum() {
+ return dataNodeFINum;
+ }
+
+ public void setDataNodeFINum(int dataNodeFINum) {
+ this.dataNodeFINum = dataNodeFINum;
+ }
+
public String toString() {
StringBuilder ret = new StringBuilder();
ret.append(String.format("FragmentInstance-%s:", getId()));
@@ -179,8 +202,10 @@ public class FragmentInstance implements IConsensusRequest {
boolean hasTimeFilter = ReadWriteIOUtils.readBool(buffer);
Filter timeFilter = hasTimeFilter ? FilterFactory.deserialize(buffer) : null;
QueryType queryType = QueryType.values()[ReadWriteIOUtils.readInt(buffer)];
+ int dataNodeFINum = ReadWriteIOUtils.readInt(buffer);
FragmentInstance fragmentInstance =
- new FragmentInstance(planFragment, id, timeFilter, queryType, timeOut, sessionInfo);
+ new FragmentInstance(
+ planFragment, id, timeFilter, queryType, timeOut, sessionInfo, dataNodeFINum);
boolean hasHostDataNode = ReadWriteIOUtils.readBool(buffer);
fragmentInstance.hostDataNode =
hasHostDataNode ? ThriftCommonsSerDeUtils.deserializeTDataNodeLocation(buffer) : null;
@@ -202,6 +227,7 @@ public class FragmentInstance implements IConsensusRequest {
timeFilter.serialize(outputStream);
}
ReadWriteIOUtils.write(type.ordinal(), outputStream);
+ ReadWriteIOUtils.write(dataNodeFINum, outputStream);
ReadWriteIOUtils.write(hostDataNode != null, outputStream);
if (hostDataNode != null) {
ThriftCommonsSerDeUtils.serializeTDataNodeLocation(hostDataNode, outputStream);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/source/AlignedLastQueryScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/source/AlignedLastQueryScanNode.java
index 07876f9f909..60648d27a2b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/source/AlignedLastQueryScanNode.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/source/AlignedLastQueryScanNode.java
@@ -38,10 +38,11 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.List;
import java.util.Objects;
+import java.util.concurrent.atomic.AtomicInteger;
import static org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.LastQueryScanNode.LAST_QUERY_HEADER_COLUMNS;
-public class AlignedLastQueryScanNode extends SeriesSourceNode {
+public class AlignedLastQueryScanNode extends LastSeriesSourceNode {
// The path of the target series which will be scanned.
private final AlignedPath seriesPath;
@@ -51,7 +52,7 @@ public class AlignedLastQueryScanNode extends SeriesSourceNode {
private final String outputViewPath;
public AlignedLastQueryScanNode(PlanNodeId id, AlignedPath seriesPath, String outputViewPath) {
- super(id);
+ super(id, new AtomicInteger(1));
this.seriesPath = seriesPath;
this.outputViewPath = outputViewPath;
}
@@ -59,14 +60,25 @@ public class AlignedLastQueryScanNode extends SeriesSourceNode {
public AlignedLastQueryScanNode(
PlanNodeId id,
AlignedPath seriesPath,
+ AtomicInteger dataNodeSeriesScanNum,
String outputViewPath,
TRegionReplicaSet regionReplicaSet) {
- super(id);
+ super(id, dataNodeSeriesScanNum);
this.seriesPath = seriesPath;
this.outputViewPath = outputViewPath;
this.regionReplicaSet = regionReplicaSet;
}
+ public AlignedLastQueryScanNode(
+ PlanNodeId id,
+ AlignedPath seriesPath,
+ AtomicInteger dataNodeSeriesScanNum,
+ String outputViewPath) {
+ super(id, dataNodeSeriesScanNum);
+ this.seriesPath = seriesPath;
+ this.outputViewPath = outputViewPath;
+ }
+
public String getOutputViewPath() {
return outputViewPath;
}
@@ -104,7 +116,7 @@ public class AlignedLastQueryScanNode extends SeriesSourceNode {
@Override
public PlanNode clone() {
return new AlignedLastQueryScanNode(
- getPlanNodeId(), seriesPath, outputViewPath, regionReplicaSet);
+ getPlanNodeId(), seriesPath, getDataNodeSeriesScanNum(), outputViewPath, regionReplicaSet);
}
@Override
@@ -157,6 +169,7 @@ public class AlignedLastQueryScanNode extends SeriesSourceNode {
protected void serializeAttributes(ByteBuffer byteBuffer) {
PlanNodeType.ALIGNED_LAST_QUERY_SCAN.serialize(byteBuffer);
seriesPath.serialize(byteBuffer);
+ ReadWriteIOUtils.write(getDataNodeSeriesScanNum().get(), byteBuffer);
ReadWriteIOUtils.write(outputViewPath == null, byteBuffer);
if (outputViewPath != null) {
ReadWriteIOUtils.write(outputViewPath, byteBuffer);
@@ -167,6 +180,7 @@ public class AlignedLastQueryScanNode extends SeriesSourceNode {
protected void serializeAttributes(DataOutputStream stream) throws IOException {
PlanNodeType.ALIGNED_LAST_QUERY_SCAN.serialize(stream);
seriesPath.serialize(stream);
+ ReadWriteIOUtils.write(getDataNodeSeriesScanNum().get(), stream);
ReadWriteIOUtils.write(outputViewPath == null, stream);
if (outputViewPath != null) {
ReadWriteIOUtils.write(outputViewPath, stream);
@@ -175,10 +189,12 @@ public class AlignedLastQueryScanNode extends SeriesSourceNode {
public static AlignedLastQueryScanNode deserialize(ByteBuffer byteBuffer) {
AlignedPath partialPath = (AlignedPath) PathDeserializeUtil.deserialize(byteBuffer);
+ int dataNodeSeriesScanNum = ReadWriteIOUtils.readInt(byteBuffer);
boolean isNull = ReadWriteIOUtils.readBool(byteBuffer);
String outputPathSymbol = isNull ? null : ReadWriteIOUtils.readString(byteBuffer);
PlanNodeId planNodeId = PlanNodeId.deserialize(byteBuffer);
- return new AlignedLastQueryScanNode(planNodeId, partialPath, outputPathSymbol);
+ return new AlignedLastQueryScanNode(
+ planNodeId, partialPath, new AtomicInteger(dataNodeSeriesScanNum), outputPathSymbol);
}
public AlignedPath getSeriesPath() {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/source/LastQueryScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/source/LastQueryScanNode.java
index c180e43bfc0..a15302b0dd3 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/source/LastQueryScanNode.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/source/LastQueryScanNode.java
@@ -38,8 +38,9 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.List;
import java.util.Objects;
+import java.util.concurrent.atomic.AtomicInteger;
-public class LastQueryScanNode extends SeriesSourceNode {
+public class LastQueryScanNode extends LastSeriesSourceNode {
public static final List<String> LAST_QUERY_HEADER_COLUMNS =
ImmutableList.of(
@@ -56,7 +57,7 @@ public class LastQueryScanNode extends SeriesSourceNode {
private TRegionReplicaSet regionReplicaSet;
public LastQueryScanNode(PlanNodeId id, MeasurementPath seriesPath, String outputViewPath) {
- super(id);
+ super(id, new AtomicInteger(1));
this.seriesPath = seriesPath;
this.outputViewPath = outputViewPath;
}
@@ -64,9 +65,20 @@ public class LastQueryScanNode extends SeriesSourceNode {
public LastQueryScanNode(
PlanNodeId id,
MeasurementPath seriesPath,
+ AtomicInteger dataNodeSeriesScanNum,
+ String outputViewPath) {
+ super(id, dataNodeSeriesScanNum);
+ this.seriesPath = seriesPath;
+ this.outputViewPath = outputViewPath;
+ }
+
+ public LastQueryScanNode(
+ PlanNodeId id,
+ MeasurementPath seriesPath,
+ AtomicInteger dataNodeSeriesScanNum,
String outputViewPath,
TRegionReplicaSet regionReplicaSet) {
- super(id);
+ super(id, dataNodeSeriesScanNum);
this.seriesPath = seriesPath;
this.outputViewPath = outputViewPath;
this.regionReplicaSet = regionReplicaSet;
@@ -115,7 +127,8 @@ public class LastQueryScanNode extends SeriesSourceNode {
@Override
public PlanNode clone() {
- return new LastQueryScanNode(getPlanNodeId(), seriesPath, outputViewPath, regionReplicaSet);
+ return new LastQueryScanNode(
+ getPlanNodeId(), seriesPath, getDataNodeSeriesScanNum(), outputViewPath, regionReplicaSet);
}
@Override
@@ -162,6 +175,7 @@ public class LastQueryScanNode extends SeriesSourceNode {
protected void serializeAttributes(ByteBuffer byteBuffer) {
PlanNodeType.LAST_QUERY_SCAN.serialize(byteBuffer);
seriesPath.serialize(byteBuffer);
+ ReadWriteIOUtils.write(getDataNodeSeriesScanNum().get(), byteBuffer);
ReadWriteIOUtils.write(outputViewPath == null, byteBuffer);
if (outputViewPath != null) {
ReadWriteIOUtils.write(outputViewPath, byteBuffer);
@@ -172,6 +186,7 @@ public class LastQueryScanNode extends SeriesSourceNode {
protected void serializeAttributes(DataOutputStream stream) throws IOException {
PlanNodeType.LAST_QUERY_SCAN.serialize(stream);
seriesPath.serialize(stream);
+ ReadWriteIOUtils.write(getDataNodeSeriesScanNum().get(), stream);
ReadWriteIOUtils.write(outputViewPath == null, stream);
if (outputViewPath != null) {
ReadWriteIOUtils.write(outputViewPath, stream);
@@ -180,10 +195,12 @@ public class LastQueryScanNode extends SeriesSourceNode {
public static LastQueryScanNode deserialize(ByteBuffer byteBuffer) {
MeasurementPath partialPath = (MeasurementPath) PathDeserializeUtil.deserialize(byteBuffer);
+ int dataNodeSeriesScanNum = ReadWriteIOUtils.readInt(byteBuffer);
boolean isNull = ReadWriteIOUtils.readBool(byteBuffer);
String outputPathSymbol = isNull ? null : ReadWriteIOUtils.readString(byteBuffer);
PlanNodeId planNodeId = PlanNodeId.deserialize(byteBuffer);
- return new LastQueryScanNode(planNodeId, partialPath, outputPathSymbol);
+ return new LastQueryScanNode(
+ planNodeId, partialPath, new AtomicInteger(dataNodeSeriesScanNum), outputPathSymbol);
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/source/LastSeriesSourceNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/source/LastSeriesSourceNode.java
new file mode 100644
index 00000000000..e3c313f6c56
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/source/LastSeriesSourceNode.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.planner.plan.node.source;
+
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId;
+
+import java.util.Objects;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public abstract class LastSeriesSourceNode extends SeriesSourceNode {
+ // The num of same series LastQueryScanNode on dispatched DataNode,
+ // use reference value here to avoid iterate twice.
+ private final AtomicInteger dataNodeSeriesScanNum;
+
+ protected LastSeriesSourceNode(PlanNodeId id, AtomicInteger dataNodeSeriesScanNum) {
+ super(id);
+ this.dataNodeSeriesScanNum = dataNodeSeriesScanNum;
+ }
+
+ public AtomicInteger getDataNodeSeriesScanNum() {
+ return dataNodeSeriesScanNum;
+ }
+
+ public abstract PartialPath getSeriesPath();
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ if (!super.equals(o)) {
+ return false;
+ }
+ LastSeriesSourceNode that = (LastSeriesSourceNode) o;
+ return dataNodeSeriesScanNum.get() == that.dataNodeSeriesScanNum.get();
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), dataNodeSeriesScanNum);
+ }
+}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/LastQueryOperatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/LastQueryOperatorTest.java
index 3dca166ea75..5986c80dc39 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/LastQueryOperatorTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/LastQueryOperatorTest.java
@@ -155,6 +155,7 @@ public class LastQueryOperatorTest {
measurementPath1,
measurementPath1.getSeriesType(),
null,
+ false,
false);
SeriesAggregationScanOperator seriesAggregationScanOperator2 =
@@ -178,6 +179,7 @@ public class LastQueryOperatorTest {
measurementPath2,
measurementPath2.getSeriesType(),
null,
+ false,
false);
LastQueryOperator lastQueryOperator =
@@ -279,6 +281,7 @@ public class LastQueryOperatorTest {
measurementPath1,
measurementPath1.getSeriesType(),
null,
+ false,
false);
SeriesAggregationScanOperator seriesAggregationScanOperator2 =
@@ -302,6 +305,7 @@ public class LastQueryOperatorTest {
measurementPath2,
measurementPath2.getSeriesType(),
null,
+ false,
false);
TsBlockBuilder builder = LastQueryUtil.createTsBlockBuilder(6);
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/LastQuerySortOperatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/LastQuerySortOperatorTest.java
index 5323f03d540..92fd0720d2a 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/LastQuerySortOperatorTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/LastQuerySortOperatorTest.java
@@ -155,6 +155,7 @@ public class LastQuerySortOperatorTest {
measurementPath1,
measurementPath1.getSeriesType(),
null,
+ false,
false);
SeriesAggregationScanOperator seriesAggregationScanOperator2 =
@@ -178,6 +179,7 @@ public class LastQuerySortOperatorTest {
measurementPath2,
measurementPath2.getSeriesType(),
null,
+ false,
false);
LastQuerySortOperator lastQuerySortOperator =
@@ -279,6 +281,7 @@ public class LastQuerySortOperatorTest {
measurementPath1,
measurementPath1.getSeriesType(),
null,
+ false,
false);
SeriesAggregationScanOperator seriesAggregationScanOperator2 =
@@ -302,6 +305,7 @@ public class LastQuerySortOperatorTest {
measurementPath2,
measurementPath2.getSeriesType(),
null,
+ false,
false);
TsBlockBuilder builder = LastQueryUtil.createTsBlockBuilder(6);
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorMemoryTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorMemoryTest.java
index b32683a4a3b..ba76afd9f58 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorMemoryTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorMemoryTest.java
@@ -529,9 +529,18 @@ public class OperatorMemoryTest {
Mockito.when(child.calculateMaxPeekMemory()).thenReturn(2048L);
Mockito.when(child.calculateMaxReturnSize()).thenReturn(1024L);
Mockito.when(child.calculateRetainedSizeAfterCallingNext()).thenReturn(512L);
-
+ OperatorContext context =
+ new OperatorContext(
+ 1,
+ Mockito.mock(PlanNodeId.class),
+ SeriesAggregationScanOperator.class.getSimpleName(),
+ Mockito.mock(DriverContext.class));
+ FragmentInstanceContext fragmentInstanceContext = Mockito.mock(FragmentInstanceContext.class);
+ Mockito.when(fragmentInstanceContext.getDataNodeQueryContext()).thenReturn(null);
+ Mockito.when(context.getDriverContext().getFragmentInstanceContext())
+ .thenReturn(fragmentInstanceContext);
UpdateLastCacheOperator updateLastCacheOperator =
- new UpdateLastCacheOperator(null, child, null, TSDataType.BOOLEAN, null, false);
+ new UpdateLastCacheOperator(context, child, null, TSDataType.BOOLEAN, null, false, false);
assertEquals(2048, updateLastCacheOperator.calculateMaxPeekMemory());
assertEquals(1024, updateLastCacheOperator.calculateMaxReturnSize());
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/UpdateLastCacheOperatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/UpdateLastCacheOperatorTest.java
index b756774bbcc..481254c1f42 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/UpdateLastCacheOperatorTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/UpdateLastCacheOperatorTest.java
@@ -232,6 +232,7 @@ public class UpdateLastCacheOperatorTest {
measurementPath,
measurementPath.getSeriesType(),
null,
+ false,
false);
}
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/plan/PipelineBuilderTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/plan/PipelineBuilderTest.java
index 50191574f46..6ac9ade2501 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/plan/PipelineBuilderTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/plan/PipelineBuilderTest.java
@@ -29,6 +29,7 @@ import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.db.queryengine.common.FragmentInstanceId;
import org.apache.iotdb.db.queryengine.common.PlanFragmentId;
import org.apache.iotdb.db.queryengine.common.QueryId;
+import org.apache.iotdb.db.queryengine.execution.fragment.DataNodeQueryContext;
import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext;
import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceStateMachine;
import org.apache.iotdb.db.queryengine.execution.operator.Operator;
@@ -811,7 +812,8 @@ public class PipelineBuilderTest {
createFragmentInstanceContext(instanceId, stateMachine);
fragmentInstanceContext.setDataRegion(dataRegion);
- return new LocalExecutionPlanContext(typeProvider, fragmentInstanceContext);
+ return new LocalExecutionPlanContext(
+ typeProvider, fragmentInstanceContext, new DataNodeQueryContext(1));
}
/**
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/plan/distribution/Util.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/plan/distribution/Util.java
index cda2cef97c2..28c8547001f 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/plan/distribution/Util.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/plan/distribution/Util.java
@@ -53,6 +53,7 @@ import org.apache.iotdb.db.queryengine.plan.parser.StatementGenerator;
import org.apache.iotdb.db.queryengine.plan.planner.LogicalPlanner;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
import org.apache.iotdb.db.queryengine.plan.statement.Statement;
+import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement;
import org.apache.iotdb.db.schemaengine.template.Template;
import org.apache.iotdb.mpp.rpc.thrift.TRegionRouteReq;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
@@ -61,6 +62,8 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
+import org.mockito.Mockito;
+
import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.Arrays;
@@ -224,6 +227,9 @@ public class Util {
analysis.setDataPartitionInfo(dataPartition);
analysis.setSchemaPartitionInfo(schemaPartition);
analysis.setSchemaTree(genSchemaTree());
+ // to avoid some special case which is not the point of test
+ analysis.setStatement(Mockito.mock(QueryStatement.class));
+ Mockito.when(analysis.getStatement().isQuery()).thenReturn(false);
return analysis;
} catch (IllegalPathException e) {
return new Analysis();
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/builder/EvaluationDAGBuilderTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/builder/EvaluationDAGBuilderTest.java
index b779a8ed676..d752a364da0 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/builder/EvaluationDAGBuilderTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/builder/EvaluationDAGBuilderTest.java
@@ -26,6 +26,7 @@ import org.apache.iotdb.db.queryengine.common.FragmentInstanceId;
import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
import org.apache.iotdb.db.queryengine.common.PlanFragmentId;
import org.apache.iotdb.db.queryengine.common.QueryId;
+import org.apache.iotdb.db.queryengine.execution.fragment.DataNodeQueryContext;
import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext;
import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceStateMachine;
import org.apache.iotdb.db.queryengine.execution.operator.Operator;
@@ -105,7 +106,8 @@ public class EvaluationDAGBuilderTest {
localExecutionPlanner.plan(
instance.getFragment().getPlanNodeTree(),
instance.getFragment().getTypeProvider(),
- mockFIContext(queryId));
+ mockFIContext(queryId),
+ new DataNodeQueryContext(1));
return driverFactories.get(0).getOperation();
} catch (Exception e) {
e.printStackTrace();