You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2022/05/31 07:53:06 UTC
[iotdb] branch query_plan_visualization updated (312a82b9c6 -> 9d9c09f41d)
This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a change to branch query_plan_visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 312a82b9c6 complete plan visualization
add 31e5893900 [IOTDB-3315] Make unset of ttl in cluster mode same as 0.13.0 version. (#6061)
add 23bb94e2e5 [IOTDB-3271] Wrong multi IP for one sender in receiver (#6037)
add bb1a88540d [ISSUES-6057] Add a method(getAllDataRegionID) to StorageEngineV2 (#6059)
add fcb27a8449 [IOTDB-3228] UDF Management in MPP Cluster: DROP FUNCTION (#6064)
add 4f101d8acf [IOTDB-3246]Improve Validation tool (#6066)
add e1dfec58e3 [IOTDB-3320] Don't return data which are not satisfied with time filter in last query (#6068)
add 88426aa3e3 [IOTDB-3319] fix IndexOutOfBoundsException when executing linear fill in group by query (#6070)
add 8500eb9cd1 Fix aligned data in memory query bug (#6065)
add c84095d093 [IOTDB-3325] Thrown an exception when using group by in raw data query (#6077)
add c01b1e1504 [IOTDB-3335] Redundant data recovered by wal in multi-leader consensus (#6076)
add 7fcb1044e7 [IOTDB-3306] Use rpc port check to avoid starting same IoTDB twice (#6069)
add 88adfbea69 [IOTDB-3229] Field is 0 rather than null when query two series in one device (#6071)
add b343478a41 [IOTDB-3339] Fix index out of bound while doing raw query (#6079)
add 48256e44f3 [IOTDB-3341] Fix timeseries id is not right while doing last query (#6082)
add 3918bf6992 [IOTDB-3346] Clear DriverTask when any RuntimeException is thrown (#6086)
add d26fa83c5d Make DataNode start DataNodeTSIServiceImpl RPC service (#6090)
add d9d2465f0d [IOTDB-3343] Aggregate existing and non-exsit timeseries together failure in AlignByDevice (#6084)
add f0fd1f5b40 [IOTDB-3165] Implement delete data (#6072)
add 1a18def1fc [IOTDB-3065]show lastest timeseries (#6063)
add ecbea664df optimize. (#6073)
new f2521616cf Merge branch 'master' into query_plan_visualization
new 9d9c09f41d add more box printer for PlanNodes
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../confignode/client/AsyncDataNodeClientPool.java | 19 ++-
...Handler.java => FunctionManagementHandler.java} | 6 +-
.../consensus/request/ConfigRequest.java | 4 +
.../consensus/request/ConfigRequestType.java | 1 +
...eleteProcedureReq.java => DropFunctionReq.java} | 38 ++---
.../iotdb/confignode/manager/ConfigManager.java | 8 +
.../apache/iotdb/confignode/manager/Manager.java | 2 +
.../iotdb/confignode/manager/UDFManager.java | 59 +++++++-
.../iotdb/confignode/persistence/UDFInfo.java | 25 +++-
.../executor/ConfigRequestExecutor.java | 8 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 6 +
.../db/integration/sync/IoTDBSyncReceiverIT.java | 5 +-
.../apache/iotdb/commons/conf/IoTDBConstant.java | 1 -
.../iotdb/commons/partition/DataPartition.java | 5 +-
.../commons/partition/DataPartitionQueryParam.java | 16 ++
.../apache/iotdb/commons/partition/Partition.java | 4 +
.../commons/partition/RegionReplicaSetInfo.java | 16 +-
.../iotdb/commons/partition/SchemaPartition.java | 5 +-
.../service/AbstractThriftServiceThread.java | 38 +----
.../udf/service/UDFRegistrationService.java | 8 +-
.../apache/iotdb/db/client/ConfigNodeClient.java | 17 +++
.../statemachine/visitor/DataExecutionVisitor.java | 17 +++
.../apache/iotdb/db/engine/StorageEngineV2.java | 11 +-
.../iotdb/db/engine/memtable/AbstractMemTable.java | 7 +-
.../iotdb/db/engine/storagegroup/DataRegion.java | 87 ++++++++++-
.../db/engine/storagegroup/TsFileProcessor.java | 2 +-
.../SmoothQueueSlidingWindowAggregator.java | 1 +
.../iotdb/db/mpp/common/header/HeaderConstant.java | 4 +-
.../execution/operator/process/FilterOperator.java | 80 +++++-----
.../operator/process/LinearFillOperator.java | 3 +-
.../operator/process/TransformOperator.java | 70 +++++----
.../operator/process/UpdateLastCacheOperator.java | 5 +
.../operator/process/merge/SingleColumnMerger.java | 4 +-
.../operator/schema/SchemaQueryMergeOperator.java | 2 -
.../schema/SchemaQueryOrderByHeatOperator.java | 161 +++++++++++++++++++++
.../execution/schedule/AbstractDriverThread.java | 8 +-
.../db/mpp/execution/schedule/DriverScheduler.java | 34 +++--
.../schedule/FragmentInstanceAbortedException.java | 1 +
.../apache/iotdb/db/mpp/plan/analyze/Analyzer.java | 144 +++++++++---------
.../mpp/plan/analyze/ClusterPartitionFetcher.java | 7 +-
.../db/mpp/plan/analyze/ClusterSchemaFetcher.java | 4 +-
.../db/mpp/plan/analyze/ExpressionAnalyzer.java | 10 +-
.../db/mpp/plan/analyze/FakeSchemaFetcherImpl.java | 6 +
.../iotdb/db/mpp/plan/analyze/ISchemaFetcher.java | 3 +
.../mpp/plan/analyze/StandaloneSchemaFetcher.java | 6 +
.../plan/execution/config/ConfigTaskVisitor.java | 7 +
...eateFunctionTask.java => DropFunctionTask.java} | 38 ++---
.../execution/config/ShowStorageGroupTask.java | 7 +-
.../db/mpp/plan/execution/config/ShowTTLTask.java | 5 +-
.../memory/StatementMemorySourceVisitor.java | 33 +++--
.../iotdb/db/mpp/plan/parser/ASTVisitor.java | 86 ++++++++++-
.../db/mpp/plan/parser/StatementGenerator.java | 15 ++
.../db/mpp/plan/planner/LocalExecutionPlanner.java | 23 ++-
.../db/mpp/plan/planner/LogicalPlanBuilder.java | 38 ++---
.../iotdb/db/mpp/plan/planner/LogicalPlanner.java | 49 ++++---
.../planner/distribution/ExchangeNodeAdder.java | 37 +----
.../plan/planner/distribution/SourceRewriter.java | 85 -----------
.../plan/planner/plan/node/PlanGraphPrinter.java | 109 ++++++++++++++
.../mpp/plan/planner/plan/node/PlanNodeType.java | 6 +-
.../db/mpp/plan/planner/plan/node/PlanVisitor.java | 5 +
.../node/metedata/read/SchemaQueryMergeNode.java | 4 +
...ntNode.java => SchemaQueryOrderByHeatNode.java} | 51 ++++---
.../metedata/read/TimeSeriesSchemaScanNode.java | 7 +
.../planner/plan/node/write/DeleteDataNode.java | 110 ++++++++++----
.../db/mpp/plan/statement/StatementVisitor.java | 10 ++
.../DeleteDataStatement.java} | 49 ++++---
.../db/mpp/plan/statement/crud/QueryStatement.java | 4 +
...onStatement.java => DropFunctionStatement.java} | 21 +--
.../query/reader/chunk/MemAlignedPageReader.java | 10 +-
.../java/org/apache/iotdb/db/service/DataNode.java | 18 ++-
.../java/org/apache/iotdb/db/service/IoTDB.java | 9 +-
.../thrift/impl/DataNodeTSIServiceImpl.java | 56 +++++--
.../service/thrift/impl/InternalServiceImpl.java | 12 ++
.../db/sync/sender/service/TransportHandler.java | 5 +-
.../db/sync/transport/client/TransportClient.java | 21 +--
.../db/tools/validate/TsFileValidationTool.java | 13 +-
.../java/org/apache/iotdb/db/wal/WALManager.java | 15 +-
.../allocation/AbstractNodeAllocationStrategy.java | 9 ++
.../db/wal/allocation/FirstCreateStrategy.java | 4 +-
.../iotdb/db/wal/buffer/AbstractWALBuffer.java | 4 +-
.../org/apache/iotdb/db/wal/buffer/WALBuffer.java | 7 +-
.../iotdb/db/wal/checkpoint/CheckpointManager.java | 40 +++--
.../iotdb/db/wal/checkpoint/MemTableInfo.java | 1 -
.../apache/iotdb/db/wal/io/CheckpointReader.java | 22 ++-
.../java/org/apache/iotdb/db/wal/node/WALNode.java | 7 +-
.../db/wal/recover/CheckpointRecoverUtils.java | 32 +++-
.../iotdb/db/wal/recover/WALNodeRecoverTask.java | 34 ++++-
.../iotdb/db/wal/utils/CheckpointFileUtils.java | 3 +-
.../apache/iotdb/db/wal/utils/WALFileUtils.java | 30 ++--
.../iotdb/db/engine/StorageEngineV2Test.java | 71 +++++++++
.../execution/operator/LinearFillOperatorTest.java | 113 ++++++++++++++-
...erTest.java => DefaultDriverSchedulerTest.java} | 2 +-
.../execution/schedule/DriverSchedulerTest.java | 2 +-
.../DriverTaskTimeoutSentinelThreadTest.java | 43 ++++++
.../iotdb/db/mpp/plan/plan/LogicalPlannerTest.java | 6 +-
.../plan/distribution/DeleteTimeseriesTest.java | 71 ---------
.../plan/node/write/DeleteDataNodeSerdeTest.java | 21 +--
.../db/sync/transport/TransportServiceTest.java | 8 +-
.../db/wal/allocation/FirstCreateStrategyTest.java | 2 +-
.../db/wal/checkpoint/CheckpointManagerTest.java | 6 +-
.../apache/iotdb/db/wal/io/CheckpointFileTest.java | 12 +-
.../iotdb/db/wal/node/ConsensusReqReaderTest.java | 14 +-
.../org/apache/iotdb/db/wal/node/WALNodeTest.java | 2 +-
.../src/main/thrift/confignode.thrift | 8 +-
thrift/src/main/thrift/mpp.thrift | 11 ++
.../iotdb/tsfile/read/common/block/TsBlock.java | 3 +-
106 files changed, 1719 insertions(+), 795 deletions(-)
rename confignode/src/main/java/org/apache/iotdb/confignode/client/handlers/{CreateFunctionHandler.java => FunctionManagementHandler.java} (89%)
copy confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/{DeleteProcedureReq.java => DropFunctionReq.java} (62%)
create mode 100644 server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/schema/SchemaQueryOrderByHeatOperator.java
copy server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/{CreateFunctionTask.java => DropFunctionTask.java} (69%)
copy server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/metedata/read/{NodePathsCountNode.java => SchemaQueryOrderByHeatNode.java} (61%)
copy server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/{internal/LastPointFetchStatement.java => crud/DeleteDataStatement.java} (55%)
copy server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/{CreateFunctionStatement.java => DropFunctionStatement.java} (76%)
create mode 100644 server/src/test/java/org/apache/iotdb/db/engine/StorageEngineV2Test.java
rename server/src/test/java/org/apache/iotdb/db/mpp/execution/schedule/{DefaultTaskSchedulerTest.java => DefaultDriverSchedulerTest.java} (99%)
delete mode 100644 server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/distribution/DeleteTimeseriesTest.java
[iotdb] 02/02: add more box printer for PlanNodes
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch query_plan_visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 9d9c09f41da96a3e0fb624a65ecdd83da81f87ef
Author: Jinrui.Zhang <xi...@gmail.com>
AuthorDate: Tue May 31 15:52:55 2022 +0800
add more box printer for PlanNodes
---
.../memory/StatementMemorySourceVisitor.java | 33 ++++---
.../plan/planner/plan/node/PlanGraphPrinter.java | 109 +++++++++++++++++++++
2 files changed, 130 insertions(+), 12 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/memory/StatementMemorySourceVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/memory/StatementMemorySourceVisitor.java
index 17cc0a4721..4ee9aeea37 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/memory/StatementMemorySourceVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/memory/StatementMemorySourceVisitor.java
@@ -37,14 +37,11 @@ import org.apache.iotdb.db.mpp.plan.statement.sys.ExplainStatement;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.block.TsBlock;
import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
-import org.apache.iotdb.tsfile.read.common.block.column.BinaryColumn;
-import org.apache.iotdb.tsfile.read.common.block.column.TimeColumn;
import org.apache.iotdb.tsfile.utils.Binary;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import java.util.Optional;
import java.util.Set;
import java.util.TreeSet;
@@ -57,18 +54,30 @@ public class StatementMemorySourceVisitor
}
@Override
- public StatementMemorySource visitExplain(ExplainStatement node, StatementMemorySourceContext context) {
+ public StatementMemorySource visitExplain(
+ ExplainStatement node, StatementMemorySourceContext context) {
context.getAnalysis().setStatement(node.getQueryStatement());
- LogicalQueryPlan logicalPlan = new LogicalPlanner(context.getQueryContext(), new ArrayList<>()).plan(context.getAnalysis());
+ LogicalQueryPlan logicalPlan =
+ new LogicalPlanner(context.getQueryContext(), new ArrayList<>())
+ .plan(context.getAnalysis());
DistributionPlanner planner = new DistributionPlanner(context.getAnalysis(), logicalPlan);
PlanNode rootWithExchange = planner.addExchangeNode(planner.rewriteSource());
- List<String> lines = rootWithExchange.accept(new PlanGraphPrinter(), new PlanGraphPrinter.GraphContext());
- StringBuilder cellValue = new StringBuilder();
- lines.forEach(line -> cellValue.append(line).append(System.lineSeparator()));
- System.out.println(cellValue);
- BinaryColumn binaryColumn = new BinaryColumn(1, Optional.empty(), new Binary[]{new Binary(cellValue.toString())});
- TsBlock tsBlock = new TsBlock(new TimeColumn(1, new long[]{0}), binaryColumn);
- DatasetHeader header = new DatasetHeader(Collections.singletonList(new ColumnHeader(IoTDBConstant.COLUMN_DISTRIBUTION_PLAN, TSDataType.TEXT)), true);
+ List<String> lines =
+ rootWithExchange.accept(new PlanGraphPrinter(), new PlanGraphPrinter.GraphContext());
+
+ TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(TSDataType.TEXT));
+ lines.forEach(
+ line -> {
+ builder.getTimeColumnBuilder().writeLong(0L);
+ builder.getColumnBuilder(0).writeBinary(new Binary(line));
+ builder.declarePosition();
+ });
+ TsBlock tsBlock = builder.build();
+ DatasetHeader header =
+ new DatasetHeader(
+ Collections.singletonList(
+ new ColumnHeader(IoTDBConstant.COLUMN_DISTRIBUTION_PLAN, TSDataType.TEXT)),
+ true);
return new StatementMemorySource(tsBlock, header);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/PlanGraphPrinter.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/PlanGraphPrinter.java
index 5ab64d1b44..f6321b7ea5 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/PlanGraphPrinter.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/PlanGraphPrinter.java
@@ -19,11 +19,23 @@
package org.apache.iotdb.db.mpp.plan.planner.plan.node;
+import org.apache.iotdb.db.mpp.plan.planner.plan.node.process.AggregationNode;
+import org.apache.iotdb.db.mpp.plan.planner.plan.node.process.DeviceMergeNode;
+import org.apache.iotdb.db.mpp.plan.planner.plan.node.process.DeviceViewNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.process.ExchangeNode;
+import org.apache.iotdb.db.mpp.plan.planner.plan.node.process.FillNode;
+import org.apache.iotdb.db.mpp.plan.planner.plan.node.process.FilterNode;
+import org.apache.iotdb.db.mpp.plan.planner.plan.node.process.FilterNullNode;
+import org.apache.iotdb.db.mpp.plan.planner.plan.node.process.GroupByLevelNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.process.LimitNode;
+import org.apache.iotdb.db.mpp.plan.planner.plan.node.process.OffsetNode;
+import org.apache.iotdb.db.mpp.plan.planner.plan.node.process.SlidingWindowAggregationNode;
+import org.apache.iotdb.db.mpp.plan.planner.plan.node.process.SortNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.process.TimeJoinNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.sink.FragmentSinkNode;
+import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.AlignedSeriesAggregationScanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.AlignedSeriesScanNode;
+import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.SeriesAggregationScanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.SeriesScanNode;
import org.apache.commons.lang3.Validate;
@@ -74,6 +86,103 @@ public class PlanGraphPrinter extends PlanVisitor<List<String>, PlanGraphPrinter
return render(node, boxValue, context);
}
+ @Override
+ public List<String> visitSeriesAggregationScan(
+ SeriesAggregationScanNode node, GraphContext context) {
+ List<String> boxValue = new ArrayList<>();
+ boxValue.add(String.format("SeriesAggregationScan-%s", node.getPlanNodeId().getId()));
+ boxValue.add(String.format("Series: %s", node.getSeriesPath()));
+ boxValue.add(String.format("PartitionId: %s", node.getRegionReplicaSet().getRegionId().id));
+ return render(node, boxValue, context);
+ }
+
+ @Override
+ public List<String> visitAlignedSeriesAggregationScan(
+ AlignedSeriesAggregationScanNode node, GraphContext context) {
+ List<String> boxValue = new ArrayList<>();
+ boxValue.add(String.format("AlignedSeriesAggregationScan-%s", node.getPlanNodeId().getId()));
+ boxValue.add(String.format("Series: %s", node.getAlignedPath()));
+ boxValue.add(String.format("PartitionId: %s", node.getRegionReplicaSet().getRegionId().id));
+ return render(node, boxValue, context);
+ }
+
+ @Override
+ public List<String> visitDeviceView(DeviceViewNode node, GraphContext context) {
+ List<String> boxValue = new ArrayList<>();
+ boxValue.add(String.format("DeviceView-%s", node.getPlanNodeId().getId()));
+ boxValue.add(String.format("DeviceCount: %d", node.getDevices().size()));
+ return render(node, boxValue, context);
+ }
+
+ @Override
+ public List<String> visitDeviceMerge(DeviceMergeNode node, GraphContext context) {
+ List<String> boxValue = new ArrayList<>();
+ boxValue.add(String.format("DeviceMerge-%s", node.getPlanNodeId().getId()));
+ boxValue.add(String.format("DeviceCount: %d", node.getDevices().size()));
+ return render(node, boxValue, context);
+ }
+
+ @Override
+ public List<String> visitFill(FillNode node, GraphContext context) {
+ List<String> boxValue = new ArrayList<>();
+ boxValue.add(String.format("Fill-%s", node.getPlanNodeId().getId()));
+ boxValue.add(String.format("Policy: %s", node.getFillDescriptor().getFillPolicy()));
+ return render(node, boxValue, context);
+ }
+
+ @Override
+ public List<String> visitFilter(FilterNode node, GraphContext context) {
+ List<String> boxValue = new ArrayList<>();
+ boxValue.add(String.format("Filter-%s", node.getPlanNodeId().getId()));
+ boxValue.add(String.format("Predicate: %s", node.getPredicate()));
+ return render(node, boxValue, context);
+ }
+
+ @Override
+ public List<String> visitFilterNull(FilterNullNode node, GraphContext context) {
+ List<String> boxValue = new ArrayList<>();
+ boxValue.add(String.format("FilterNull-%s", node.getPlanNodeId().getId()));
+ return render(node, boxValue, context);
+ }
+
+ @Override
+ public List<String> visitGroupByLevel(GroupByLevelNode node, GraphContext context) {
+ List<String> boxValue = new ArrayList<>();
+ boxValue.add(String.format("GroupByLevel-%s", node.getPlanNodeId().getId()));
+ return render(node, boxValue, context);
+ }
+
+ @Override
+ public List<String> visitSlidingWindowAggregation(
+ SlidingWindowAggregationNode node, GraphContext context) {
+ List<String> boxValue = new ArrayList<>();
+ boxValue.add(String.format("SlidingWindowAggregation-%s", node.getPlanNodeId().getId()));
+ return render(node, boxValue, context);
+ }
+
+ @Override
+ public List<String> visitOffset(OffsetNode node, GraphContext context) {
+ List<String> boxValue = new ArrayList<>();
+ boxValue.add(String.format("Offset-%s", node.getPlanNodeId().getId()));
+ boxValue.add(String.format("value: %d", node.getOffset()));
+ return render(node, boxValue, context);
+ }
+
+ @Override
+ public List<String> visitRowBasedSeriesAggregate(AggregationNode node, GraphContext context) {
+ List<String> boxValue = new ArrayList<>();
+ boxValue.add(String.format("Aggregation-%s", node.getPlanNodeId().getId()));
+ return render(node, boxValue, context);
+ }
+
+ @Override
+ public List<String> visitSort(SortNode node, GraphContext context) {
+ List<String> boxValue = new ArrayList<>();
+ boxValue.add(String.format("Sort-%s", node.getPlanNodeId().getId()));
+ boxValue.add(String.format("OrderBy: %s", node.getSortOrder()));
+ return render(node, boxValue, context);
+ }
+
@Override
public List<String> visitExchange(ExchangeNode node, GraphContext context) {
List<String> boxValue = new ArrayList<>();
[iotdb] 01/02: Merge branch 'master' into query_plan_visualization
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch query_plan_visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit f2521616cf9049c59e9fc93c8b2bb07221f9418f
Merge: 312a82b9c6 ecbea664df
Author: Jinrui.Zhang <xi...@gmail.com>
AuthorDate: Tue May 31 15:02:19 2022 +0800
Merge branch 'master' into query_plan_visualization
.../confignode/client/AsyncDataNodeClientPool.java | 19 ++-
...Handler.java => FunctionManagementHandler.java} | 6 +-
.../consensus/request/ConfigRequest.java | 4 +
.../consensus/request/ConfigRequestType.java | 1 +
.../consensus/request/write/DropFunctionReq.java | 56 +++++++
.../iotdb/confignode/manager/ConfigManager.java | 8 +
.../apache/iotdb/confignode/manager/Manager.java | 2 +
.../iotdb/confignode/manager/UDFManager.java | 59 +++++++-
.../iotdb/confignode/persistence/UDFInfo.java | 25 +++-
.../executor/ConfigRequestExecutor.java | 8 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 6 +
.../db/integration/sync/IoTDBSyncReceiverIT.java | 5 +-
.../apache/iotdb/commons/conf/IoTDBConstant.java | 1 -
.../iotdb/commons/partition/DataPartition.java | 5 +-
.../commons/partition/DataPartitionQueryParam.java | 16 ++
.../apache/iotdb/commons/partition/Partition.java | 4 +
.../commons/partition/RegionReplicaSetInfo.java | 16 +-
.../iotdb/commons/partition/SchemaPartition.java | 5 +-
.../service/AbstractThriftServiceThread.java | 38 +----
.../udf/service/UDFRegistrationService.java | 8 +-
.../apache/iotdb/db/client/ConfigNodeClient.java | 17 +++
.../statemachine/visitor/DataExecutionVisitor.java | 17 +++
.../apache/iotdb/db/engine/StorageEngineV2.java | 11 +-
.../iotdb/db/engine/memtable/AbstractMemTable.java | 7 +-
.../iotdb/db/engine/storagegroup/DataRegion.java | 87 ++++++++++-
.../db/engine/storagegroup/TsFileProcessor.java | 2 +-
.../SmoothQueueSlidingWindowAggregator.java | 1 +
.../iotdb/db/mpp/common/header/HeaderConstant.java | 4 +-
.../execution/operator/process/FilterOperator.java | 80 +++++-----
.../operator/process/LinearFillOperator.java | 3 +-
.../operator/process/TransformOperator.java | 70 +++++----
.../operator/process/UpdateLastCacheOperator.java | 5 +
.../operator/process/merge/SingleColumnMerger.java | 4 +-
.../operator/schema/SchemaQueryMergeOperator.java | 2 -
.../schema/SchemaQueryOrderByHeatOperator.java | 161 +++++++++++++++++++++
.../execution/schedule/AbstractDriverThread.java | 8 +-
.../db/mpp/execution/schedule/DriverScheduler.java | 34 +++--
.../schedule/FragmentInstanceAbortedException.java | 1 +
.../apache/iotdb/db/mpp/plan/analyze/Analyzer.java | 144 +++++++++---------
.../mpp/plan/analyze/ClusterPartitionFetcher.java | 7 +-
.../db/mpp/plan/analyze/ClusterSchemaFetcher.java | 4 +-
.../db/mpp/plan/analyze/ExpressionAnalyzer.java | 10 +-
.../db/mpp/plan/analyze/FakeSchemaFetcherImpl.java | 6 +
.../iotdb/db/mpp/plan/analyze/ISchemaFetcher.java | 3 +
.../mpp/plan/analyze/StandaloneSchemaFetcher.java | 6 +
.../plan/execution/config/ConfigTaskVisitor.java | 7 +
.../plan/execution/config/DropFunctionTask.java | 99 +++++++++++++
.../execution/config/ShowStorageGroupTask.java | 7 +-
.../db/mpp/plan/execution/config/ShowTTLTask.java | 5 +-
.../iotdb/db/mpp/plan/parser/ASTVisitor.java | 86 ++++++++++-
.../db/mpp/plan/parser/StatementGenerator.java | 15 ++
.../db/mpp/plan/planner/LocalExecutionPlanner.java | 23 ++-
.../db/mpp/plan/planner/LogicalPlanBuilder.java | 38 ++---
.../iotdb/db/mpp/plan/planner/LogicalPlanner.java | 49 ++++---
.../planner/distribution/ExchangeNodeAdder.java | 37 +----
.../plan/planner/distribution/SourceRewriter.java | 85 -----------
.../mpp/plan/planner/plan/node/PlanNodeType.java | 6 +-
.../db/mpp/plan/planner/plan/node/PlanVisitor.java | 5 +
.../node/metedata/read/SchemaQueryMergeNode.java | 4 +
.../metedata/read/SchemaQueryOrderByHeatNode.java | 99 +++++++++++++
.../metedata/read/TimeSeriesSchemaScanNode.java | 7 +
.../planner/plan/node/write/DeleteDataNode.java | 110 ++++++++++----
.../db/mpp/plan/statement/StatementVisitor.java | 10 ++
.../plan/statement/crud/DeleteDataStatement.java | 73 ++++++++++
.../db/mpp/plan/statement/crud/QueryStatement.java | 4 +
.../statement/metadata/DropFunctionStatement.java | 60 ++++++++
.../query/reader/chunk/MemAlignedPageReader.java | 10 +-
.../java/org/apache/iotdb/db/service/DataNode.java | 18 ++-
.../java/org/apache/iotdb/db/service/IoTDB.java | 9 +-
.../thrift/impl/DataNodeTSIServiceImpl.java | 56 +++++--
.../service/thrift/impl/InternalServiceImpl.java | 12 ++
.../db/sync/sender/service/TransportHandler.java | 5 +-
.../db/sync/transport/client/TransportClient.java | 21 +--
.../db/tools/validate/TsFileValidationTool.java | 13 +-
.../java/org/apache/iotdb/db/wal/WALManager.java | 15 +-
.../allocation/AbstractNodeAllocationStrategy.java | 9 ++
.../db/wal/allocation/FirstCreateStrategy.java | 4 +-
.../iotdb/db/wal/buffer/AbstractWALBuffer.java | 4 +-
.../org/apache/iotdb/db/wal/buffer/WALBuffer.java | 7 +-
.../iotdb/db/wal/checkpoint/CheckpointManager.java | 40 +++--
.../iotdb/db/wal/checkpoint/MemTableInfo.java | 1 -
.../apache/iotdb/db/wal/io/CheckpointReader.java | 22 ++-
.../java/org/apache/iotdb/db/wal/node/WALNode.java | 7 +-
.../db/wal/recover/CheckpointRecoverUtils.java | 32 +++-
.../iotdb/db/wal/recover/WALNodeRecoverTask.java | 34 ++++-
.../iotdb/db/wal/utils/CheckpointFileUtils.java | 3 +-
.../apache/iotdb/db/wal/utils/WALFileUtils.java | 30 ++--
.../iotdb/db/engine/StorageEngineV2Test.java | 71 +++++++++
.../execution/operator/LinearFillOperatorTest.java | 113 ++++++++++++++-
...erTest.java => DefaultDriverSchedulerTest.java} | 2 +-
.../execution/schedule/DriverSchedulerTest.java | 2 +-
.../DriverTaskTimeoutSentinelThreadTest.java | 43 ++++++
.../iotdb/db/mpp/plan/plan/LogicalPlannerTest.java | 6 +-
.../plan/distribution/DeleteTimeseriesTest.java | 71 ---------
.../plan/node/write/DeleteDataNodeSerdeTest.java | 21 +--
.../db/sync/transport/TransportServiceTest.java | 8 +-
.../db/wal/allocation/FirstCreateStrategyTest.java | 2 +-
.../db/wal/checkpoint/CheckpointManagerTest.java | 6 +-
.../apache/iotdb/db/wal/io/CheckpointFileTest.java | 12 +-
.../iotdb/db/wal/node/ConsensusReqReaderTest.java | 14 +-
.../org/apache/iotdb/db/wal/node/WALNodeTest.java | 2 +-
.../src/main/thrift/confignode.thrift | 8 +-
thrift/src/main/thrift/mpp.thrift | 11 ++
.../iotdb/tsfile/read/common/block/TsBlock.java | 3 +-
104 files changed, 1884 insertions(+), 678 deletions(-)
diff --cc node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
index ee7584b435,d7e08bcadb..b311245411
--- a/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
+++ b/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
@@@ -93,10 -93,8 +93,9 @@@ public class IoTDBConstant
public static final String COLUMN_TAGS = "tags";
public static final String COLUMN_ATTRIBUTES = "attributes";
public static final String COLUMN_IS_ALIGNED = "isAligned";
+ public static final String COLUMN_DISTRIBUTION_PLAN = "distribution plan";
public static final String QUERY_ID = "queryId";
public static final String STATEMENT = "statement";
- public static final String TLL_NOT_SET = "not set";
public static final String COLUMN_ROLE = "role";
public static final String COLUMN_USER = "user";
diff --cc server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java
index 4b0c9a0b42,890dac8d9c..f0de05ee72
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java
@@@ -1763,14 -1774,79 +1775,86 @@@ public class ASTVisitor extends IoTDBSq
return deleteStorageGroupStatement;
}
+ // Explain ========================================================================
-
+ @Override
+ public Statement visitExplain(IoTDBSqlParser.ExplainContext ctx) {
+ QueryStatement queryStatement = (QueryStatement) visitSelectStatement(ctx.selectStatement());
+ return new ExplainStatement(queryStatement);
+ }
+
+ @Override
+ public Statement visitDeleteStatement(IoTDBSqlParser.DeleteStatementContext ctx) {
+ DeleteDataStatement statement = new DeleteDataStatement();
+ List<IoTDBSqlParser.PrefixPathContext> prefixPaths = ctx.prefixPath();
+ List<PartialPath> pathList = new ArrayList<>();
+ for (IoTDBSqlParser.PrefixPathContext prefixPath : prefixPaths) {
+ pathList.add(parsePrefixPath(prefixPath));
+ }
+ statement.setPathList(pathList);
+ if (ctx.whereClause() != null) {
+ WhereCondition whereCondition = parseWhereClause(ctx.whereClause());
+ TimeRange timeRange = parseDeleteTimeRange(whereCondition.getPredicate());
+ statement.setTimeRange(timeRange);
+ } else {
+ statement.setTimeRange(new TimeRange(Long.MIN_VALUE, Long.MAX_VALUE));
+ }
+ return statement;
+ }
+
+ private TimeRange parseDeleteTimeRange(Expression predicate) {
+ if (predicate instanceof LogicAndExpression) {
+ TimeRange leftTimeRange =
+ parseDeleteTimeRange(((LogicAndExpression) predicate).getLeftExpression());
+ TimeRange rightTimeRange =
+ parseDeleteTimeRange(((LogicAndExpression) predicate).getRightExpression());
+ return new TimeRange(
+ Math.max(leftTimeRange.getMin(), rightTimeRange.getMin()),
+ Math.min(leftTimeRange.getMax(), rightTimeRange.getMax()));
+ } else if (predicate instanceof CompareBinaryExpression) {
+ if (((CompareBinaryExpression) predicate).getLeftExpression() instanceof TimestampOperand) {
+ return parseTimeRange(
+ predicate.getExpressionType(),
+ ((CompareBinaryExpression) predicate).getLeftExpression(),
+ ((CompareBinaryExpression) predicate).getRightExpression());
+ } else {
+ return parseTimeRange(
+ predicate.getExpressionType(),
+ ((CompareBinaryExpression) predicate).getRightExpression(),
+ ((CompareBinaryExpression) predicate).getLeftExpression());
+ }
+ } else {
+ throw new SemanticException(DELETE_RANGE_ERROR_MSG);
+ }
+ }
+
+ private TimeRange parseTimeRange(
+ ExpressionType expressionType, Expression timeExpression, Expression valueExpression) {
+ if (!(timeExpression instanceof TimestampOperand)
+ || !(valueExpression instanceof ConstantOperand)) {
+ throw new SemanticException(DELETE_ONLY_SUPPORT_TIME_EXP_ERROR_MSG);
+ }
+
+ if (((ConstantOperand) valueExpression).getDataType() != TSDataType.INT64) {
+ throw new SemanticException("The datatype of timestamp should be LONG.");
+ }
+
+ long time = Long.parseLong(((ConstantOperand) valueExpression).getValueString());
+ switch (expressionType) {
+ case LESS_THAN:
+ return new TimeRange(Long.MIN_VALUE, time - 1);
+ case LESS_EQUAL:
+ return new TimeRange(Long.MIN_VALUE, time);
+ case GREATER_THAN:
+ return new TimeRange(time + 1, Long.MAX_VALUE);
+ case GREATER_EQUAL:
+ return new TimeRange(time, Long.MAX_VALUE);
+ case EQUAL_TO:
+ return new TimeRange(time, time);
+ default:
+ throw new SemanticException(DELETE_RANGE_ERROR_MSG);
+ }
+ }
+
/** function for parsing file path used by LOAD statement. */
public String parseFilePath(String src) {
return src.substring(1, src.length() - 1);
diff --cc server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/StatementVisitor.java
index 724609ef7b,44f7b0919c..a697ad882a
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/StatementVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/StatementVisitor.java
@@@ -223,7 -228,7 +229,11 @@@ public abstract class StatementVisitor<
return visitStatement(showChildNodesStatement, context);
}
+ public R visitExplain(ExplainStatement explainStatement, C context) {
+ return visitStatement(explainStatement, context);
+ }
++
+ public R visitDeleteData(DeleteDataStatement deleteDataStatement, C context) {
+ return visitStatement(deleteDataStatement, context);
+ }
}