You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2024/04/26 10:15:57 UTC

(iotdb) branch ty/TableModelGrammar updated (4059f88df97 -> 9e3e9d6cda8)

This is an automated email from the ASF dual-hosted git repository.

jackietien pushed a change to branch ty/TableModelGrammar
in repository https://gitbox.apache.org/repos/asf/iotdb.git


    from 4059f88df97 merge
     add ba853b3147b Pipe: Fixed the bug that aggregate processor cannot clear the last received time after drop (#12383)
     add 9051aa57908 Update iotdb-doap.rdf for v1.3.1 release (#12390)
     add b947dd83b6c Remove useless try-catch for RegionPriorityMap (#12389)
     add 7f4724dd3d7 Enhance concurrent control of data partition policy table (#12388)
     add ac648a3a2a0 Deduplication IoTConsensus Configuration And Remove Unuseless File Delete (#12385)
     add 3bfc0ea84a5 Provide more precise info for exception during conf loading (#12382)
     add e825af4aade feature add iotdb backup (#12365)
     add b7e9d6ec3d5 Some region migration related work (#12376)
     add 6b2e4b15925 Fix effective ratio calculation method  (#12394)
     add 1ce494b5f17 Fixed backup data backend running issues (#12396)
     add 74eb4e2e392 Pipe: Fixed the bug that pure data transfer is forbidden when SchemaRegion is in Simple consensus (#12397)
     add 638be8ae212 Pipe: Added batch to schema snapshot execution in template activation and timeseries creation & Fixed the bug that the "CreateMultiTimeSeries" group with alias is not idempotent (#12380)
     add bd22fe65461 Pipe: distinguish between old and new process subtasks before and after stuck restart to avoid unexpected hashmap override and removal (#12400)
     add c36f7144be7 Fixed concurrency issues with RegionMigration thread pool  (#12403)
     add 4d7362659d6 Fixed insertTablet http api lost data event return SUCCESS_STATUS (#12387)
     add ed6f56b25ab Fix partition cache after region migration (#12404)
     add e1e0055c940 Fix IoTDBTimezoneIT
     add c911c406bd1 Pipe: Fixed FileNotFoundException caused by processing files deleted by compaction & Changed the "isEmpty()" judgement to avoid "getDevice()" attempts to read from a deleted file when using fileTimeIndex (#12405)
     add 719834aea89 Pipe: refine procedure timeout message (#12402)
     add f5cf782df5b Pipe: Two stage aggregate framework & count-point-processor plugin (#12328)
     add 03064328b43 Change log level and opt error msg for source path of view is deleted (#12409)
     new c9e7b534ebc Merge remote-tracking branch 'origin/master' into ty/TableModelGrammar
     new edd9f243828 partial
     add e715db52084 Pipe: remove unnecessary fsync operations for file pieces' transferring (#12410)
     add c232ee02239 fix pipe snapshot parser
     add 8064ac58aa3 Pipe: Properly handle OUT_OF_TTL status code on receiver side when data syncing (#12414)
     add a8411f66837 Fix Concurrency Instability When IoTConsensus LogDispatcher Exits (#12411)
     add 544a3bccbb9 update Export/ImportDataTestIT (#12399)
     add 1d30805f266 Pipe: Added check for wal disable and ratis consensus when enabling data region processing (#12407)
     add 447bf54a356 revert -XX:GuaranteedSafepointInterval=0 option (#12417)
     add 95384d4a9eb Randomly choosing coordinator of add region (#12416)
     add d23729f876a [RatisConsensus] update ratis snapshot version (#12419)
     add 98b2272dbc1 Pipe: Reduce exception messages to avoid excess rpc payload and show pipe response (#12415)
     add 52bc1889d53 Copy tsfile, its resources, and mods to the same directory when using multiple data directories  (#12420)
     add d866bc5dbaf fix snapshot parser null point.
     add 1b981972bd7 Pipe: Support InsertRows in PipeInsertNodeTabletInsertionEvent (#12286)
     add 3d87ecc1346 Refactor MultiTsFileDeviceIterator.MeasurementIterator (#12221)
     new 37f5135e55b Merge remote-tracking branch 'origin/master' into ty/TableModelGrammar
     new 9e3e9d6cda8 support SortNode, MergeSortNode and TopKNode

The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../org/apache/iotdb/db/it/IoTDBTimeZoneIT.java    |   2 +-
 .../apache/iotdb/tools/it/ExportDataTestIT.java    |   6 +-
 .../apache/iotdb/tools/it/ImportDataTestIT.java    |   4 +-
 .../cli/src/assembly/resources/tools/backup.bat    | 125 +++
 .../cli/src/assembly/resources/tools/backup.sh     | 134 +++
 .../org/apache/iotdb/tool/AbstractDataTool.java    |   1 +
 .../java/org/apache/iotdb/tool/ExportData.java     |  21 +-
 .../java/org/apache/iotdb/tool/ImportData.java     |  73 +-
 .../org/apache/iotdb/tool/IoTDBDataBackTool.java   | 985 +++++++++++++++++++++
 .../confignode/conf/ConfigNodeDescriptor.java      |   1 +
 .../confignode/conf/ConfigNodeRemoveCheck.java     |   5 +-
 .../confignode/conf/SystemPropertiesUtils.java     |   3 +-
 .../consensus/request/ConfigPhysicalPlan.java      |   1 +
 .../consensus/request/ConfigPhysicalPlanType.java  |   1 +
 .../request/ConfigPhysicalPlanVisitor.java         |   6 +
 .../consensus/request/auth/AuthorPlan.java         |   3 +
 .../response/pipe/task/PipeTableResp.java          |   8 +-
 .../confignode/manager/PermissionManager.java      |   3 +-
 .../iotdb/confignode/manager/ProcedureManager.java |  93 +-
 .../iotdb/confignode/manager/load/LoadManager.java |   8 +-
 .../manager/load/balancer/PartitionBalancer.java   |  10 +-
 .../manager/load/balancer/RouteBalancer.java       |  32 +-
 .../manager/partition/PartitionManager.java        |  14 +-
 .../receiver/IoTDBConfigNodeReceiverAgent.java     |   6 +-
 .../protocol/IoTDBConfigRegionAirGapConnector.java |   5 +-
 .../pipe/event/PipeConfigRegionSnapshotEvent.java  |  53 +-
 .../pipe/event/PipeConfigRegionWritePlanEvent.java |  46 +-
 .../extractor/ConfigRegionListeningFilter.java     |   5 +-
 .../pipe/extractor/IoTDBConfigRegionExtractor.java |  13 +-
 .../receiver/protocol/IoTDBConfigNodeReceiver.java |   1 +
 .../PipeConfigPhysicalPlanTSStatusVisitor.java     |   9 +
 .../iotdb/confignode/persistence/AuthorInfo.java   |   3 +
 .../persistence/executor/ConfigPlanExecutor.java   |   1 +
 .../partition/DatabasePartitionTable.java          |  11 +-
 .../persistence/partition/PartitionInfo.java       |  11 +-
 .../schema/CNPhysicalPlanGenerator.java            |   9 +-
 .../procedure/env/RegionMaintainHandler.java       | 122 ++-
 .../impl/pipe/task/AlterPipeProcedureV2.java       |   2 +-
 .../impl/region/AddRegionPeerProcedure.java        |  28 +-
 .../impl/region/RemoveRegionPeerProcedure.java     |   2 +-
 .../confignode/persistence/AuthorInfoTest.java     |  21 +
 .../persistence/CNPhysicalPlanGeneratorTest.java   |   3 +
 .../consensus/iot/IoTConsensusServerImpl.java      |  47 +-
 .../consensus/iot/logdispatcher/LogDispatcher.java |   8 +
 .../apache/iotdb/consensus/iot/ReplicateTest.java  |  36 +-
 .../src/assembly/resources/conf/datanode-env.bat   |   3 -
 .../src/assembly/resources/conf/datanode-env.sh    |   3 -
 .../assembly/resources/conf/logback-datanode.xml   |  15 -
 .../java/org/apache/iotdb/db/conf/IoTDBConfig.java |   2 +-
 .../org/apache/iotdb/db/conf/IoTDBDescriptor.java  |  12 +-
 .../db/conf/rest/IoTDBRestServiceDescriptor.java   |   1 +
 .../metadata/view/BrokenViewException.java         |  13 +-
 .../PipeDataRegionProcessorConstructor.java        |   3 +
 .../agent/receiver/PipeDataNodeReceiverAgent.java  |   5 -
 .../db/pipe/agent/task/PipeDataNodeTaskAgent.java  |   9 +
 .../request/PipeTransferTabletInsertNodeReq.java   |  46 +-
 .../airgap/IoTDBDataRegionAirGapConnector.java     |   9 +-
 .../airgap/IoTDBSchemaRegionAirGapConnector.java   |   5 +-
 .../protocol/legacy/IoTDBLegacyPipeConnector.java  |  14 +-
 .../connector/protocol/opcua/OpcUaConnector.java   |   4 +-
 .../thrift/sync/IoTDBDataRegionSyncConnector.java  |  10 +-
 .../protocol/websocket/WebSocketConnector.java     |  17 +-
 .../websocket/WebSocketConnectorServer.java        |   5 +-
 .../db/pipe/event/common/row/PipeRowCollector.java |   4 +-
 .../schema/PipeSchemaRegionSnapshotEvent.java      |  64 +-
 .../schema/PipeSchemaRegionWritePlanEvent.java     |  44 +-
 .../tablet/PipeInsertNodeTabletInsertionEvent.java | 122 ++-
 .../common/tablet/PipeRawTabletInsertionEvent.java |   5 +
 .../tablet/TabletInsertionDataContainer.java       |  15 +-
 .../common/tsfile/PipeTsFileInsertionEvent.java    |  34 +
 .../common/tsfile/TsFileInsertionPointCounter.java | 169 ++++
 .../common/watermark/PipeWatermarkEvent.java}      |  21 +-
 .../dataregion/DataRegionWatermarkInjector.java    |  76 ++
 .../dataregion/IoTDBDataRegionExtractor.java       |  82 +-
 .../PipeHistoricalDataRegionTsFileExtractor.java   |  24 +-
 .../realtime/PipeRealtimeDataRegionExtractor.java  |   6 +-
 .../schemaregion/IoTDBSchemaRegionExtractor.java   |  13 +-
 .../processor/aggregate/AggregateProcessor.java    |   3 +-
 .../pipe/processor/twostage/combiner/Combiner.java | 107 +++
 .../twostage/combiner/PipeCombineHandler.java      | 198 +++++
 .../combiner/PipeCombineHandlerManager.java        | 163 ++++
 .../twostage/exchange/payload/CombineRequest.java  | 139 +++
 .../payload/FetchCombineResultRequest.java         | 119 +++
 .../payload/FetchCombineResultResponse.java        |  99 +++
 .../twostage/exchange/payload/RequestType.java     |  16 +-
 .../receiver/TwoStageAggregateReceiver.java        |  82 ++
 .../exchange/sender/TwoStageAggregateSender.java   | 235 +++++
 .../processor/twostage/operator/CountOperator.java |  32 +-
 .../processor/twostage/operator/Operator.java}     |  12 +-
 .../twostage/plugin/TwoStageCountProcessor.java    | 391 ++++++++
 .../processor/twostage/state/CountState.java}      |  26 +-
 .../db/pipe/processor/twostage/state/State.java}   |  11 +-
 .../legacy/IoTDBLegacyPipeReceiverAgent.java       |   1 -
 .../protocol/thrift/IoTDBDataNodeReceiver.java     |  72 +-
 .../thrift/IoTDBDataNodeReceiverAgent.java         |   3 +
 .../visitor/PipePlanToStatementVisitor.java        | 105 ++-
 .../visitor/PipeStatementTSStatusVisitor.java      |  57 +-
 .../visitor/PipeStatementToBatchVisitor.java       | 185 ++++
 .../pipe/task/connection/PipeEventCollector.java   |   3 +-
 .../subtask/processor/PipeProcessorSubtask.java    |  57 +-
 .../rest/utils/InsertTabletSortDataUtils.java      |  16 +-
 .../execution/executor/RegionWriteExecutor.java    |  12 +-
 .../process/join/merge/MergeSortComparator.java    |  20 +
 .../analyze/cache/partition/PartitionCache.java    |   1 +
 .../visitor/CompleteMeasurementSchemaVisitor.java  |   6 +-
 .../plan/planner/TableOperatorGenerator.java       | 156 +++-
 .../planner/plan/node/write/InsertRowsNode.java    |   9 +-
 .../plan/relational/planner/node/LimitNode.java    |   8 +-
 .../relational/planner/node/MergeSortNode.java     |  33 +-
 .../plan/relational/planner/node/OffsetNode.java   |   8 +-
 .../plan/relational/planner/node/SortNode.java     |  12 +-
 .../plan/relational/planner/node/TopKNode.java     |  43 +-
 .../iotdb/db/service/RegionMigrateService.java     |  10 +-
 .../impl/ReadChunkCompactionPerformer.java         |  12 +-
 .../execute/utils/MultiTsFileDeviceIterator.java   | 146 +--
 .../dataregion/snapshot/SnapshotLoader.java        |  12 +-
 .../dataregion/tsfile/TsFileResource.java          |   2 +-
 .../storageengine/dataregion/wal/node/WALNode.java |   7 +-
 .../broker/SubscriptionPrefetchingQueue.java       |  28 +-
 .../db/tools/schema/SRStatementGenerator.java      | 104 ++-
 .../db/auth/user/LocalFileUserManagerTest.java     |   8 +
 .../extractor/IoTDBDataRegionExtractorTest.java    |  32 +-
 .../utils/MultiTsFileDeviceIteratorTest.java       |  70 ++
 .../dataregion/wal/WALManagerTest.java             |   5 +
 .../dataregion/wal/node/WALNodeTest.java           |   5 +
 .../wal/node/WalDeleteOutdatedNewTest.java         |   5 +
 .../db/utils/SchemaRegionSnapshotParserTest.java   |  86 +-
 .../src/assembly/resources/sbin/start-all.sh       |   2 +-
 .../src/assembly/resources/sbin/stop-all.sh        |   2 +-
 .../commons/auth/authorizer/BasicAuthorizer.java   |  12 +-
 .../iotdb/commons/auth/authorizer/IAuthorizer.java |   2 +
 .../iotdb/commons/auth/user/BasicUserManager.java  |  16 +-
 .../iotdb/commons/auth/user/IUserManager.java      |   3 +-
 .../commons/concurrent/IoTDBThreadPoolFactory.java |  12 -
 .../apache/iotdb/commons/conf/CommonConfig.java    |  41 +
 .../iotdb/commons/conf/CommonDescriptor.java       |  22 +
 .../commons/consensus/index/ProgressIndex.java     |   7 +-
 .../commons/consensus/index/ProgressIndexType.java |   7 +-
 .../consensus/index/impl/HybridProgressIndex.java  |   4 +
 .../consensus/index/impl/MetaProgressIndex.java    |   2 +-
 .../consensus/index/impl/StateProgressIndex.java   | 234 +++++
 .../iotdb/commons/pipe/config/PipeConfig.java      |  30 +
 .../config/constant/PipeExtractorConstant.java     |   4 +
 .../config/constant/PipeProcessorConstant.java     |   2 +
 .../request/IoTDBConnectorRequestVersion.java      |   1 +
 .../commons/pipe/event/PipeSnapshotEvent.java      |  19 +-
 .../commons/pipe/event/PipeWritePlanEvent.java     |  35 +-
 .../pipe/plugin/builtin/BuiltinPipePlugin.java     |   2 +
 .../TwoStageCountProcessor.java}                   |   8 +-
 .../commons/pipe/receiver/IoTDBFileReceiver.java   |  20 +-
 .../commons/pipe/task/meta/PipeStaticMeta.java     |  29 +-
 .../iotdb/commons/pipe/task/meta/PipeTaskMeta.java |  28 +-
 .../org/apache/iotdb/commons/utils/FileUtils.java  |  18 +
 iotdb-doap.rdf                                     |  26 +-
 pom.xml                                            |   2 +-
 155 files changed, 5488 insertions(+), 910 deletions(-)
 create mode 100644 iotdb-client/cli/src/assembly/resources/tools/backup.bat
 create mode 100755 iotdb-client/cli/src/assembly/resources/tools/backup.sh
 create mode 100644 iotdb-client/cli/src/main/java/org/apache/iotdb/tool/IoTDBDataBackTool.java
 create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/TsFileInsertionPointCounter.java
 copy iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{storageengine/dataregion/compaction/schedule/constant/CompactionType.java => pipe/event/common/watermark/PipeWatermarkEvent.java} (68%)
 create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/DataRegionWatermarkInjector.java
 create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/combiner/Combiner.java
 create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/combiner/PipeCombineHandler.java
 create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/combiner/PipeCombineHandlerManager.java
 create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/CombineRequest.java
 create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/FetchCombineResultRequest.java
 create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/FetchCombineResultResponse.java
 copy iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribeResponseType.java => iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/RequestType.java (77%)
 create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/receiver/TwoStageAggregateReceiver.java
 create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/sender/TwoStageAggregateSender.java
 copy integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/Cluster1Env.java => iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/operator/CountOperator.java (52%)
 copy iotdb-core/{node-commons/src/main/java/org/apache/iotdb/commons/pipe/task/PipeTask.java => datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/operator/Operator.java} (80%)
 create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/plugin/TwoStageCountProcessor.java
 copy iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{schemaengine/template/alter/TemplateAlterInfo.java => pipe/processor/twostage/state/CountState.java} (69%)
 copy iotdb-core/{confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/IProcedureFactory.java => datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/state/State.java} (79%)
 create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeStatementToBatchVisitor.java
 create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/consensus/index/impl/StateProgressIndex.java
 copy iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/plugin/builtin/processor/{downsampling/SwingingDoorTrendingSamplingProcessor.java => twostage/TwoStageCountProcessor.java} (79%)


(iotdb) 03/04: Merge remote-tracking branch 'origin/master' into ty/TableModelGrammar

Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jackietien pushed a commit to branch ty/TableModelGrammar
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit 37f5135e55bc1e1ce648783f92250226568e0821
Merge: edd9f243828 3d87ecc1346
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Fri Apr 26 15:53:57 2024 +0800

    Merge remote-tracking branch 'origin/master' into ty/TableModelGrammar

 .../apache/iotdb/tools/it/ExportDataTestIT.java    |   6 +-
 .../apache/iotdb/tools/it/ImportDataTestIT.java    |   4 +-
 .../org/apache/iotdb/tool/AbstractDataTool.java    |   1 +
 .../java/org/apache/iotdb/tool/ExportData.java     |  21 +--
 .../java/org/apache/iotdb/tool/ImportData.java     |  73 +++++------
 .../consensus/request/ConfigPhysicalPlan.java      |   1 +
 .../consensus/request/ConfigPhysicalPlanType.java  |   1 +
 .../request/ConfigPhysicalPlanVisitor.java         |   6 +
 .../consensus/request/auth/AuthorPlan.java         |   3 +
 .../response/pipe/task/PipeTableResp.java          |   8 +-
 .../confignode/manager/PermissionManager.java      |   3 +-
 .../iotdb/confignode/manager/ProcedureManager.java |   3 +-
 .../receiver/IoTDBConfigNodeReceiverAgent.java     |   6 +-
 .../protocol/IoTDBConfigRegionAirGapConnector.java |   5 +-
 .../pipe/event/PipeConfigRegionSnapshotEvent.java  |  53 ++++----
 .../pipe/event/PipeConfigRegionWritePlanEvent.java |  46 +++++--
 .../extractor/ConfigRegionListeningFilter.java     |   5 +-
 .../receiver/protocol/IoTDBConfigNodeReceiver.java |   1 +
 .../PipeConfigPhysicalPlanTSStatusVisitor.java     |   9 ++
 .../iotdb/confignode/persistence/AuthorInfo.java   |   3 +
 .../persistence/executor/ConfigPlanExecutor.java   |   1 +
 .../schema/CNPhysicalPlanGenerator.java            |   9 +-
 .../procedure/env/RegionMaintainHandler.java       |   3 +-
 .../impl/pipe/task/AlterPipeProcedureV2.java       |   2 +-
 .../confignode/persistence/AuthorInfoTest.java     |  21 +++
 .../persistence/CNPhysicalPlanGeneratorTest.java   |   3 +
 .../consensus/iot/logdispatcher/LogDispatcher.java |   8 ++
 .../apache/iotdb/consensus/iot/ReplicateTest.java  |  36 ++++-
 .../src/assembly/resources/conf/datanode-env.bat   |   3 -
 .../src/assembly/resources/conf/datanode-env.sh    |   3 -
 .../java/org/apache/iotdb/db/conf/IoTDBConfig.java |   2 +-
 .../request/PipeTransferTabletInsertNodeReq.java   |  46 ++-----
 .../airgap/IoTDBDataRegionAirGapConnector.java     |   9 +-
 .../airgap/IoTDBSchemaRegionAirGapConnector.java   |   5 +-
 .../protocol/legacy/IoTDBLegacyPipeConnector.java  |  14 +-
 .../connector/protocol/opcua/OpcUaConnector.java   |   4 +-
 .../thrift/sync/IoTDBDataRegionSyncConnector.java  |  10 +-
 .../protocol/websocket/WebSocketConnector.java     |  17 ++-
 .../websocket/WebSocketConnectorServer.java        |   5 +-
 .../db/pipe/event/common/row/PipeRowCollector.java |   4 +-
 .../schema/PipeSchemaRegionSnapshotEvent.java      |   9 ++
 .../schema/PipeSchemaRegionWritePlanEvent.java     |  44 +++++--
 .../tablet/PipeInsertNodeTabletInsertionEvent.java | 121 ++++++++++++-----
 .../tablet/TabletInsertionDataContainer.java       |  15 ++-
 .../common/tsfile/PipeTsFileInsertionEvent.java    |   9 ++
 .../dataregion/IoTDBDataRegionExtractor.java       |  43 +++++-
 .../realtime/PipeRealtimeDataRegionExtractor.java  |   6 +-
 .../processor/aggregate/AggregateProcessor.java    |   2 +-
 .../twostage/plugin/TwoStageCountProcessor.java    |   3 +
 .../legacy/IoTDBLegacyPipeReceiverAgent.java       |   1 -
 .../protocol/thrift/IoTDBDataNodeReceiver.java     |  13 +-
 .../visitor/PipePlanToStatementVisitor.java        | 105 +++++++++++----
 .../visitor/PipeStatementTSStatusVisitor.java      |   5 +-
 .../pipe/task/connection/PipeEventCollector.java   |   3 +-
 .../subtask/processor/PipeProcessorSubtask.java    |  38 +++---
 .../planner/plan/node/write/InsertRowsNode.java    |   9 +-
 .../impl/ReadChunkCompactionPerformer.java         |  12 +-
 .../execute/utils/MultiTsFileDeviceIterator.java   | 146 ++++++++++++---------
 .../dataregion/snapshot/SnapshotLoader.java        |  12 +-
 .../broker/SubscriptionPrefetchingQueue.java       |  28 ++--
 .../db/tools/schema/SRStatementGenerator.java      | 104 ++++++++++-----
 .../db/auth/user/LocalFileUserManagerTest.java     |   8 ++
 .../extractor/IoTDBDataRegionExtractorTest.java    |  32 ++---
 .../utils/MultiTsFileDeviceIteratorTest.java       |  70 ++++++++++
 .../dataregion/wal/WALManagerTest.java             |   5 +
 .../dataregion/wal/node/WALNodeTest.java           |   5 +
 .../wal/node/WalDeleteOutdatedNewTest.java         |   5 +
 .../db/utils/SchemaRegionSnapshotParserTest.java   |  86 +++++++++++-
 .../src/assembly/resources/sbin/start-all.sh       |   2 +-
 .../src/assembly/resources/sbin/stop-all.sh        |   2 +-
 .../commons/auth/authorizer/BasicAuthorizer.java   |  12 +-
 .../iotdb/commons/auth/authorizer/IAuthorizer.java |   2 +
 .../iotdb/commons/auth/user/BasicUserManager.java  |  16 ++-
 .../iotdb/commons/auth/user/IUserManager.java      |   3 +-
 .../commons/pipe/event/PipeSnapshotEvent.java      |  19 ++-
 .../commons/pipe/event/PipeWritePlanEvent.java     |  35 +++--
 .../commons/pipe/receiver/IoTDBFileReceiver.java   |   6 +-
 .../commons/pipe/task/meta/PipeStaticMeta.java     |  29 ++--
 .../iotdb/commons/pipe/task/meta/PipeTaskMeta.java |  28 ++--
 pom.xml                                            |   2 +-
 80 files changed, 1069 insertions(+), 489 deletions(-)



(iotdb) 01/04: Merge remote-tracking branch 'origin/master' into ty/TableModelGrammar

Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jackietien pushed a commit to branch ty/TableModelGrammar
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit c9e7b534ebc0245886b93d78b2650af2fade9daa
Merge: 4059f88df97 03064328b43
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Thu Apr 25 11:37:38 2024 +0800

    Merge remote-tracking branch 'origin/master' into ty/TableModelGrammar

 .../org/apache/iotdb/db/it/IoTDBTimeZoneIT.java    |   2 +-
 .../cli/src/assembly/resources/tools/backup.bat    | 125 +++
 .../cli/src/assembly/resources/tools/backup.sh     | 134 +++
 .../org/apache/iotdb/tool/IoTDBDataBackTool.java   | 985 +++++++++++++++++++++
 .../confignode/conf/ConfigNodeDescriptor.java      |   1 +
 .../confignode/conf/ConfigNodeRemoveCheck.java     |   5 +-
 .../confignode/conf/SystemPropertiesUtils.java     |   3 +-
 .../iotdb/confignode/manager/ProcedureManager.java |  90 +-
 .../iotdb/confignode/manager/load/LoadManager.java |   8 +-
 .../manager/load/balancer/PartitionBalancer.java   |  10 +-
 .../manager/load/balancer/RouteBalancer.java       |  32 +-
 .../manager/partition/PartitionManager.java        |  14 +-
 .../pipe/extractor/IoTDBConfigRegionExtractor.java |  13 +-
 .../partition/DatabasePartitionTable.java          |  11 +-
 .../persistence/partition/PartitionInfo.java       |  11 +-
 .../procedure/env/RegionMaintainHandler.java       | 119 ++-
 .../impl/region/AddRegionPeerProcedure.java        |  28 +-
 .../impl/region/RemoveRegionPeerProcedure.java     |   2 +-
 .../consensus/iot/IoTConsensusServerImpl.java      |  47 +-
 .../assembly/resources/conf/logback-datanode.xml   |  15 -
 .../org/apache/iotdb/db/conf/IoTDBDescriptor.java  |  12 +-
 .../db/conf/rest/IoTDBRestServiceDescriptor.java   |   1 +
 .../metadata/view/BrokenViewException.java         |  13 +-
 .../PipeDataRegionProcessorConstructor.java        |   3 +
 .../agent/receiver/PipeDataNodeReceiverAgent.java  |   5 -
 .../db/pipe/agent/task/PipeDataNodeTaskAgent.java  |   9 +
 .../schema/PipeSchemaRegionSnapshotEvent.java      |  55 +-
 .../tablet/PipeInsertNodeTabletInsertionEvent.java |   5 +
 .../common/tablet/PipeRawTabletInsertionEvent.java |   5 +
 .../common/tsfile/PipeTsFileInsertionEvent.java    |  25 +
 .../common/tsfile/TsFileInsertionPointCounter.java | 169 ++++
 .../common/watermark/PipeWatermarkEvent.java}      |  23 +-
 .../dataregion/DataRegionWatermarkInjector.java    |  76 ++
 .../dataregion/IoTDBDataRegionExtractor.java       |  39 +-
 .../PipeHistoricalDataRegionTsFileExtractor.java   |  24 +-
 .../schemaregion/IoTDBSchemaRegionExtractor.java   |  13 +-
 .../processor/aggregate/AggregateProcessor.java    |   1 +
 .../pipe/processor/twostage/combiner/Combiner.java | 107 +++
 .../twostage/combiner/PipeCombineHandler.java      | 198 +++++
 .../combiner/PipeCombineHandlerManager.java        | 163 ++++
 .../twostage/exchange/payload/CombineRequest.java  | 139 +++
 .../payload/FetchCombineResultRequest.java         | 119 +++
 .../payload/FetchCombineResultResponse.java        |  99 +++
 .../twostage/exchange/payload/RequestType.java}    |  36 +-
 .../receiver/TwoStageAggregateReceiver.java        |  82 ++
 .../exchange/sender/TwoStageAggregateSender.java   | 235 +++++
 .../processor/twostage/operator/CountOperator.java |  52 ++
 .../processor/twostage/operator/Operator.java}     |  16 +-
 .../twostage/plugin/TwoStageCountProcessor.java    | 388 ++++++++
 .../pipe/processor/twostage/state/CountState.java} |  36 +-
 .../db/pipe/processor/twostage/state/State.java}   |  18 +-
 .../protocol/thrift/IoTDBDataNodeReceiver.java     |  59 +-
 .../thrift/IoTDBDataNodeReceiverAgent.java         |   3 +
 .../visitor/PipeStatementTSStatusVisitor.java      |  52 +-
 .../visitor/PipeStatementToBatchVisitor.java       | 185 ++++
 .../subtask/processor/PipeProcessorSubtask.java    |  21 +-
 .../rest/utils/InsertTabletSortDataUtils.java      |  16 +-
 .../execution/executor/RegionWriteExecutor.java    |  12 +-
 .../analyze/cache/partition/PartitionCache.java    |   1 +
 .../visitor/CompleteMeasurementSchemaVisitor.java  |   6 +-
 .../iotdb/db/service/RegionMigrateService.java     |  10 +-
 .../dataregion/tsfile/TsFileResource.java          |   2 +-
 .../storageengine/dataregion/wal/node/WALNode.java |   7 +-
 .../commons/concurrent/IoTDBThreadPoolFactory.java |  12 -
 .../apache/iotdb/commons/conf/CommonConfig.java    |  41 +
 .../iotdb/commons/conf/CommonDescriptor.java       |  22 +
 .../commons/consensus/index/ProgressIndex.java     |   7 +-
 .../commons/consensus/index/ProgressIndexType.java |   7 +-
 .../consensus/index/impl/HybridProgressIndex.java  |   4 +
 .../consensus/index/impl/MetaProgressIndex.java    |   2 +-
 .../consensus/index/impl/StateProgressIndex.java   | 234 +++++
 .../iotdb/commons/pipe/config/PipeConfig.java      |  30 +
 .../config/constant/PipeExtractorConstant.java     |   4 +
 .../config/constant/PipeProcessorConstant.java     |   2 +
 .../request/IoTDBConnectorRequestVersion.java      |   1 +
 .../pipe/plugin/builtin/BuiltinPipePlugin.java     |   2 +
 .../twostage/TwoStageCountProcessor.java}          |  23 +-
 .../commons/pipe/receiver/IoTDBFileReceiver.java   |  14 +-
 .../org/apache/iotdb/commons/utils/FileUtils.java  |  18 +
 iotdb-doap.rdf                                     |  26 +-
 80 files changed, 4234 insertions(+), 410 deletions(-)



(iotdb) 02/04: partial

Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jackietien pushed a commit to branch ty/TableModelGrammar
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit edd9f2438286619cdd29b2179e7e6fa56e7d52cf
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Fri Apr 26 01:14:29 2024 +0800

    partial
---
 .../iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java | 8 ++++++++
 .../queryengine/plan/relational/planner/node/MergeSortNode.java   | 8 +++++++-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java
index 41f5b7f41b4..767325bb302 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java
@@ -31,6 +31,7 @@ import org.apache.iotdb.db.queryengine.execution.operator.Operator;
 import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext;
 import org.apache.iotdb.db.queryengine.execution.operator.process.FilterAndProjectOperator;
 import org.apache.iotdb.db.queryengine.execution.operator.process.LimitOperator;
+import org.apache.iotdb.db.queryengine.execution.operator.process.MergeSortOperator;
 import org.apache.iotdb.db.queryengine.execution.operator.process.OffsetOperator;
 import org.apache.iotdb.db.queryengine.execution.operator.sink.IdentitySinkOperator;
 import org.apache.iotdb.db.queryengine.execution.operator.source.AlignedSeriesScanOperator;
@@ -478,6 +479,13 @@ public class TableOperatorGenerator extends PlanVisitor<Operator, LocalExecution
 
   @Override
   public Operator visitMergeSort(MergeSortNode node, LocalExecutionPlanContext context) {
+    OperatorContext operatorContext = context.getDriverContext().addOperatorContext(context.getNextOperatorId(), node.getPlanNodeId(),
+        MergeSortOperator.class.getSimpleName());
+    List<Operator> children = new ArrayList<>(node.getChildren().size());
+    for (PlanNode child : node.getChildren()) {
+      children.add(this.process(child, context));
+    }
+
     return super.visitMergeSort(node, context);
   }
 
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/MergeSortNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/MergeSortNode.java
index 58a94fc3b9c..680d3812bac 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/MergeSortNode.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/MergeSortNode.java
@@ -22,6 +22,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
 import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId;
 import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.MultiChildProcessNode;
 import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.OrderByParameter;
+import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol;
 
 import java.io.DataOutputStream;
 import java.io.IOException;
@@ -47,7 +48,7 @@ public class MergeSortNode extends MultiChildProcessNode {
 
   @Override
   public List<String> getOutputColumnNames() {
-    return null;
+    return children.get(0).getOutputColumnNames();
   }
 
   @Override
@@ -55,4 +56,9 @@ public class MergeSortNode extends MultiChildProcessNode {
 
   @Override
   protected void serializeAttributes(DataOutputStream stream) throws IOException {}
+
+  @Override
+  public List<Symbol> getOutputSymbols() {
+    return super.getOutputSymbols();
+  }
 }


(iotdb) 04/04: support SortNode, MergeSortNode and TopKNode

Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jackietien pushed a commit to branch ty/TableModelGrammar
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit 9e3e9d6cda88e2ac7eedcd071d4e9eeb0e502544
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Fri Apr 26 18:15:45 2024 +0800

    support SortNode, MergeSortNode and TopKNode
---
 .../process/join/merge/MergeSortComparator.java    |  20 +++
 .../plan/planner/TableOperatorGenerator.java       | 162 +++++++++++++++++++--
 .../plan/relational/planner/node/LimitNode.java    |   8 +-
 .../relational/planner/node/MergeSortNode.java     |  29 ++--
 .../plan/relational/planner/node/OffsetNode.java   |   8 +-
 .../plan/relational/planner/node/SortNode.java     |  12 +-
 .../plan/relational/planner/node/TopKNode.java     |  43 +++++-
 7 files changed, 251 insertions(+), 31 deletions(-)

diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/MergeSortComparator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/MergeSortComparator.java
index c299a368e73..db6f36206f9 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/MergeSortComparator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/MergeSortComparator.java
@@ -19,6 +19,7 @@
 
 package org.apache.iotdb.db.queryengine.execution.operator.process.join.merge;
 
+import org.apache.iotdb.db.queryengine.plan.relational.planner.SortOrder;
 import org.apache.iotdb.db.queryengine.plan.statement.component.NullOrdering;
 import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering;
 import org.apache.iotdb.db.queryengine.plan.statement.component.SortItem;
@@ -66,6 +67,25 @@ public class MergeSortComparator {
     return list.size() == 1 ? list.get(0) : new ComparatorChain<>(list);
   }
 
+  public static Comparator<SortKey> getComparatorForTable(
+      List<SortOrder> sortOrderList, List<Integer> indexList, List<TSDataType> dataTypeList) {
+
+    // use code-gen compile this comparator
+    List<Comparator<SortKey>> list = new ArrayList<>(indexList.size());
+    for (int i = 0; i < indexList.size(); i++) {
+      int index = indexList.get(i);
+      if (index == -2) {
+        continue;
+      }
+      TSDataType dataType = dataTypeList.get(i);
+      SortOrder sortOrder = sortOrderList.get(i);
+      list.add(
+          genSingleComparator(sortOrder.isAscending(), index, dataType, sortOrder.isNullsFirst()));
+    }
+
+    return list.size() == 1 ? list.get(0) : new ComparatorChain<>(list);
+  }
+
   public static Comparator<SortKey> getComparator(TSDataType dataType, int index, boolean asc) {
     Comparator<SortKey> comparator;
     switch (dataType) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java
index 767325bb302..09e7b9cae42 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java
@@ -20,6 +20,7 @@
 package org.apache.iotdb.db.queryengine.plan.planner;
 
 import org.apache.iotdb.commons.path.AlignedPath;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
 import org.apache.iotdb.db.queryengine.common.FragmentInstanceId;
 import org.apache.iotdb.db.queryengine.execution.driver.DataDriverContext;
 import org.apache.iotdb.db.queryengine.execution.exchange.MPPDataExchangeManager;
@@ -33,6 +34,8 @@ import org.apache.iotdb.db.queryengine.execution.operator.process.FilterAndProje
 import org.apache.iotdb.db.queryengine.execution.operator.process.LimitOperator;
 import org.apache.iotdb.db.queryengine.execution.operator.process.MergeSortOperator;
 import org.apache.iotdb.db.queryengine.execution.operator.process.OffsetOperator;
+import org.apache.iotdb.db.queryengine.execution.operator.process.SortOperator;
+import org.apache.iotdb.db.queryengine.execution.operator.process.TopKOperator;
 import org.apache.iotdb.db.queryengine.execution.operator.sink.IdentitySinkOperator;
 import org.apache.iotdb.db.queryengine.execution.operator.source.AlignedSeriesScanOperator;
 import org.apache.iotdb.db.queryengine.execution.operator.source.relational.TableScanOperator;
@@ -47,6 +50,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.InputLocation
 import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.SeriesScanOptions;
 import org.apache.iotdb.db.queryengine.plan.relational.metadata.ColumnSchema;
 import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata;
+import org.apache.iotdb.db.queryengine.plan.relational.planner.OrderingScheme;
 import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol;
 import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode;
 import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LimitNode;
@@ -69,6 +73,7 @@ import org.apache.tsfile.read.filter.basic.Filter;
 import org.apache.tsfile.write.schema.IMeasurementSchema;
 import org.apache.tsfile.write.schema.MeasurementSchema;
 
+import java.io.File;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -81,6 +86,7 @@ import java.util.stream.Collectors;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static java.util.Objects.requireNonNull;
+import static org.apache.iotdb.db.queryengine.execution.operator.process.join.merge.MergeSortComparator.getComparatorForTable;
 import static org.apache.iotdb.db.queryengine.execution.operator.source.relational.TableScanOperator.constructAlignedPath;
 import static org.apache.iotdb.db.queryengine.plan.analyze.PredicateUtils.convertPredicateToFilter;
 import static org.apache.iotdb.db.queryengine.plan.expression.leaf.TimestampOperand.TIMESTAMP_EXPRESSION_STRING;
@@ -477,18 +483,6 @@ public class TableOperatorGenerator extends PlanVisitor<Operator, LocalExecution
     return new OffsetOperator(operatorContext, node.getCount(), child);
   }
 
-  @Override
-  public Operator visitMergeSort(MergeSortNode node, LocalExecutionPlanContext context) {
-    OperatorContext operatorContext = context.getDriverContext().addOperatorContext(context.getNextOperatorId(), node.getPlanNodeId(),
-        MergeSortOperator.class.getSimpleName());
-    List<Operator> children = new ArrayList<>(node.getChildren().size());
-    for (PlanNode child : node.getChildren()) {
-      children.add(this.process(child, context));
-    }
-
-    return super.visitMergeSort(node, context);
-  }
-
   @Override
   public Operator visitOutput(OutputNode node, LocalExecutionPlanContext context) {
     TypeProvider typeProvider = context.getTypeProvider();
@@ -522,13 +516,153 @@ public class TableOperatorGenerator extends PlanVisitor<Operator, LocalExecution
         context);
   }
 
+  @Override
+  public Operator visitMergeSort(MergeSortNode node, LocalExecutionPlanContext context) {
+    OperatorContext operatorContext =
+        context
+            .getDriverContext()
+            .addOperatorContext(
+                context.getNextOperatorId(),
+                node.getPlanNodeId(),
+                MergeSortOperator.class.getSimpleName());
+    List<Operator> children = new ArrayList<>(node.getChildren().size());
+    for (PlanNode child : node.getChildren()) {
+      children.add(this.process(child, context));
+    }
+    List<TSDataType> dataTypes = getOutputColumnTypes(node, context.getTypeProvider());
+    int sortItemsCount = node.getOrderingScheme().getOrderBy().size();
+
+    List<Integer> sortItemIndexList = new ArrayList<>(sortItemsCount);
+    List<TSDataType> sortItemDataTypeList = new ArrayList<>(sortItemsCount);
+    genSortInformation(
+        node.getOutputSymbols(),
+        node.getOrderingScheme(),
+        sortItemIndexList,
+        sortItemDataTypeList,
+        context.getTypeProvider());
+
+    return new MergeSortOperator(
+        operatorContext,
+        children,
+        dataTypes,
+        getComparatorForTable(
+            node.getOrderingScheme().getOrderingList(), sortItemIndexList, sortItemDataTypeList));
+  }
+
   @Override
   public Operator visitSort(SortNode node, LocalExecutionPlanContext context) {
-    return super.visitSort(node, context);
+    OperatorContext operatorContext =
+        context
+            .getDriverContext()
+            .addOperatorContext(
+                context.getNextOperatorId(),
+                node.getPlanNodeId(),
+                SortOperator.class.getSimpleName());
+    List<TSDataType> dataTypes = getOutputColumnTypes(node, context.getTypeProvider());
+    int sortItemsCount = node.getOrderingScheme().getOrderBy().size();
+
+    List<Integer> sortItemIndexList = new ArrayList<>(sortItemsCount);
+    List<TSDataType> sortItemDataTypeList = new ArrayList<>(sortItemsCount);
+    genSortInformation(
+        node.getOutputSymbols(),
+        node.getOrderingScheme(),
+        sortItemIndexList,
+        sortItemDataTypeList,
+        context.getTypeProvider());
+
+    String filePrefix =
+        IoTDBDescriptor.getInstance().getConfig().getSortTmpDir()
+            + File.separator
+            + operatorContext.getDriverContext().getFragmentInstanceContext().getId().getFullId()
+            + File.separator
+            + operatorContext.getDriverContext().getPipelineId()
+            + File.separator;
+
+    context.getDriverContext().setHaveTmpFile(true);
+    context.getDriverContext().getFragmentInstanceContext().setMayHaveTmpFile(true);
+
+    Operator child = node.getChild().accept(this, context);
+
+    return new SortOperator(
+        operatorContext,
+        child,
+        dataTypes,
+        filePrefix,
+        getComparatorForTable(
+            node.getOrderingScheme().getOrderingList(), sortItemIndexList, sortItemDataTypeList));
   }
 
   @Override
   public Operator visitTopK(TopKNode node, LocalExecutionPlanContext context) {
-    return super.visitTopK(node, context);
+    OperatorContext operatorContext =
+        context
+            .getDriverContext()
+            .addOperatorContext(
+                context.getNextOperatorId(),
+                node.getPlanNodeId(),
+                TopKOperator.class.getSimpleName());
+    List<Operator> children = new ArrayList<>(node.getChildren().size());
+    for (PlanNode child : node.getChildren()) {
+      children.add(this.process(child, context));
+    }
+    List<TSDataType> dataTypes = getOutputColumnTypes(node, context.getTypeProvider());
+    int sortItemsCount = node.getOrderingScheme().getOrderBy().size();
+
+    List<Integer> sortItemIndexList = new ArrayList<>(sortItemsCount);
+    List<TSDataType> sortItemDataTypeList = new ArrayList<>(sortItemsCount);
+    genSortInformation(
+        node.getOutputSymbols(),
+        node.getOrderingScheme(),
+        sortItemIndexList,
+        sortItemDataTypeList,
+        context.getTypeProvider());
+    return new TopKOperator(
+        operatorContext,
+        children,
+        dataTypes,
+        getComparatorForTable(
+            node.getOrderingScheme().getOrderingList(), sortItemIndexList, sortItemDataTypeList),
+        node.getCount(),
+        node.isChildrenDataInOrder());
+  }
+
+  private List<TSDataType> getOutputColumnTypes(PlanNode node, TypeProvider typeProvider) {
+    return node.getOutputSymbols().stream()
+        .filter(s -> !TIMESTAMP_EXPRESSION_STRING.equalsIgnoreCase(s.getName()))
+        .map(s -> getTSDataType(typeProvider.getTableModelType(s)))
+        .collect(Collectors.toList());
+  }
+
+  private void genSortInformation(
+      List<Symbol> outputSymbols,
+      OrderingScheme orderingScheme,
+      List<Integer> sortItemIndexList,
+      List<TSDataType> sortItemDataTypeList,
+      TypeProvider typeProvider) {
+    Map<Symbol, Integer> columnIndex = new HashMap<>();
+    int index = 0;
+    for (Symbol symbol : outputSymbols) {
+      if (TIMESTAMP_EXPRESSION_STRING.equalsIgnoreCase(symbol.getName())) {
+        continue;
+      }
+      columnIndex.put(symbol, index++);
+    }
+    orderingScheme
+        .getOrderBy()
+        .forEach(
+            sortItem -> {
+              if (TIMESTAMP_EXPRESSION_STRING.equalsIgnoreCase(sortItem.getName())) {
+                sortItemIndexList.add(-1);
+                sortItemDataTypeList.add(TSDataType.INT64);
+              } else {
+                Integer i = columnIndex.get(sortItem);
+                if (i == null) {
+                  throw new IllegalStateException(
+                      "Sort Item %s is not included in children's output columns");
+                }
+                sortItemIndexList.add(i);
+                sortItemDataTypeList.add(getTSDataType(typeProvider.getTableModelType(sortItem)));
+              }
+            });
   }
 }
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/LimitNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/LimitNode.java
index e05545a8f97..d3399cb1e16 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/LimitNode.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/LimitNode.java
@@ -16,6 +16,7 @@ package org.apache.iotdb.db.queryengine.plan.relational.planner.node;
 
 import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
 import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor;
 import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.SingleChildProcessNode;
 import org.apache.iotdb.db.queryengine.plan.relational.planner.OrderingScheme;
 import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol;
@@ -45,9 +46,14 @@ public class LimitNode extends SingleChildProcessNode {
     return new LimitNode(id, child, count, tiesResolvingScheme);
   }
 
+  @Override
+  public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+    return visitor.visitLimit(this, context);
+  }
+
   @Override
   public List<String> getOutputColumnNames() {
-    return null;
+    throw new UnsupportedOperationException();
   }
 
   @Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/MergeSortNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/MergeSortNode.java
index 680d3812bac..68c1ffdc17e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/MergeSortNode.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/MergeSortNode.java
@@ -20,8 +20,9 @@ package org.apache.iotdb.db.queryengine.plan.relational.planner.node;
 
 import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
 import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor;
 import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.MultiChildProcessNode;
-import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.OrderByParameter;
+import org.apache.iotdb.db.queryengine.plan.relational.planner.OrderingScheme;
 import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol;
 
 import java.io.DataOutputStream;
@@ -30,25 +31,29 @@ import java.nio.ByteBuffer;
 import java.util.List;
 
 public class MergeSortNode extends MultiChildProcessNode {
-  private final OrderByParameter mergeOrderParameter;
+  private final OrderingScheme orderingScheme;
 
-  private final List<String> outputColumns;
+  private final List<Symbol> outputSymbols;
 
-  public MergeSortNode(
-      PlanNodeId id, OrderByParameter mergeOrderParameter, List<String> outputColumns) {
+  public MergeSortNode(PlanNodeId id, OrderingScheme orderingScheme, List<Symbol> outputSymbols) {
     super(id);
-    this.mergeOrderParameter = mergeOrderParameter;
-    this.outputColumns = outputColumns;
+    this.orderingScheme = orderingScheme;
+    this.outputSymbols = outputSymbols;
   }
 
   @Override
   public PlanNode clone() {
-    return null;
+    return new MergeSortNode(getPlanNodeId(), orderingScheme, outputSymbols);
+  }
+
+  @Override
+  public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+    return visitor.visitMergeSort(this, context);
   }
 
   @Override
   public List<String> getOutputColumnNames() {
-    return children.get(0).getOutputColumnNames();
+    throw new UnsupportedOperationException();
   }
 
   @Override
@@ -59,6 +64,10 @@ public class MergeSortNode extends MultiChildProcessNode {
 
   @Override
   public List<Symbol> getOutputSymbols() {
-    return super.getOutputSymbols();
+    return outputSymbols;
+  }
+
+  public OrderingScheme getOrderingScheme() {
+    return orderingScheme;
   }
 }
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/OffsetNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/OffsetNode.java
index 806ff99d317..feca446b9d0 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/OffsetNode.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/OffsetNode.java
@@ -16,6 +16,7 @@ package org.apache.iotdb.db.queryengine.plan.relational.planner.node;
 
 import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
 import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor;
 import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.SingleChildProcessNode;
 import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol;
 
@@ -39,7 +40,12 @@ public class OffsetNode extends SingleChildProcessNode {
 
   @Override
   public List<String> getOutputColumnNames() {
-    return null;
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+    return visitor.visitOffset(this, context);
   }
 
   @Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/SortNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/SortNode.java
index d51b131683f..474101ddd72 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/SortNode.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/SortNode.java
@@ -16,6 +16,7 @@ package org.apache.iotdb.db.queryengine.plan.relational.planner.node;
 
 import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
 import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor;
 import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.SingleChildProcessNode;
 import org.apache.iotdb.db.queryengine.plan.relational.planner.OrderingScheme;
 import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol;
@@ -42,7 +43,12 @@ public class SortNode extends SingleChildProcessNode {
 
   @Override
   public List<String> getOutputColumnNames() {
-    return null;
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+    return visitor.visitSort(this, context);
   }
 
   @Override
@@ -55,4 +61,8 @@ public class SortNode extends SingleChildProcessNode {
   public List<Symbol> getOutputSymbols() {
     return child.getOutputSymbols();
   }
+
+  public OrderingScheme getOrderingScheme() {
+    return orderingScheme;
+  }
 }
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TopKNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TopKNode.java
index d356d25f1d0..b554e852933 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TopKNode.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TopKNode.java
@@ -21,8 +21,10 @@ package org.apache.iotdb.db.queryengine.plan.relational.planner.node;
 
 import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
 import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor;
 import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.MultiChildProcessNode;
 import org.apache.iotdb.db.queryengine.plan.relational.planner.OrderingScheme;
+import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol;
 
 import java.io.DataOutputStream;
 import java.io.IOException;
@@ -33,22 +35,38 @@ public class TopKNode extends MultiChildProcessNode {
 
   private final OrderingScheme orderingScheme;
 
-  private final long count;
+  private final int count;
 
-  public TopKNode(PlanNodeId id, OrderingScheme scheme, long count) {
+  private final List<Symbol> outputSymbols;
+
+  private final boolean childrenDataInOrder;
+
+  public TopKNode(
+      PlanNodeId id,
+      OrderingScheme scheme,
+      int count,
+      List<Symbol> outputSymbols,
+      boolean childrenDataInOrder) {
     super(id);
     this.orderingScheme = scheme;
     this.count = count;
+    this.outputSymbols = outputSymbols;
+    this.childrenDataInOrder = childrenDataInOrder;
   }
 
   @Override
   public PlanNode clone() {
-    return null;
+    return new TopKNode(getPlanNodeId(), orderingScheme, count, outputSymbols, childrenDataInOrder);
+  }
+
+  @Override
+  public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+    return visitor.visitTopK(this, context);
   }
 
   @Override
   public List<String> getOutputColumnNames() {
-    return null;
+    throw new UnsupportedOperationException();
   }
 
   @Override
@@ -56,4 +74,21 @@ public class TopKNode extends MultiChildProcessNode {
 
   @Override
   protected void serializeAttributes(DataOutputStream stream) throws IOException {}
+
+  @Override
+  public List<Symbol> getOutputSymbols() {
+    return outputSymbols;
+  }
+
+  public OrderingScheme getOrderingScheme() {
+    return orderingScheme;
+  }
+
+  public int getCount() {
+    return count;
+  }
+
+  public boolean isChildrenDataInOrder() {
+    return childrenDataInOrder;
+  }
 }