You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ca...@apache.org on 2024/07/16 07:11:54 UTC
(iotdb) branch beyyes/sort_transform_elimate_topk updated (4982cef78b4 -> 4188b0edba9)
This is an automated email from the ASF dual-hosted git repository.
caogaofei pushed a change to branch beyyes/sort_transform_elimate_topk
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 4982cef78b4 add SortElimination rule
add 0526599c961 Fixed the bug that add column procedure is not idempotent & Added tests for table plan / procedures
add 8db2c3dbc9e Delete SchemaQueryMergeNode in LogicalPlanner
add ce6c81af584 Merge master0710
add 5997327f41f Fixed the bug that "show tables" fails when the database is first created and has no tables under it
new 22e003e9c31 Merge branch 'ty/TableModelGrammar' into beyyes/sort_transform_elimate_topk
add 3673568a4ed Fixed the bug that the configMTree can not be recovered when there are tables
add 3145d189530 Introduce IterativeOptimizer and adapt ProjectOffPushDownRules, InlineProjectionsRules
new 173d6eeb3e1 merge
new 4188b0edba9 add streamsort and sort elimination
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/daily-it.yml | 50 +
.github/workflows/multi-language-client.yml | 15 +-
distribution/src/assembly/all.xml | 9 -
distribution/src/assembly/confignode.xml | 9 -
distribution/src/assembly/datanode.xml | 9 -
docker/src/main/Dockerfile-1.0.0-datanode | 4 +-
.../apache/iotdb/SubscriptionSessionExample.java | 40 +-
integration-test/pom.xml | 14 +
.../it/env/cluster/config/MppCommonConfig.java | 6 +
.../env/cluster/config/MppSharedCommonConfig.java | 7 +
.../iotdb/it/env/cluster/env/AbstractEnv.java | 13 +-
.../it/env/remote/config/RemoteCommonConfig.java | 5 +
.../iotdb/it/env/remote/env/RemoteServerEnv.java | 6 +
.../org/apache/iotdb/itbase/category/DailyIT.java | 14 +-
.../java/org/apache/iotdb/itbase/env/BaseEnv.java | 3 +
.../org/apache/iotdb/itbase/env/CommonConfig.java | 2 +
.../apache/iotdb/cli/it/StartClientScriptIT.java | 65 +
.../it/cluster/IoTDBClusterNodeGetterIT.java | 1 -
.../it/partition/IoTDBTimePartitionIT.java | 150 ++
.../IoTDBRegionMigrateReliabilityITFramework.java | 55 +-
.../pass/IoTDBRegionMigrateClusterCrashIT.java | 3 +
.../pass/IoTDBRegionMigrateConfigNodeCrashIT.java | 3 +
.../pass/IoTDBRegionMigrateNormalIT.java | 3 +
...rateCoordinatorCrashWhenRemoveRemotePeerIT.java | 3 +
.../IoTDBRegionMigrateDataNodeCrashIT.java | 6 +
...nMigrateOriginalCrashWhenDeleteLocalPeerIT.java | 3 +
...MigrateOriginalCrashWhenRemoveRemotePeerIT.java | 3 +
.../confignode/it/utils/ConfigNodeTestUtils.java | 1 -
.../apache/iotdb/db/it/IoTDBJDBCMetadataIT.java | 135 ++
.../org/apache/iotdb/db/it/IoTDBRestServiceIT.java | 17 +-
.../iotdb/db/it/IoTDBSetConfigurationIT.java | 90 +-
...oTDBAlignByDeviceWithTemplateAggregationIT.java | 29 +
.../iotdb/db/it/auth/IoTDBSeriesPermissionIT.java | 2 +-
.../apache/iotdb/db/it/query/IoTDBResultSetIT.java | 2 +-
.../pipe/it/autocreate/IoTDBPipeAutoDropIT.java | 6 +
.../pipe/it/autocreate/IoTDBPipeDataSinkIT.java | 100 +-
.../pipe/it/autocreate/IoTDBPipeExtractorIT.java | 2 +-
.../pipe/it/manual/IoTDBPipeMetaHistoricalIT.java | 6 +-
.../relational/it/schema/IoTDBDatabaseIT.java | 2 +-
.../iotdb/relational/it/schema/IoTDBTableIT.java | 61 +-
.../it/session/IoTDBTableModelSessionIT.java | 25 +-
.../session/pool/IoTDBTableModelSessionPoolIT.java | 47 +-
.../it/dual/IoTDBSubscriptionConsumerGroupIT.java | 318 ++--
.../it/dual/IoTDBSubscriptionTopicIT.java | 300 ++--
.../it/local/IoTDBSubscriptionBasicIT.java | 302 +---
.../it/local/IoTDBSubscriptionDataTypeIT.java | 368 ++++
.../java/org/apache/iotdb/cli/AbstractCli.java | 27 +-
.../java/org/apache/iotdb/tool/ImportData.java | 2 +
.../java/org/apache/iotdb/tool/ImportSchema.java | 2 +
.../java/org/apache/iotdb/tool/ImportTsFile.java | 31 +-
.../client-py/iotdb/utils/IoTDBRpcDataSet.py | 2 +-
iotdb-client/client-py/requirements.txt | 4 +-
iotdb-client/client-py/requirements_dev.txt | 8 +-
iotdb-client/client-py/resources/setup.py | 6 +-
iotdb-client/client-py/tests/test_todf.py | 6 +-
.../org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java | 23 +-
.../org/apache/iotdb/jdbc/IoTDBResultMetadata.java | 44 +-
.../apache/iotdb/jdbc/IoTDBJDBCResultSetTest.java | 19 +-
.../java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java | 2 +-
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 2 +-
.../iotdb/rpc/subscription/config/TopicConfig.java | 23 +-
.../rpc/subscription/config/TopicConstant.java | 8 +-
.../payload/poll/SubscriptionCommitContext.java | 2 +
.../subscription/payload/poll/TabletsPayload.java | 10 +-
.../response/PipeSubscribeSubscribeResp.java | 32 +-
.../response/PipeSubscribeUnsubscribeResp.java | 32 +-
.../java/org/apache/iotdb/session/Session.java | 29 +-
.../org/apache/iotdb/session/pool/SessionPool.java | 28 +
.../consumer/SubscriptionConsumer.java | 118 +-
.../consumer/SubscriptionProvider.java | 10 +-
.../consumer/SubscriptionPullConsumer.java | 25 +
.../consumer/SubscriptionPushConsumer.java | 29 +-
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 2 +-
.../src/assembly/resources/conf/confignode-env.sh | 6 +-
.../assembly/resources/sbin/start-confignode.sh | 1 +
.../CnToCnInternalServiceAsyncRequestManager.java | 10 +
.../CnToDnInternalServiceAsyncRequestManager.java | 10 +
.../handlers/ConfigNodeAsyncRequestContext.java | 6 +
.../iotdb/confignode/conf/ConfigNodeConfig.java | 12 -
.../confignode/conf/ConfigNodeDescriptor.java | 21 +-
.../confignode/conf/ConfigNodeStartupCheck.java | 8 +-
.../conf/ConfigNodeSystemPropertiesHandler.java | 18 +-
.../confignode/conf/SystemPropertiesUtils.java | 97 +-
.../request/write/database/SetTTLPlan.java | 6 +
.../response/pipe/task/PipeTableResp.java | 67 +-
.../iotdb/confignode/manager/ClusterManager.java | 12 +-
.../iotdb/confignode/manager/ConfigManager.java | 39 +-
.../apache/iotdb/confignode/manager/IManager.java | 11 +
.../iotdb/confignode/manager/TTLManager.java | 16 +-
.../partition/DataPartitionPolicyTable.java | 2 +-
.../manager/node/ClusterNodeStartUtils.java | 12 +-
.../iotdb/confignode/manager/node/NodeManager.java | 15 +-
.../pipe/agent/task/PipeConfigNodeTaskAgent.java | 37 +-
.../protocol/IoTDBConfigRegionConnector.java | 13 +-
.../runtime/heartbeat/PipeHeartbeat.java | 4 +
.../runtime/heartbeat/PipeHeartbeatParser.java | 39 +-
.../pipe/event/PipeConfigRegionSnapshotEvent.java | 9 +-
.../pipe/execution/PipeConfigNodeSubtask.java | 2 +
.../resource/PipeConfigNodeResourceManager.java | 51 +
.../PipeConfigNodeSnapshotResourceManager.java | 11 +-
.../manager/schema/ClusterSchemaManager.java | 64 +-
.../iotdb/confignode/persistence/TTLInfo.java | 24 +-
.../confignode/persistence/pipe/PipeTaskInfo.java | 51 +-
.../persistence/schema/ClusterSchemaInfo.java | 33 +-
.../confignode/persistence/schema/ConfigMTree.java | 83 +-
.../persistence/schema/TemplateTable.java | 10 +-
.../procedure/env/RegionMaintainHandler.java | 17 +-
.../impl/pipe/task/AlterPipeProcedureV2.java | 44 +-
.../impl/schema/AlterLogicalViewProcedure.java | 2 +-
.../impl/schema/DataNodeRegionTaskExecutor.java | 16 +-
.../impl/schema/DeactivateTemplateProcedure.java | 12 +-
.../impl/schema/DeleteLogicalViewProcedure.java | 8 +-
.../impl/schema/DeleteTimeSeriesProcedure.java | 18 +-
.../procedure/impl/schema/SchemaUtils.java | 17 +-
.../impl/schema/SetTemplateProcedure.java | 3 +-
.../impl/schema/table/AddTableColumnProcedure.java | 201 ++-
.../impl/schema/table/CreateTableProcedure.java | 134 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 33 +-
.../request/ConfigPhysicalPlanSerDeTest.java | 76 +
.../iotdb/confignode/persistence/TTLInfoTest.java | 46 +-
.../persistence/schema/ConfigMTreeTest.java | 81 +
.../schema/table/AddTableColumnProcedureTest.java | 62 +
.../schema/table/CreateTableProcedureTest.java | 73 +
.../org/apache/iotdb/consensus/IConsensus.java | 8 +
.../apache/iotdb/consensus/iot/IoTConsensus.java | 6 +
.../iot/logdispatcher/IndexController.java | 4 +-
.../apache/iotdb/consensus/pipe/PipeConsensus.java | 6 +
.../ratis/ApplicationStateMachineProxy.java | 9 +-
.../iotdb/consensus/ratis/RatisConsensus.java | 12 +-
.../apache/iotdb/consensus/ratis/utils/Utils.java | 14 +-
.../iotdb/consensus/simple/SimpleConsensus.java | 5 +
.../apache/iotdb/consensus/iot/StabilityTest.java | 4 +
.../iotdb/consensus/ratis/RatisConsensusTest.java | 7 +
.../consensus/simple/SimpleConsensusTest.java | 6 +-
iotdb-core/datanode/pom.xml | 4 +-
.../src/assembly/resources/conf/datanode-env.sh | 6 +-
.../db/conf/DataNodeSystemPropertiesHandler.java | 16 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 18 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 303 ++--
.../org/apache/iotdb/db/conf/IoTDBStartCheck.java | 17 +-
.../dataregion/DataRegionStateMachine.java | 2 +-
.../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 95 +-
.../IoTDBDataNodeCacheLeaderClientManager.java | 8 +-
.../evolvable/batch/PipeTabletEventBatch.java | 133 ++
.../PipeTabletEventPlainBatch.java} | 121 +-
.../batch/PipeTabletEventTsFileBatch.java | 425 +++++
.../PipeTransferBatchReqBuilder.java | 96 +-
.../request/PipeTransferTabletRawReq.java | 133 +-
.../airgap/IoTDBDataRegionAirGapConnector.java | 3 +-
.../protocol/legacy/IoTDBLegacyPipeConnector.java | 14 +-
.../pipeconsensus/PipeConsensusAsyncConnector.java | 12 +-
.../pipeconsensus/PipeConsensusSyncConnector.java | 2 +
.../PipeConsensusTsFileInsertionEventHandler.java | 2 +
.../PipeConsensusTransferBatchReqBuilder.java | 12 +-
.../request/PipeConsensusTsFileSealReq.java | 9 +-
.../request/PipeConsensusTsFileSealWithModReq.java | 2 +
.../async/IoTDBDataRegionAsyncConnector.java | 238 ++-
.../PipeTransferTabletBatchEventHandler.java | 31 +-
...Handler.java => PipeTransferTsFileHandler.java} | 134 +-
.../thrift/sync/IoTDBDataRegionSyncConnector.java | 129 +-
.../thrift/sync/IoTDBSchemaRegionConnector.java | 13 +-
.../protocol/writeback/WriteBackConnector.java | 3 +-
.../pipe/connector/util/PipeTabletEventSorter.java | 202 +++
.../consensus/PipeConsensusConnectorMetrics.java | 19 +-
.../consensus/ProgressIndexDataNodeManager.java | 20 +-
.../schema/PipeSchemaRegionSnapshotEvent.java | 4 +-
.../tablet/PipeInsertNodeTabletInsertionEvent.java | 24 +-
.../common/tablet/PipeRawTabletInsertionEvent.java | 4 +-
.../tablet/TabletInsertionDataContainer.java | 6 +-
.../common/tsfile/PipeTsFileInsertionEvent.java | 50 +-
.../common/tsfile/TsFileInsertionPointCounter.java | 24 +-
.../container/TsFileInsertionDataContainer.java | 78 +
.../TsFileInsertionDataContainerProvider.java | 122 ++
.../query/TsFileInsertionQueryDataContainer.java} | 121 +-
.../TsFileInsertionQueryDataTabletIterator.java} | 22 +-
.../scan/AlignedSinglePageWholeChunkReader.java | 170 ++
.../container/scan/SinglePageWholeChunkReader.java | 116 ++
.../scan/TsFileInsertionScanDataContainer.java | 386 ++++
.../PipeHistoricalDataRegionTsFileExtractor.java | 71 +-
.../realtime/PipeRealtimeDataRegionExtractor.java | 43 +-
.../PipeRealtimeDataRegionHybridExtractor.java | 6 +-
.../realtime/assigner/DisruptorQueue.java | 12 +-
.../PipeDataNodeRemainingEventAndTimeMetrics.java | 20 +
.../PipeDataNodeRemainingEventAndTimeOperator.java | 29 +-
.../iotdb/db/pipe/metric/PipeProcessorMetrics.java | 37 +-
.../iotdb/db/pipe/metric/PipeResourceMetrics.java | 14 +-
.../downsampling/PartialPathLastObjectCache.java | 4 +-
.../pipeconsensus/PipeConsensusReceiver.java | 154 +-
.../pipeconsensus/PipeConsensusReceiverAgent.java | 53 +-
...nager.java => PipeDataNodeResourceManager.java} | 8 +-
.../db/pipe/resource/memory/PipeMemoryBlock.java | 18 +-
.../db/pipe/resource/memory/PipeMemoryManager.java | 63 +-
...oryWeighUtil.java => PipeMemoryWeightUtil.java} | 63 +-
.../pipe/resource/tsfile/PipeTsFileResource.java | 102 +-
.../resource/tsfile/PipeTsFileResourceManager.java | 10 +-
.../pipe/resource/wal/PipeWALResourceManager.java | 34 +-
.../hardlink/PipeWALHardlinkResourceManager.java | 23 +-
.../selfhost/PipeWALSelfHostResourceManager.java | 8 +-
.../pipe/task/builder/PipeDataNodeTaskBuilder.java | 23 +-
.../pipe/task/connection/PipeEventCollector.java | 29 +-
.../db/pipe/task/stage/PipeTaskProcessorStage.java | 20 +-
.../subtask/connector/PipeConnectorSubtask.java | 62 +-
.../PipeRealtimePriorityBlockingQueue.java | 11 +-
.../subtask/processor/PipeProcessorSubtask.java | 28 +-
.../iotdb/db/protocol/client/ConfigNodeClient.java | 58 +-
.../iotdb/db/protocol/client/ConfigNodeInfo.java | 2 +-
.../DnToCnInternalServiceAsyncRequestManager.java | 10 +
.../impl/DataNodeInternalRPCServiceImpl.java | 49 +-
.../db/queryengine/common/MPPQueryContext.java | 51 +-
.../common/header/ColumnHeaderConstant.java | 10 +-
.../fragment/FragmentInstanceContext.java | 19 +
.../fragment/FragmentInstanceExecution.java | 2 +
.../execution/load/LoadTsFileManager.java | 72 +-
.../operator/schema/source/DeviceSchemaSource.java | 2 +-
.../AbstractSeriesAggregationScanOperator.java | 2 +-
.../operator/source/AlignedSeriesScanOperator.java | 4 +-
.../operator/source/SeriesScanOperator.java | 3 +-
.../execution/operator/source/SeriesScanUtil.java | 3 +-
.../execution/schedule/AbstractDriverThread.java | 12 +-
.../schedule/DriverTaskAbortedException.java | 3 +
.../metric/load/LoadTsFileCostMetricsSet.java | 10 +-
.../iotdb/db/queryengine/plan/Coordinator.java | 2 +-
.../queryengine/plan/analyze/AnalyzeVisitor.java | 4 +-
.../plan/analyze/LoadTsfileAnalyzer.java | 4 +-
.../analyze/cache/partition/PartitionCache.java | 10 +-
.../analyze/cache/schema/DataNodeTTLCache.java | 43 +-
.../cache/schema/TimeSeriesSchemaCache.java | 2 +-
.../cache/schema/dualkeycache/impl/CacheStats.java | 2 -
.../schema/dualkeycache/impl/DualKeyCacheImpl.java | 2 +-
.../schema/lastcache/DataNodeLastCacheManager.java | 3 -
.../execution/config/TableConfigTaskVisitor.java | 30 +-
.../config/executor/ClusterConfigTaskExecutor.java | 82 +-
.../config/metadata/GetTimeSlotListTask.java | 6 +-
.../config/metadata/ShowVariablesTask.java | 10 +-
.../config/metadata/relational/ShowTablesTask.java | 37 +-
.../plan/optimization/AggregationPushDown.java | 28 +-
.../plan/planner/LocalExecutionPlanner.java | 13 +-
.../planner/memory/MemoryReservationManager.java | 46 +
.../NotThreadSafeMemoryReservationManager.java | 94 +
.../ThreadSafeMemoryReservationManager.java} | 33 +-
.../plan/planner/plan/node/PlanGraphPrinter.java | 5 +
.../plan/planner/plan/node/PlanNode.java | 14 +-
.../plan/planner/plan/node/PlanVisitor.java | 5 +
.../plan/node/pipe/PipeEnrichedInsertNode.java | 5 -
.../planner/plan/node/write/DeleteDataNode.java | 18 +-
.../plan/planner/plan/node/write/InsertNode.java | 44 +-
.../planner/plan/node/write/InsertRowsNode.java | 11 +
.../plan/planner/plan/node/write/SearchNode.java | 51 +
.../querystats/PlanOptimizersStatsCollector.java | 75 +
.../querystats/QueryPlanOptimizerStatistics.java | 57 +
.../querystats/QueryPlanOptimizerStats.java | 84 +
.../plan/relational/planner/Assignments.java | 20 +
.../relational/planner/ExpressionExtractor.java | 117 ++
.../planner/ExpressionSymbolInliner.java | 65 +
.../plan/relational/planner/LogicalPlanner.java | 88 +-
.../plan/relational/planner/PlanBuilder.java | 4 +-
.../plan/relational/planner/PlanNodeSearcher.java | 226 +++
.../plan/relational/planner/QueryPlanner.java | 4 +-
.../plan/relational/planner/SymbolsExtractor.java | 200 +++
.../plan/relational/planner/ir/IrUtils.java | 5 +
.../GroupReference.java} | 72 +-
.../planner/iterative/IterativeOptimizer.java | 365 ++++
.../plan/relational/planner/iterative/Lookup.java | 65 +
.../plan/relational/planner/iterative/Memo.java | 225 +++
.../plan/relational/planner/iterative/Plans.java | 54 +
.../plan/relational/planner/iterative/Rule.java | 87 +
.../relational/planner/iterative/RuleIndex.java | 76 +
.../relational/planner/iterative/RuleStats.java | 53 +
.../planner/iterative/RuleStatsRecorder.java | 45 +
.../planner/iterative/rule/InlineProjections.java | 220 +++
.../iterative/rule/ProjectOffPushDownRule.java | 67 +
.../planner/iterative/rule/PruneFilterColumns.java | 46 +
.../planner/iterative/rule/PruneLimitColumns.java | 36 +
.../planner/iterative/rule/PruneOffsetColumns.java | 36 +
.../iterative/rule/PruneOutputSourceColumns.java | 43 +
.../iterative/rule/PruneProjectColumns.java | 39 +
.../planner/iterative/rule/PruneSortColumns.java | 44 +
.../iterative/rule/PruneTableScanColumns.java | 102 ++
.../rule/RemoveRedundantIdentityProjections.java | 48 +
.../relational/planner/iterative/rule/Util.java | 137 ++
.../relational/planner/node/ChildReplacer.java | 32 +
.../plan/relational/planner/node/CollectNode.java | 12 +
.../plan/relational/planner/node/FilterNode.java | 6 +
.../plan/relational/planner/node/LimitNode.java | 11 +
.../relational/planner/node/MergeSortNode.java | 18 +
.../plan/relational/planner/node/OffsetNode.java | 6 +
.../plan/relational/planner/node/OutputNode.java | 6 +
.../plan/relational/planner/node/Patterns.java | 429 +++++
.../plan/relational/planner/node/ProjectNode.java | 10 +
.../plan/relational/planner/node/SortNode.java | 6 +
.../relational/planner/node/StreamSortNode.java | 12 +
.../relational/planner/node/TableScanNode.java | 12 +
.../plan/relational/planner/node/TopKNode.java | 22 +
.../optimizations/AdaptivePlanOptimizer.java | 56 +
.../planner/optimizations/PlanOptimizer.java | 77 +
.../planner/optimizations/PruneUnUsedColumns.java | 151 --
.../RemoveRedundantIdentityProjections.java | 97 -
.../plan/relational/utils/MoreLists.java | 53 +
.../plan/relational/utils/matching/Capture.java} | 31 +-
.../plan/relational/utils/matching/Captures.java | 86 +
.../relational/utils/matching/DefaultPrinter.java | 77 +
.../plan/relational/utils/matching/Match.java | 66 +
.../plan/relational/utils/matching/Pattern.java | 120 ++
.../relational/utils/matching/PatternVisitor.java | 46 +
.../plan/relational/utils/matching/Property.java | 89 +
.../relational/utils/matching/PropertyPattern.java | 51 +
.../utils/matching/pattern/CapturePattern.java | 53 +
.../utils/matching/pattern/EqualsPattern.java | 53 +
.../utils/matching/pattern/FilterPattern.java | 55 +
.../utils/matching/pattern/TypeOfPattern.java | 59 +
.../utils/matching/pattern/WithPattern.java | 65 +
.../plan/scheduler/load/LoadTsFileScheduler.java | 28 +-
.../statement/metadata/ShowDatabaseStatement.java | 24 +-
.../mtree/impl/pbtree/schemafile/SchemaFile.java | 10 +-
.../db/schemaengine/table/DataNodeTableCache.java | 140 +-
.../iotdb/db/schemaengine/table/ITableCache.java | 22 +-
.../java/org/apache/iotdb/db/service/DataNode.java | 3 +-
.../iotdb/db/service/metrics/WritingMetrics.java | 186 +-
.../iotdb/db/storageengine/StorageEngine.java | 6 -
.../db/storageengine/dataregion/DataRegion.java | 230 ++-
.../fast/reader/CompactionChunkReader.java | 17 +-
.../readchunk/loader/InstantChunkLoader.java | 12 +-
.../readchunk/loader/InstantPageLoader.java | 7 +-
.../compaction/repair/RepairDataFileScanUtil.java | 23 +-
.../compaction/schedule/CompactionScheduler.java | 5 +-
.../compaction/schedule/CompactionTaskManager.java | 8 +
.../estimator/AbstractCompactionEstimator.java | 6 +-
.../FastCompactionInnerCompactionEstimator.java | 15 +-
.../FastCrossSpaceCompactionEstimator.java | 15 +-
.../ReadChunkInnerCompactionEstimator.java | 19 +-
.../RepairUnsortedFileCompactionEstimator.java | 11 +-
.../dataregion/flush/CompressionRatio.java | 7 +
.../dataregion/memtable/AbstractMemTable.java | 32 +-
.../dataregion/memtable/TsFileProcessor.java | 15 +-
.../read/reader/common/PriorityMergeReader.java | 34 +-
.../storageengine/dataregion/wal/WALManager.java | 32 +-
.../dataregion/wal/buffer/WALBuffer.java | 53 +-
.../dataregion/wal/io/CheckpointWriter.java | 2 +-
.../storageengine/dataregion/wal/io/LogWriter.java | 53 +-
.../dataregion/wal/io/WALFileVersion.java | 61 +
.../dataregion/wal/io/WALInputStream.java | 166 +-
.../dataregion/wal/io/WALMetaData.java | 21 +-
.../storageengine/dataregion/wal/io/WALWriter.java | 36 +-
.../storageengine/dataregion/wal/node/WALNode.java | 18 +-
.../dataregion/wal/recover/WALNodeRecoverTask.java | 76 +-
...{WALRecoverWriter.java => WALRepairWriter.java} | 39 +-
.../file/UnsealedTsFileRecoverPerformer.java | 19 +
.../dataregion/wal/utils/WALEntryPosition.java | 1 -
.../dataregion/wal/utils/WALInsertNodeCache.java | 4 +-
.../agent/SubscriptionBrokerAgent.java | 3 +-
.../agent/SubscriptionConsumerAgent.java | 2 +-
.../subscription/agent/SubscriptionTopicAgent.java | 17 +
.../db/subscription/broker/SubscriptionBroker.java | 27 +-
.../broker/SubscriptionPrefetchingQueue.java | 243 ++-
.../broker/SubscriptionPrefetchingTabletQueue.java | 192 ++
.../SubscriptionPrefetchingTabletsQueue.java | 265 ---
.../broker/SubscriptionPrefetchingTsFileQueue.java | 388 ++--
.../db/subscription/event/SubscriptionEvent.java | 295 +++-
.../event/SubscriptionEventBinaryCache.java | 59 +-
.../event/SubscriptionTsFileEvent.java | 247 ---
.../batch/SubscriptionPipeTabletEventBatch.java | 149 ++
.../batch/SubscriptionPipeTsFileEventBatch.java | 61 +
.../event/pipe/SubscriptionPipeEmptyEvent.java} | 25 +-
.../event/pipe/SubscriptionPipeEvents.java} | 20 +-
.../pipe/SubscriptionPipeTabletBatchEvents.java} | 35 +-
.../pipe/SubscriptionPipeTsFileBatchEvents.java | 82 +
.../pipe/SubscriptionPipeTsFilePlainEvent.java | 56 +
.../receiver/SubscriptionReceiverV1.java | 39 +-
.../java/org/apache/iotdb/db/utils/MmapUtil.java | 8 +
.../pipe/connector/PipeTabletEventSorterTest.java | 218 +++
.../event/TsFileInsertionDataContainerTest.java | 232 ++-
.../PipeDataNodeSnapshotResourceManagerTest.java | 20 +-
.../execution/operator/OperatorMemoryTest.java | 19 +-
.../plan/planner/node/PlanGraphPrinterTest.java | 2 +-
.../plan/relational/analyzer/AnalyzerTest.java | 82 +-
.../analyzer/LimitOffsetPushDownTest.java | 45 +-
.../plan/relational/analyzer/SortTest.java | 1333 ++++----------
.../compaction/AbstractCompactionTest.java | 3 +
...eCompactionWithFastPerformerValidationTest.java | 4 +
...actionWithReadPointPerformerValidationTest.java | 4 +
...eCrossSpaceCompactionWithFastPerformerTest.java | 5 +
.../storageengine/dataregion/wal/WALTestUtils.java | 4 +-
.../wal/compression/WALCompressionTest.java | 39 +-
.../dataregion/wal/node/WALNodeTest.java | 2 +-
...verWriterTest.java => WALRepairWriterTest.java} | 31 +-
.../wal/utils/WALInsertNodeCacheTest.java | 2 +-
.../iotdb/metrics/core/type/IoTDBCounter.java | 4 +-
.../apache/iotdb/metrics/config/MetricConfig.java | 8 +-
iotdb-core/node-commons/pom.xml | 69 +-
.../resources/conf/generate_properties.bat | 65 -
.../assembly/resources/conf/generate_properties.sh | 59 -
.../resources/conf/iotdb-system.properties | 1855 +-------------------
...properties => iotdb-system.properties.template} | 1112 ++++++------
.../AsyncConfigNodeInternalServiceClient.java | 28 +
.../async/AsyncDataNodeInternalServiceClient.java | 28 +
.../client/request/AsyncRequestManager.java | 9 +-
.../{Utils.java => TestConnectionUtils.java} | 37 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 79 +-
.../iotdb/commons/conf/CommonDescriptor.java | 57 +-
.../iotdb/commons/conf/ConfigurationFileUtils.java | 124 +-
.../apache/iotdb/commons/conf/IoTDBConstant.java | 1 -
.../consensus/index/impl/RecoverProgressIndex.java | 5 +-
.../commons/file/SystemPropertiesHandler.java | 62 +-
.../iotdb/commons/pipe/config/PipeConfig.java | 5 +
.../config/constant/PipeConnectorConstant.java | 12 +-
.../config/constant/PipeExtractorConstant.java | 6 +-
.../request/PipeConsensusTransferFileSealReq.java | 11 +
.../PipeConsensusTransferFileSealWithModReq.java | 19 +
.../pipe/connector/protocol/IoTDBConnector.java | 40 +-
.../connector/protocol/IoTDBSslSyncConnector.java | 16 +-
.../iotdb/commons/pipe/event/EnrichedEvent.java | 2 +-
.../commons/pipe/pattern/IoTDBPipePattern.java | 4 +
.../commons}/pipe/resource/log/PipeLogManager.java | 7 +-
.../commons}/pipe/resource/log/PipeLogStatus.java | 6 +-
.../pipe/task/connection/BlockingPendingQueue.java | 22 +-
.../iotdb/commons/pipe/task/meta/PipeTaskMeta.java | 19 +-
.../task/subtask/PipeAbstractConnectorSubtask.java | 56 +-
.../apache/iotdb/commons/schema/table/TsTable.java | 18 +-
.../schema/table/TsTableInternalRPCType.java | 30 +-
.../schema/table/TsTableInternalRPCUtil.java | 12 +-
.../table/column/TsTableColumnSchemaUtil.java | 4 +-
.../apache/iotdb/commons/schema/ttl/TTLCache.java | 41 +-
.../iotdb/commons/service/metric/enums/Metric.java | 11 +-
.../subscription/config/SubscriptionConfig.java | 31 +-
.../iotdb/commons/utils/CommonDateTimeUtils.java | 10 +
.../iotdb/commons/utils/TimePartitionUtils.java | 35 +-
.../commons/utils/TimePartitionUtilsTest.java | 92 +
iotdb-doap.rdf | 29 +-
.../src/main/thrift/confignode.thrift | 48 +-
pom.xml | 14 +-
430 files changed, 15691 insertions(+), 8671 deletions(-)
create mode 100644 .github/workflows/daily-it.yml
copy iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/CheckpointWriter.java => integration-test/src/main/java/org/apache/iotdb/itbase/category/DailyIT.java (66%)
create mode 100644 integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBTimePartitionIT.java
create mode 100644 integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBJDBCMetadataIT.java
create mode 100644 integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionDataTypeIT.java
create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/resource/PipeConfigNodeResourceManager.java
create mode 100644 iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedureTest.java
create mode 100644 iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedureTest.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventBatch.java
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/{builder/PipeEventBatch.java => batch/PipeTabletEventPlainBatch.java} (60%)
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventTsFileBatch.java
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/{builder => batch}/PipeTransferBatchReqBuilder.java (55%)
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/{PipeTransferTsFileInsertionEventHandler.java => PipeTransferTsFileHandler.java} (66%)
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/PipeTabletEventSorter.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/TsFileInsertionDataContainer.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/TsFileInsertionDataContainerProvider.java
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/{TsFileInsertionDataContainer.java => container/query/TsFileInsertionQueryDataContainer.java} (77%)
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/{TsFileInsertionDataTabletIterator.java => container/query/TsFileInsertionQueryDataTabletIterator.java} (89%)
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/scan/AlignedSinglePageWholeChunkReader.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/scan/SinglePageWholeChunkReader.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/scan/TsFileInsertionScanDataContainer.java
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/{PipeResourceManager.java => PipeDataNodeResourceManager.java} (93%)
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/{PipeMemoryWeighUtil.java => PipeMemoryWeightUtil.java} (56%)
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/memory/MemoryReservationManager.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/memory/NotThreadSafeMemoryReservationManager.java
copy iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{pipe/resource/wal/selfhost/PipeWALSelfHostResourceManager.java => queryengine/plan/planner/memory/ThreadSafeMemoryReservationManager.java} (50%)
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/SearchNode.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/execution/querystats/PlanOptimizersStatsCollector.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/execution/querystats/QueryPlanOptimizerStatistics.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/execution/querystats/QueryPlanOptimizerStats.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ExpressionExtractor.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ExpressionSymbolInliner.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/PlanNodeSearcher.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SymbolsExtractor.java
copy iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/{node/OffsetNode.java => iterative/GroupReference.java} (51%)
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/IterativeOptimizer.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/Lookup.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/Memo.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/Plans.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/Rule.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/RuleIndex.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/RuleStats.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/RuleStatsRecorder.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/InlineProjections.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/ProjectOffPushDownRule.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneFilterColumns.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneLimitColumns.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneOffsetColumns.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneOutputSourceColumns.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneProjectColumns.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneSortColumns.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneTableScanColumns.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/RemoveRedundantIdentityProjections.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/Util.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/ChildReplacer.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/Patterns.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/AdaptivePlanOptimizer.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PlanOptimizer.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PruneUnUsedColumns.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/RemoveRedundantIdentityProjections.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/MoreLists.java
copy iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{pipe/resource/log/PipeLogManager.java => queryengine/plan/relational/utils/matching/Capture.java} (56%)
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/matching/Captures.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/matching/DefaultPrinter.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/matching/Match.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/matching/Pattern.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/matching/PatternVisitor.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/matching/Property.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/matching/PropertyPattern.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/matching/pattern/CapturePattern.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/matching/pattern/EqualsPattern.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/matching/pattern/FilterPattern.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/matching/pattern/TypeOfPattern.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/matching/pattern/WithPattern.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALFileVersion.java
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/{WALRecoverWriter.java => WALRepairWriter.java} (60%)
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTabletQueue.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTabletsQueue.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/SubscriptionTsFileEvent.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTabletEventBatch.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTsFileEventBatch.java
copy iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{storageengine/dataregion/wal/io/CheckpointWriter.java => subscription/event/pipe/SubscriptionPipeEmptyEvent.java} (68%)
copy iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{storageengine/dataregion/wal/io/CheckpointWriter.java => subscription/event/pipe/SubscriptionPipeEvents.java} (68%)
copy iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{pipe/resource/wal/selfhost/PipeWALSelfHostResourceManager.java => subscription/event/pipe/SubscriptionPipeTabletBatchEvents.java} (55%)
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/pipe/SubscriptionPipeTsFileBatchEvents.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/pipe/SubscriptionPipeTsFilePlainEvent.java
create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/connector/PipeTabletEventSorterTest.java
rename iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/{WALRecoverWriterTest.java => WALRepairWriterTest.java} (93%)
delete mode 100644 iotdb-core/node-commons/src/assembly/resources/conf/generate_properties.bat
delete mode 100644 iotdb-core/node-commons/src/assembly/resources/conf/generate_properties.sh
copy iotdb-core/node-commons/src/assembly/resources/conf/{iotdb-system.properties => iotdb-system.properties.template} (81%)
rename iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/request/{Utils.java => TestConnectionUtils.java} (71%)
rename iotdb-core/{datanode/src/main/java/org/apache/iotdb/db => node-commons/src/main/java/org/apache/iotdb/commons}/pipe/resource/log/PipeLogManager.java (88%)
rename iotdb-core/{datanode/src/main/java/org/apache/iotdb/db => node-commons/src/main/java/org/apache/iotdb/commons}/pipe/resource/log/PipeLogStatus.java (87%)
create mode 100644 iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/TimePartitionUtilsTest.java
(iotdb) 03/03: add streamsort and sort elimination
Posted by ca...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
caogaofei pushed a commit to branch beyyes/sort_transform_elimate_topk
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 4188b0edba9a9472dac77ef51d9608f2c7825aa6
Author: Beyyes <cg...@foxmail.com>
AuthorDate: Tue Jul 16 15:11:38 2024 +0800
add streamsort and sort elimination
---
.../relational/planner/node/StreamSortNode.java | 12 +
.../analyzer/LimitOffsetPushDownTest.java | 9 +-
.../plan/relational/analyzer/SortTest.java | 486 +++++----------------
3 files changed, 133 insertions(+), 374 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/StreamSortNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/StreamSortNode.java
index 6f5b7582418..7ee9dd322fa 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/StreamSortNode.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/StreamSortNode.java
@@ -26,11 +26,13 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor;
import org.apache.iotdb.db.queryengine.plan.relational.planner.OrderingScheme;
import com.google.common.base.Objects;
+import com.google.common.collect.Iterables;
import org.apache.tsfile.utils.ReadWriteIOUtils;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.List;
public class StreamSortNode extends SortNode {
@@ -50,6 +52,16 @@ public class StreamSortNode extends SortNode {
return streamCompareKeyEndIndex;
}
+ @Override
+ public PlanNode replaceChildren(List<PlanNode> newChildren) {
+ return new StreamSortNode(
+ id,
+ Iterables.getOnlyElement(newChildren),
+ orderingScheme,
+ partial,
+ streamCompareKeyEndIndex);
+ }
+
@Override
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitStreamSort(this, context);
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/LimitOffsetPushDownTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/LimitOffsetPushDownTest.java
index 31b33a772b4..59490b86269 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/LimitOffsetPushDownTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/LimitOffsetPushDownTest.java
@@ -28,6 +28,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.LogicalPlanner;
import org.apache.iotdb.db.queryengine.plan.relational.planner.distribute.TableDistributionPlanner;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CollectNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.MergeSortNode;
+import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.PushPredicateIntoTableScan;
@@ -108,7 +109,7 @@ public class LimitOffsetPushDownTest {
}
// order by all tags, limit can be pushed into TableScan, pushLimitToEachDevice==false
- // Output - Limit - Offset - Project - MergeSort - Sort - Project - TableScan
+ // Output - Limit - Offset - Project - MergeSort - Project - TableScan
@Test
public void orderByAllTagsTest() {
sql =
@@ -128,9 +129,9 @@ public class LimitOffsetPushDownTest {
(MergeSortNode)
getChildrenNode(distributedQueryPlan.getFragments().get(0).getPlanNodeTree(), 5);
assertTrue(mergeSortNode.getChildren().get(0) instanceof ExchangeNode);
- assertTrue(mergeSortNode.getChildren().get(1) instanceof SortNode);
+ assertTrue(mergeSortNode.getChildren().get(1) instanceof ProjectNode);
assertTrue(mergeSortNode.getChildren().get(2) instanceof ExchangeNode);
- tableScanNode = (TableScanNode) getChildrenNode(mergeSortNode.getChildren().get(1), 2);
+ tableScanNode = (TableScanNode) getChildrenNode(mergeSortNode.getChildren().get(1), 1);
assertEquals(4, tableScanNode.getDeviceEntries().size());
assertEquals(DESC, tableScanNode.getScanOrder());
assertTrue(tableScanNode.getPushDownLimit() == 15 && tableScanNode.getPushDownOffset() == 0);
@@ -138,7 +139,7 @@ public class LimitOffsetPushDownTest {
tableScanNode =
(TableScanNode)
- getChildrenNode(distributedQueryPlan.getFragments().get(1).getPlanNodeTree(), 3);
+ getChildrenNode(distributedQueryPlan.getFragments().get(1).getPlanNodeTree(), 2);
assertEquals(2, tableScanNode.getDeviceEntries().size());
assertEquals(DESC, tableScanNode.getScanOrder());
assertTrue(tableScanNode.getPushDownLimit() == 15 && tableScanNode.getPushDownOffset() == 0);
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/SortTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/SortTest.java
index 3286313881a..4501848c444 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/SortTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/SortTest.java
@@ -79,6 +79,10 @@ public class SortTest {
LogicalPlanner logicalPlanner;
LogicalQueryPlan logicalQueryPlan;
PlanNode rootNode;
+ OutputNode outputNode;
+ PlanNode mergeSortNode;
+ ProjectNode projectNode;
+ StreamSortNode streamSortNode;
TableDistributionPlanner distributionPlanner;
DistributedQueryPlan distributedQueryPlan;
TableScanNode tableScanNode;
@@ -704,72 +708,40 @@ public class SortTest {
.plan(actualAnalysis);
rootNode = logicalQueryPlan.getRootNode();
- // LogicalPlan: `Output - Limit - Offset - Project - StreamSort - Project - Filter - TableScan`
+ // LogicalPlan: `Output-Limit-Offset-Project-StreamSort-Project-Filter-TableScan`
assertTrue(rootNode instanceof OutputNode);
assertTrue(getChildrenNode(rootNode, 1) instanceof LimitNode);
assertTrue(getChildrenNode(rootNode, 2) instanceof OffsetNode);
assertTrue(getChildrenNode(rootNode, 3) instanceof ProjectNode);
assertTrue(getChildrenNode(rootNode, 4) instanceof StreamSortNode);
- StreamSortNode streamSortNode =
- (StreamSortNode)
- rootNode
- .getChildren()
- .get(0)
- .getChildren()
- .get(0)
- .getChildren()
- .get(0)
- .getChildren()
- .get(0);
- assertTrue(streamSortNode.getChildren().get(0) instanceof ProjectNode);
- assertTrue(streamSortNode.getChildren().get(0).getChildren().get(0) instanceof FilterNode);
- assertTrue(
- streamSortNode.getChildren().get(0).getChildren().get(0).getChildren().get(0)
- instanceof TableScanNode);
- tableScanNode =
- (TableScanNode)
- streamSortNode.getChildren().get(0).getChildren().get(0).getChildren().get(0);
+ streamSortNode = (StreamSortNode) getChildrenNode(rootNode, 4);
+ assertTrue(getChildrenNode(streamSortNode, 1) instanceof ProjectNode);
+ assertTrue(getChildrenNode(streamSortNode, 2) instanceof FilterNode);
+ assertTrue(getChildrenNode(streamSortNode, 3) instanceof TableScanNode);
+ tableScanNode = (TableScanNode) getChildrenNode(streamSortNode, 3);
assertEquals("testdb.table1", tableScanNode.getQualifiedObjectName().toString());
assertEquals(8, tableScanNode.getAssignments().size());
assertEquals(6, tableScanNode.getDeviceEntries().size());
assertEquals(5, tableScanNode.getIdAndAttributeIndexMap().size());
- // DistributePlan: `Output - Limit - Offset - Project - MergeSort - StreamSort - Project -
- // Filter - TableScan`
+ // DistributePlan: `Output-Limit-Offset-Project-MergeSort-StreamSort-Project-Filter-TableScan`
distributionPlanner = new TableDistributionPlanner(actualAnalysis, logicalQueryPlan, context);
distributedQueryPlan = distributionPlanner.plan();
assertEquals(3, distributedQueryPlan.getFragments().size());
- assertTrue(
- distributedQueryPlan.getFragments().get(0).getPlanNodeTree().getChildren().get(0)
- instanceof OutputNode);
- OutputNode outputNode =
+ outputNode =
(OutputNode)
distributedQueryPlan.getFragments().get(0).getPlanNodeTree().getChildren().get(0);
- assertTrue(outputNode.getChildren().get(0) instanceof LimitNode);
- assertTrue(outputNode.getChildren().get(0).getChildren().get(0) instanceof OffsetNode);
- assertTrue(
- outputNode.getChildren().get(0).getChildren().get(0).getChildren().get(0)
- instanceof ProjectNode);
- MergeSortNode mergeSortNode =
- (MergeSortNode)
- outputNode
- .getChildren()
- .get(0)
- .getChildren()
- .get(0)
- .getChildren()
- .get(0)
- .getChildren()
- .get(0);
+ assertTrue(getChildrenNode(outputNode, 1) instanceof LimitNode);
+ assertTrue(getChildrenNode(outputNode, 2) instanceof OffsetNode);
+ assertTrue(getChildrenNode(outputNode, 3) instanceof ProjectNode);
+ MergeSortNode mergeSortNode = (MergeSortNode) getChildrenNode(outputNode, 4);
assertTrue(mergeSortNode.getChildren().get(0) instanceof ExchangeNode);
assertTrue(mergeSortNode.getChildren().get(1) instanceof StreamSortNode);
assertTrue(mergeSortNode.getChildren().get(2) instanceof ExchangeNode);
streamSortNode = (StreamSortNode) mergeSortNode.getChildren().get(1);
- assertTrue(streamSortNode.getChildren().get(0) instanceof ProjectNode);
- assertTrue(streamSortNode.getChildren().get(0).getChildren().get(0) instanceof FilterNode);
- tableScanNode =
- (TableScanNode)
- streamSortNode.getChildren().get(0).getChildren().get(0).getChildren().get(0);
+ assertTrue(getChildrenNode(streamSortNode, 1) instanceof ProjectNode);
+ assertTrue(getChildrenNode(streamSortNode, 2) instanceof FilterNode);
+ tableScanNode = (TableScanNode) getChildrenNode(streamSortNode, 3);
assertEquals(4, tableScanNode.getDeviceEntries().size());
assertEquals(
Arrays.asList(
@@ -783,47 +755,12 @@ public class SortTest {
assertEquals(DESC, tableScanNode.getScanOrder());
// DistributePlan: `IdentitySink - StreamSort - Project - Filter - TableScan`
- assertTrue(
- distributedQueryPlan.getFragments().get(1).getPlanNodeTree() instanceof IdentitySinkNode);
- assertTrue(
- distributedQueryPlan.getFragments().get(1).getPlanNodeTree().getChildren().get(0)
- instanceof SortNode);
- assertTrue(
- distributedQueryPlan
- .getFragments()
- .get(1)
- .getPlanNodeTree()
- .getChildren()
- .get(0)
- .getChildren()
- .get(0)
- instanceof ProjectNode);
- assertTrue(
- distributedQueryPlan
- .getFragments()
- .get(1)
- .getPlanNodeTree()
- .getChildren()
- .get(0)
- .getChildren()
- .get(0)
- .getChildren()
- .get(0)
- instanceof FilterNode);
- tableScanNode =
- (TableScanNode)
- distributedQueryPlan
- .getFragments()
- .get(1)
- .getPlanNodeTree()
- .getChildren()
- .get(0)
- .getChildren()
- .get(0)
- .getChildren()
- .get(0)
- .getChildren()
- .get(0);
+ streamSortNode =
+ (StreamSortNode)
+ distributedQueryPlan.getFragments().get(1).getPlanNodeTree().getChildren().get(0);
+ assertTrue(getChildrenNode(streamSortNode, 1) instanceof ProjectNode);
+ assertTrue(getChildrenNode(streamSortNode, 2) instanceof FilterNode);
+ tableScanNode = (TableScanNode) getChildrenNode(streamSortNode, 3);
assertEquals(2, tableScanNode.getDeviceEntries().size());
assertEquals(
Arrays.asList("table1.shenzhen.B2.ZZ", "table1.shenzhen.B1.XX"),
@@ -846,70 +783,39 @@ public class SortTest {
.plan(actualAnalysis);
rootNode = logicalQueryPlan.getRootNode();
- // OutputNode - LimitNode - OffsetNode - ProjectNode - SortNode - ProjectNode - FilterNode -
- // TableScanNode
+ // LogicalPlan: `Output-Limit-Offset-Project-StreamSort-Project-Filter-TableScan`
assertTrue(rootNode instanceof OutputNode);
- assertTrue(rootNode.getChildren().get(0) instanceof LimitNode);
- assertTrue(rootNode.getChildren().get(0).getChildren().get(0) instanceof OffsetNode);
- assertTrue(
- rootNode.getChildren().get(0).getChildren().get(0).getChildren().get(0)
- instanceof ProjectNode);
- SortNode sortNode =
- (SortNode)
- rootNode
- .getChildren()
- .get(0)
- .getChildren()
- .get(0)
- .getChildren()
- .get(0)
- .getChildren()
- .get(0);
- assertTrue(sortNode.getChildren().get(0) instanceof ProjectNode);
- assertTrue(sortNode.getChildren().get(0).getChildren().get(0) instanceof FilterNode);
- tableScanNode =
- (TableScanNode) sortNode.getChildren().get(0).getChildren().get(0).getChildren().get(0);
+ assertTrue(getChildrenNode(rootNode, 1) instanceof LimitNode);
+ assertTrue(getChildrenNode(rootNode, 2) instanceof OffsetNode);
+ assertTrue(getChildrenNode(rootNode, 3) instanceof ProjectNode);
+ assertTrue(getChildrenNode(rootNode, 4) instanceof StreamSortNode);
+ StreamSortNode streamSortNode = (StreamSortNode) getChildrenNode(rootNode, 4);
+ assertTrue(getChildrenNode(streamSortNode, 1) instanceof ProjectNode);
+ assertTrue(getChildrenNode(streamSortNode, 2) instanceof FilterNode);
+ assertTrue(getChildrenNode(streamSortNode, 3) instanceof TableScanNode);
+ tableScanNode = (TableScanNode) getChildrenNode(streamSortNode, 3);
assertEquals("testdb.table1", tableScanNode.getQualifiedObjectName().toString());
assertEquals(8, tableScanNode.getAssignments().size());
assertEquals(6, tableScanNode.getDeviceEntries().size());
assertEquals(5, tableScanNode.getIdAndAttributeIndexMap().size());
- // OutputNode - LimitNode - OffsetNode - ProjectNode - MergeSortNode - SortNode - ProjectNode -
- // FilterNode -
- // TableScanNode
+ // DistributePlan: `Output-Limit-Offset-Project-MergeSort-Project-Filter-TableScan`
distributionPlanner = new TableDistributionPlanner(actualAnalysis, logicalQueryPlan, context);
distributedQueryPlan = distributionPlanner.plan();
assertEquals(3, distributedQueryPlan.getFragments().size());
- assertTrue(
- distributedQueryPlan.getFragments().get(0).getPlanNodeTree().getChildren().get(0)
- instanceof OutputNode);
- OutputNode outputNode =
+ outputNode =
(OutputNode)
distributedQueryPlan.getFragments().get(0).getPlanNodeTree().getChildren().get(0);
- assertTrue(outputNode.getChildren().get(0) instanceof LimitNode);
- assertTrue(outputNode.getChildren().get(0).getChildren().get(0) instanceof OffsetNode);
- assertTrue(
- outputNode.getChildren().get(0).getChildren().get(0).getChildren().get(0)
- instanceof ProjectNode);
- MergeSortNode mergeSortNode =
- (MergeSortNode)
- outputNode
- .getChildren()
- .get(0)
- .getChildren()
- .get(0)
- .getChildren()
- .get(0)
- .getChildren()
- .get(0);
+ assertTrue(getChildrenNode(outputNode, 1) instanceof LimitNode);
+ assertTrue(getChildrenNode(outputNode, 2) instanceof OffsetNode);
+ assertTrue(getChildrenNode(outputNode, 3) instanceof ProjectNode);
+ MergeSortNode mergeSortNode = (MergeSortNode) getChildrenNode(outputNode, 4);
assertTrue(mergeSortNode.getChildren().get(0) instanceof ExchangeNode);
- assertTrue(mergeSortNode.getChildren().get(1) instanceof SortNode);
+ assertTrue(mergeSortNode.getChildren().get(1) instanceof ProjectNode);
assertTrue(mergeSortNode.getChildren().get(2) instanceof ExchangeNode);
- sortNode = (SortNode) mergeSortNode.getChildren().get(1);
- assertTrue(sortNode.getChildren().get(0) instanceof ProjectNode);
- assertTrue(sortNode.getChildren().get(0).getChildren().get(0) instanceof FilterNode);
- tableScanNode =
- (TableScanNode) sortNode.getChildren().get(0).getChildren().get(0).getChildren().get(0);
+ projectNode = (ProjectNode) mergeSortNode.getChildren().get(1);
+ assertTrue(getChildrenNode(projectNode, 1) instanceof FilterNode);
+ tableScanNode = (TableScanNode) getChildrenNode(projectNode, 2);
assertEquals(4, tableScanNode.getDeviceEntries().size());
assertEquals(
Arrays.asList(
@@ -922,48 +828,12 @@ public class SortTest {
.collect(Collectors.toList()));
assertEquals(DESC, tableScanNode.getScanOrder());
- // IdentitySinkNode - SortNode - ProjectNode - FilterNode - TableScanNode
- assertTrue(
- distributedQueryPlan.getFragments().get(1).getPlanNodeTree() instanceof IdentitySinkNode);
- assertTrue(
- distributedQueryPlan.getFragments().get(1).getPlanNodeTree().getChildren().get(0)
- instanceof SortNode);
- assertTrue(
- distributedQueryPlan
- .getFragments()
- .get(1)
- .getPlanNodeTree()
- .getChildren()
- .get(0)
- .getChildren()
- .get(0)
- instanceof ProjectNode);
- assertTrue(
- distributedQueryPlan
- .getFragments()
- .get(1)
- .getPlanNodeTree()
- .getChildren()
- .get(0)
- .getChildren()
- .get(0)
- .getChildren()
- .get(0)
- instanceof FilterNode);
- tableScanNode =
- (TableScanNode)
- distributedQueryPlan
- .getFragments()
- .get(1)
- .getPlanNodeTree()
- .getChildren()
- .get(0)
- .getChildren()
- .get(0)
- .getChildren()
- .get(0)
- .getChildren()
- .get(0);
+ // DistributePlan: `IdentitySink-Project-Filter-TableScan`
+ projectNode =
+ (ProjectNode)
+ distributedQueryPlan.getFragments().get(1).getPlanNodeTree().getChildren().get(0);
+ assertTrue(getChildrenNode(projectNode, 1) instanceof FilterNode);
+ tableScanNode = (TableScanNode) getChildrenNode(projectNode, 2);
assertEquals(2, tableScanNode.getDeviceEntries().size());
assertEquals(
Arrays.asList("table1.shenzhen.B2.ZZ", "table1.shenzhen.B1.XX"),
@@ -986,73 +856,41 @@ public class SortTest {
.plan(actualAnalysis);
rootNode = logicalQueryPlan.getRootNode();
- // OutputNode - LimitNode - OffsetNode - ProjectNode - SortNode - ProjectNode - FilterNode -
- // TableScanNode
+ // LogicalPlan: `Output-Limit-Offset-Project-StreamSort-Project-Filter-TableScan`
assertTrue(rootNode instanceof OutputNode);
- assertTrue(rootNode.getChildren().get(0) instanceof LimitNode);
- assertTrue(rootNode.getChildren().get(0).getChildren().get(0) instanceof OffsetNode);
- assertTrue(
- rootNode.getChildren().get(0).getChildren().get(0).getChildren().get(0)
- instanceof ProjectNode);
- SortNode sortNode =
- (SortNode)
- rootNode
- .getChildren()
- .get(0)
- .getChildren()
- .get(0)
- .getChildren()
- .get(0)
- .getChildren()
- .get(0);
- assertTrue(sortNode.getChildren().get(0) instanceof ProjectNode);
- assertTrue(sortNode.getChildren().get(0).getChildren().get(0) instanceof FilterNode);
- tableScanNode =
- (TableScanNode) sortNode.getChildren().get(0).getChildren().get(0).getChildren().get(0);
+ assertTrue(getChildrenNode(rootNode, 1) instanceof LimitNode);
+ assertTrue(getChildrenNode(rootNode, 2) instanceof OffsetNode);
+ assertTrue(getChildrenNode(rootNode, 3) instanceof ProjectNode);
+ assertTrue(getChildrenNode(rootNode, 4) instanceof StreamSortNode);
+ streamSortNode = (StreamSortNode) getChildrenNode(rootNode, 4);
+ assertEquals(2, streamSortNode.getStreamCompareKeyEndIndex());
+ assertTrue(getChildrenNode(streamSortNode, 1) instanceof ProjectNode);
+ assertTrue(getChildrenNode(streamSortNode, 2) instanceof FilterNode);
+ assertTrue(getChildrenNode(streamSortNode, 3) instanceof TableScanNode);
+ tableScanNode = (TableScanNode) getChildrenNode(streamSortNode, 3);
assertEquals("testdb.table1", tableScanNode.getQualifiedObjectName().toString());
assertEquals(8, tableScanNode.getAssignments().size());
assertEquals(6, tableScanNode.getDeviceEntries().size());
assertEquals(5, tableScanNode.getIdAndAttributeIndexMap().size());
- // OutputNode - LimitNode - OffsetNode - ProjectNode - MergeSortNode - SortNode - ProjectNode -
- // FilterNode -
- // TableScanNode
+ // DistributePlan: `Output-Limit-Offset-Project-MergeSort-StreamSort-Project-Filter-TableScan`
distributionPlanner = new TableDistributionPlanner(actualAnalysis, logicalQueryPlan, context);
distributedQueryPlan = distributionPlanner.plan();
assertEquals(3, distributedQueryPlan.getFragments().size());
- assertTrue(
- distributedQueryPlan.getFragments().get(0).getPlanNodeTree().getChildren().get(0)
- instanceof OutputNode);
- OutputNode outputNode =
+ outputNode =
(OutputNode)
distributedQueryPlan.getFragments().get(0).getPlanNodeTree().getChildren().get(0);
- assertTrue(outputNode.getChildren().get(0) instanceof LimitNode);
- assertTrue(outputNode.getChildren().get(0).getChildren().get(0) instanceof OffsetNode);
- assertTrue(
- outputNode.getChildren().get(0).getChildren().get(0).getChildren().get(0)
- instanceof ProjectNode);
- MergeSortNode mergeSortNode =
- (MergeSortNode)
- outputNode
- .getChildren()
- .get(0)
- .getChildren()
- .get(0)
- .getChildren()
- .get(0)
- .getChildren()
- .get(0);
+ assertTrue(getChildrenNode(outputNode, 1) instanceof LimitNode);
+ assertTrue(getChildrenNode(outputNode, 2) instanceof OffsetNode);
+ assertTrue(getChildrenNode(outputNode, 3) instanceof ProjectNode);
+ MergeSortNode mergeSortNode = (MergeSortNode) getChildrenNode(outputNode, 4);
assertTrue(mergeSortNode.getChildren().get(0) instanceof ExchangeNode);
- assertTrue(mergeSortNode.getChildren().get(1) instanceof SortNode);
+ assertTrue(mergeSortNode.getChildren().get(1) instanceof StreamSortNode);
assertTrue(mergeSortNode.getChildren().get(2) instanceof ExchangeNode);
- sortNode = (SortNode) mergeSortNode.getChildren().get(1);
- assertTrue(sortNode.getChildren().get(0) instanceof ProjectNode);
- assertTrue(sortNode.getChildren().get(0).getChildren().get(0) instanceof FilterNode);
- assertTrue(
- sortNode.getChildren().get(0).getChildren().get(0).getChildren().get(0)
- instanceof TableScanNode);
- tableScanNode =
- (TableScanNode) sortNode.getChildren().get(0).getChildren().get(0).getChildren().get(0);
+ streamSortNode = (StreamSortNode) mergeSortNode.getChildren().get(1);
+ assertTrue(getChildrenNode(streamSortNode, 1) instanceof ProjectNode);
+ assertTrue(getChildrenNode(streamSortNode, 2) instanceof FilterNode);
+ tableScanNode = (TableScanNode) getChildrenNode(streamSortNode, 3);
assertEquals(4, tableScanNode.getDeviceEntries().size());
assertEquals(
Arrays.asList(
@@ -1065,48 +903,13 @@ public class SortTest {
.collect(Collectors.toList()));
assertEquals(ASC, tableScanNode.getScanOrder());
- // IdentitySinkNode - SortNode - ProjectNode - FilterNode - TableScanNode
- assertTrue(
- distributedQueryPlan.getFragments().get(1).getPlanNodeTree() instanceof IdentitySinkNode);
- assertTrue(
- distributedQueryPlan.getFragments().get(1).getPlanNodeTree().getChildren().get(0)
- instanceof SortNode);
- assertTrue(
- distributedQueryPlan
- .getFragments()
- .get(1)
- .getPlanNodeTree()
- .getChildren()
- .get(0)
- .getChildren()
- .get(0)
- instanceof ProjectNode);
- assertTrue(
- distributedQueryPlan
- .getFragments()
- .get(1)
- .getPlanNodeTree()
- .getChildren()
- .get(0)
- .getChildren()
- .get(0)
- .getChildren()
- .get(0)
- instanceof FilterNode);
- tableScanNode =
- (TableScanNode)
- distributedQueryPlan
- .getFragments()
- .get(1)
- .getPlanNodeTree()
- .getChildren()
- .get(0)
- .getChildren()
- .get(0)
- .getChildren()
- .get(0)
- .getChildren()
- .get(0);
+ // DistributePlan: `IdentitySink - StreamSort - Project - Filter - TableScan`
+ streamSortNode =
+ (StreamSortNode)
+ distributedQueryPlan.getFragments().get(1).getPlanNodeTree().getChildren().get(0);
+ assertTrue(getChildrenNode(streamSortNode, 1) instanceof ProjectNode);
+ assertTrue(getChildrenNode(streamSortNode, 2) instanceof FilterNode);
+ tableScanNode = (TableScanNode) getChildrenNode(streamSortNode, 3);
assertEquals(2, tableScanNode.getDeviceEntries().size());
assertEquals(
Arrays.asList("table1.shenzhen.B2.ZZ", "table1.shenzhen.B1.XX"),
@@ -1129,70 +932,48 @@ public class SortTest {
.plan(actualAnalysis);
rootNode = logicalQueryPlan.getRootNode();
- // OutputNode - LimitNode - OffsetNode - ProjectNode - SortNode - ProjectNode - FilterNode -
- // TableScanNode
+ context = new MPPQueryContext(sql, queryId, sessionInfo, null, null);
+ actualAnalysis = analyzeSQL(sql, metadata);
+ logicalQueryPlan =
+ new LogicalPlanner(context, metadata, sessionInfo, planOptimizerList, WarningCollector.NOOP)
+ .plan(actualAnalysis);
+ rootNode = logicalQueryPlan.getRootNode();
+
+ // LogicalPlan: `Output-Limit-Offset-Project-StreamSort-Project-Filter-TableScan`
assertTrue(rootNode instanceof OutputNode);
- assertTrue(rootNode.getChildren().get(0) instanceof LimitNode);
- assertTrue(rootNode.getChildren().get(0).getChildren().get(0) instanceof OffsetNode);
- assertTrue(
- rootNode.getChildren().get(0).getChildren().get(0).getChildren().get(0)
- instanceof ProjectNode);
- SortNode sortNode =
- (SortNode)
- rootNode
- .getChildren()
- .get(0)
- .getChildren()
- .get(0)
- .getChildren()
- .get(0)
- .getChildren()
- .get(0);
- assertTrue(sortNode.getChildren().get(0) instanceof ProjectNode);
- assertTrue(sortNode.getChildren().get(0).getChildren().get(0) instanceof FilterNode);
- tableScanNode =
- (TableScanNode) sortNode.getChildren().get(0).getChildren().get(0).getChildren().get(0);
+ assertTrue(getChildrenNode(rootNode, 1) instanceof LimitNode);
+ assertTrue(getChildrenNode(rootNode, 2) instanceof OffsetNode);
+ assertTrue(getChildrenNode(rootNode, 3) instanceof ProjectNode);
+ assertTrue(getChildrenNode(rootNode, 4) instanceof StreamSortNode);
+ streamSortNode = (StreamSortNode) getChildrenNode(rootNode, 4);
+ assertEquals(3, streamSortNode.getStreamCompareKeyEndIndex());
+ assertTrue(getChildrenNode(streamSortNode, 1) instanceof ProjectNode);
+ assertTrue(getChildrenNode(streamSortNode, 2) instanceof FilterNode);
+ assertTrue(getChildrenNode(streamSortNode, 3) instanceof TableScanNode);
+ tableScanNode = (TableScanNode) getChildrenNode(streamSortNode, 3);
assertEquals("testdb.table1", tableScanNode.getQualifiedObjectName().toString());
assertEquals(8, tableScanNode.getAssignments().size());
assertEquals(6, tableScanNode.getDeviceEntries().size());
assertEquals(5, tableScanNode.getIdAndAttributeIndexMap().size());
- // OutputNode - LimitNode - OffsetNode - ProjectNode - MergeSortNode - SortNode - ProjectNode -
- // FilterNode -
- // TableScanNode
+ // DistributePlan: `Output-Limit-Offset-Project-MergeSort-StreamSort-Project-Filter-TableScan`
distributionPlanner = new TableDistributionPlanner(actualAnalysis, logicalQueryPlan, context);
distributedQueryPlan = distributionPlanner.plan();
assertEquals(3, distributedQueryPlan.getFragments().size());
- assertTrue(
- distributedQueryPlan.getFragments().get(0).getPlanNodeTree().getChildren().get(0)
- instanceof OutputNode);
- OutputNode outputNode =
+ outputNode =
(OutputNode)
distributedQueryPlan.getFragments().get(0).getPlanNodeTree().getChildren().get(0);
- assertTrue(outputNode.getChildren().get(0) instanceof LimitNode);
- assertTrue(outputNode.getChildren().get(0).getChildren().get(0) instanceof OffsetNode);
- assertTrue(
- outputNode.getChildren().get(0).getChildren().get(0).getChildren().get(0)
- instanceof ProjectNode);
- MergeSortNode mergeSortNode =
- (MergeSortNode)
- outputNode
- .getChildren()
- .get(0)
- .getChildren()
- .get(0)
- .getChildren()
- .get(0)
- .getChildren()
- .get(0);
+ assertTrue(getChildrenNode(outputNode, 1) instanceof LimitNode);
+ assertTrue(getChildrenNode(outputNode, 2) instanceof OffsetNode);
+ assertTrue(getChildrenNode(outputNode, 3) instanceof ProjectNode);
+ MergeSortNode mergeSortNode = (MergeSortNode) getChildrenNode(outputNode, 4);
assertTrue(mergeSortNode.getChildren().get(0) instanceof ExchangeNode);
- assertTrue(mergeSortNode.getChildren().get(1) instanceof SortNode);
+ assertTrue(mergeSortNode.getChildren().get(1) instanceof StreamSortNode);
assertTrue(mergeSortNode.getChildren().get(2) instanceof ExchangeNode);
- sortNode = (SortNode) mergeSortNode.getChildren().get(1);
- assertTrue(sortNode.getChildren().get(0) instanceof ProjectNode);
- assertTrue(sortNode.getChildren().get(0).getChildren().get(0) instanceof FilterNode);
- tableScanNode =
- (TableScanNode) sortNode.getChildren().get(0).getChildren().get(0).getChildren().get(0);
+ streamSortNode = (StreamSortNode) mergeSortNode.getChildren().get(1);
+ assertTrue(getChildrenNode(streamSortNode, 1) instanceof ProjectNode);
+ assertTrue(getChildrenNode(streamSortNode, 2) instanceof FilterNode);
+ tableScanNode = (TableScanNode) getChildrenNode(streamSortNode, 3);
assertEquals(4, tableScanNode.getDeviceEntries().size());
assertEquals(
Arrays.asList(
@@ -1205,48 +986,13 @@ public class SortTest {
.collect(Collectors.toList()));
assertEquals(ASC, tableScanNode.getScanOrder());
- // IdentitySinkNode - SortNode - ProjectNode - ProjectNode - FilterNode - TableScanNode
- assertTrue(
- distributedQueryPlan.getFragments().get(1).getPlanNodeTree() instanceof IdentitySinkNode);
- assertTrue(
- distributedQueryPlan.getFragments().get(1).getPlanNodeTree().getChildren().get(0)
- instanceof SortNode);
- assertTrue(
- distributedQueryPlan
- .getFragments()
- .get(1)
- .getPlanNodeTree()
- .getChildren()
- .get(0)
- .getChildren()
- .get(0)
- instanceof ProjectNode);
- assertTrue(
- distributedQueryPlan
- .getFragments()
- .get(1)
- .getPlanNodeTree()
- .getChildren()
- .get(0)
- .getChildren()
- .get(0)
- .getChildren()
- .get(0)
- instanceof FilterNode);
- tableScanNode =
- (TableScanNode)
- distributedQueryPlan
- .getFragments()
- .get(1)
- .getPlanNodeTree()
- .getChildren()
- .get(0)
- .getChildren()
- .get(0)
- .getChildren()
- .get(0)
- .getChildren()
- .get(0);
+ // DistributePlan: `IdentitySink - StreamSort - Project - Filter - TableScan`
+ streamSortNode =
+ (StreamSortNode)
+ distributedQueryPlan.getFragments().get(1).getPlanNodeTree().getChildren().get(0);
+ assertTrue(getChildrenNode(streamSortNode, 1) instanceof ProjectNode);
+ assertTrue(getChildrenNode(streamSortNode, 2) instanceof FilterNode);
+ tableScanNode = (TableScanNode) getChildrenNode(streamSortNode, 3);
assertEquals(2, tableScanNode.getDeviceEntries().size());
assertEquals(
Arrays.asList("table1.shenzhen.B2.ZZ", "table1.shenzhen.B1.XX"),
(iotdb) 01/03: Merge branch 'ty/TableModelGrammar' into beyyes/sort_transform_elimate_topk
Posted by ca...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
caogaofei pushed a commit to branch beyyes/sort_transform_elimate_topk
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 22e003e9c3142c16b1a9e4d9bd13d01d547d22a0
Merge: 4982cef78b4 5997327f41f
Author: Beyyes <cg...@foxmail.com>
AuthorDate: Mon Jul 15 11:04:06 2024 +0800
Merge branch 'ty/TableModelGrammar' into beyyes/sort_transform_elimate_topk
.github/workflows/daily-it.yml | 50 +
.github/workflows/multi-language-client.yml | 15 +-
distribution/src/assembly/all.xml | 9 -
distribution/src/assembly/confignode.xml | 9 -
distribution/src/assembly/datanode.xml | 9 -
docker/src/main/Dockerfile-1.0.0-datanode | 4 +-
.../apache/iotdb/SubscriptionSessionExample.java | 40 +-
integration-test/pom.xml | 14 +
.../it/env/cluster/config/MppCommonConfig.java | 6 +
.../env/cluster/config/MppSharedCommonConfig.java | 7 +
.../iotdb/it/env/cluster/env/AbstractEnv.java | 13 +-
.../it/env/remote/config/RemoteCommonConfig.java | 5 +
.../iotdb/it/env/remote/env/RemoteServerEnv.java | 6 +
.../org/apache/iotdb/itbase/category/DailyIT.java | 14 +-
.../java/org/apache/iotdb/itbase/env/BaseEnv.java | 3 +
.../org/apache/iotdb/itbase/env/CommonConfig.java | 2 +
.../apache/iotdb/cli/it/StartClientScriptIT.java | 65 +
.../it/cluster/IoTDBClusterNodeGetterIT.java | 1 -
.../it/partition/IoTDBTimePartitionIT.java | 150 ++
.../IoTDBRegionMigrateReliabilityITFramework.java | 55 +-
.../pass/IoTDBRegionMigrateClusterCrashIT.java | 3 +
.../pass/IoTDBRegionMigrateConfigNodeCrashIT.java | 3 +
.../pass/IoTDBRegionMigrateNormalIT.java | 3 +
...rateCoordinatorCrashWhenRemoveRemotePeerIT.java | 3 +
.../IoTDBRegionMigrateDataNodeCrashIT.java | 6 +
...nMigrateOriginalCrashWhenDeleteLocalPeerIT.java | 3 +
...MigrateOriginalCrashWhenRemoveRemotePeerIT.java | 3 +
.../confignode/it/utils/ConfigNodeTestUtils.java | 1 -
.../apache/iotdb/db/it/IoTDBJDBCMetadataIT.java | 135 ++
.../org/apache/iotdb/db/it/IoTDBRestServiceIT.java | 17 +-
.../iotdb/db/it/IoTDBSetConfigurationIT.java | 90 +-
...oTDBAlignByDeviceWithTemplateAggregationIT.java | 29 +
.../iotdb/db/it/auth/IoTDBSeriesPermissionIT.java | 2 +-
.../apache/iotdb/db/it/query/IoTDBResultSetIT.java | 2 +-
.../pipe/it/autocreate/IoTDBPipeAutoDropIT.java | 6 +
.../pipe/it/autocreate/IoTDBPipeDataSinkIT.java | 100 +-
.../pipe/it/autocreate/IoTDBPipeExtractorIT.java | 2 +-
.../pipe/it/manual/IoTDBPipeMetaHistoricalIT.java | 6 +-
.../relational/it/schema/IoTDBDatabaseIT.java | 2 +-
.../iotdb/relational/it/schema/IoTDBTableIT.java | 61 +-
.../it/session/IoTDBTableModelSessionIT.java | 25 +-
.../session/pool/IoTDBTableModelSessionPoolIT.java | 47 +-
.../it/dual/IoTDBSubscriptionConsumerGroupIT.java | 318 ++--
.../it/dual/IoTDBSubscriptionTopicIT.java | 300 ++--
.../it/local/IoTDBSubscriptionBasicIT.java | 302 +---
.../it/local/IoTDBSubscriptionDataTypeIT.java | 368 ++++
.../java/org/apache/iotdb/cli/AbstractCli.java | 27 +-
.../java/org/apache/iotdb/tool/ImportData.java | 2 +
.../java/org/apache/iotdb/tool/ImportSchema.java | 2 +
.../java/org/apache/iotdb/tool/ImportTsFile.java | 31 +-
.../client-py/iotdb/utils/IoTDBRpcDataSet.py | 2 +-
iotdb-client/client-py/requirements.txt | 4 +-
iotdb-client/client-py/requirements_dev.txt | 8 +-
iotdb-client/client-py/resources/setup.py | 6 +-
iotdb-client/client-py/tests/test_todf.py | 6 +-
.../org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java | 23 +-
.../org/apache/iotdb/jdbc/IoTDBResultMetadata.java | 44 +-
.../apache/iotdb/jdbc/IoTDBJDBCResultSetTest.java | 19 +-
.../java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java | 2 +-
.../iotdb/rpc/subscription/config/TopicConfig.java | 23 +-
.../rpc/subscription/config/TopicConstant.java | 8 +-
.../payload/poll/SubscriptionCommitContext.java | 2 +
.../subscription/payload/poll/TabletsPayload.java | 10 +-
.../response/PipeSubscribeSubscribeResp.java | 32 +-
.../response/PipeSubscribeUnsubscribeResp.java | 32 +-
.../java/org/apache/iotdb/session/Session.java | 29 +-
.../org/apache/iotdb/session/pool/SessionPool.java | 28 +
.../consumer/SubscriptionConsumer.java | 118 +-
.../consumer/SubscriptionProvider.java | 10 +-
.../consumer/SubscriptionPullConsumer.java | 25 +
.../consumer/SubscriptionPushConsumer.java | 29 +-
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 2 +-
.../src/assembly/resources/conf/confignode-env.sh | 6 +-
.../assembly/resources/sbin/start-confignode.sh | 1 +
.../CnToCnInternalServiceAsyncRequestManager.java | 10 +
.../CnToDnInternalServiceAsyncRequestManager.java | 10 +
.../handlers/ConfigNodeAsyncRequestContext.java | 6 +
.../iotdb/confignode/conf/ConfigNodeConfig.java | 12 -
.../confignode/conf/ConfigNodeDescriptor.java | 21 +-
.../confignode/conf/ConfigNodeStartupCheck.java | 8 +-
.../conf/ConfigNodeSystemPropertiesHandler.java | 18 +-
.../confignode/conf/SystemPropertiesUtils.java | 97 +-
.../request/write/database/SetTTLPlan.java | 6 +
.../response/pipe/task/PipeTableResp.java | 67 +-
.../iotdb/confignode/manager/ClusterManager.java | 12 +-
.../iotdb/confignode/manager/ConfigManager.java | 39 +-
.../apache/iotdb/confignode/manager/IManager.java | 11 +
.../iotdb/confignode/manager/TTLManager.java | 16 +-
.../partition/DataPartitionPolicyTable.java | 2 +-
.../manager/node/ClusterNodeStartUtils.java | 12 +-
.../iotdb/confignode/manager/node/NodeManager.java | 15 +-
.../pipe/agent/task/PipeConfigNodeTaskAgent.java | 37 +-
.../protocol/IoTDBConfigRegionConnector.java | 13 +-
.../runtime/heartbeat/PipeHeartbeat.java | 4 +
.../runtime/heartbeat/PipeHeartbeatParser.java | 39 +-
.../pipe/event/PipeConfigRegionSnapshotEvent.java | 9 +-
.../pipe/execution/PipeConfigNodeSubtask.java | 2 +
.../resource/PipeConfigNodeResourceManager.java | 51 +
.../PipeConfigNodeSnapshotResourceManager.java | 11 +-
.../manager/schema/ClusterSchemaManager.java | 64 +-
.../iotdb/confignode/persistence/TTLInfo.java | 24 +-
.../confignode/persistence/pipe/PipeTaskInfo.java | 51 +-
.../persistence/schema/ClusterSchemaInfo.java | 33 +-
.../confignode/persistence/schema/ConfigMTree.java | 66 +-
.../persistence/schema/TemplateTable.java | 10 +-
.../procedure/env/RegionMaintainHandler.java | 17 +-
.../impl/pipe/task/AlterPipeProcedureV2.java | 44 +-
.../impl/schema/AlterLogicalViewProcedure.java | 2 +-
.../impl/schema/DataNodeRegionTaskExecutor.java | 16 +-
.../impl/schema/DeactivateTemplateProcedure.java | 12 +-
.../impl/schema/DeleteLogicalViewProcedure.java | 8 +-
.../impl/schema/DeleteTimeSeriesProcedure.java | 18 +-
.../procedure/impl/schema/SchemaUtils.java | 17 +-
.../impl/schema/SetTemplateProcedure.java | 3 +-
.../impl/schema/table/AddTableColumnProcedure.java | 201 ++-
.../impl/schema/table/CreateTableProcedure.java | 134 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 33 +-
.../request/ConfigPhysicalPlanSerDeTest.java | 76 +
.../iotdb/confignode/persistence/TTLInfoTest.java | 46 +-
.../schema/table/AddTableColumnProcedureTest.java | 62 +
.../schema/table/CreateTableProcedureTest.java | 73 +
.../org/apache/iotdb/consensus/IConsensus.java | 8 +
.../apache/iotdb/consensus/iot/IoTConsensus.java | 6 +
.../iot/logdispatcher/IndexController.java | 4 +-
.../apache/iotdb/consensus/pipe/PipeConsensus.java | 6 +
.../ratis/ApplicationStateMachineProxy.java | 9 +-
.../iotdb/consensus/ratis/RatisConsensus.java | 12 +-
.../apache/iotdb/consensus/ratis/utils/Utils.java | 14 +-
.../iotdb/consensus/simple/SimpleConsensus.java | 5 +
.../apache/iotdb/consensus/iot/StabilityTest.java | 4 +
.../iotdb/consensus/ratis/RatisConsensusTest.java | 7 +
.../consensus/simple/SimpleConsensusTest.java | 6 +-
iotdb-core/datanode/pom.xml | 4 +-
.../src/assembly/resources/conf/datanode-env.sh | 6 +-
.../db/conf/DataNodeSystemPropertiesHandler.java | 16 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 18 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 303 ++--
.../org/apache/iotdb/db/conf/IoTDBStartCheck.java | 17 +-
.../dataregion/DataRegionStateMachine.java | 2 +-
.../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 95 +-
.../IoTDBDataNodeCacheLeaderClientManager.java | 8 +-
.../evolvable/batch/PipeTabletEventBatch.java | 133 ++
.../PipeTabletEventPlainBatch.java} | 121 +-
.../batch/PipeTabletEventTsFileBatch.java | 425 +++++
.../PipeTransferBatchReqBuilder.java | 96 +-
.../request/PipeTransferTabletRawReq.java | 133 +-
.../airgap/IoTDBDataRegionAirGapConnector.java | 3 +-
.../protocol/legacy/IoTDBLegacyPipeConnector.java | 14 +-
.../pipeconsensus/PipeConsensusAsyncConnector.java | 12 +-
.../pipeconsensus/PipeConsensusSyncConnector.java | 2 +
.../PipeConsensusTsFileInsertionEventHandler.java | 2 +
.../PipeConsensusTransferBatchReqBuilder.java | 12 +-
.../request/PipeConsensusTsFileSealReq.java | 9 +-
.../request/PipeConsensusTsFileSealWithModReq.java | 2 +
.../async/IoTDBDataRegionAsyncConnector.java | 238 ++-
.../PipeTransferTabletBatchEventHandler.java | 31 +-
...Handler.java => PipeTransferTsFileHandler.java} | 134 +-
.../thrift/sync/IoTDBDataRegionSyncConnector.java | 129 +-
.../thrift/sync/IoTDBSchemaRegionConnector.java | 13 +-
.../protocol/writeback/WriteBackConnector.java | 3 +-
.../pipe/connector/util/PipeTabletEventSorter.java | 202 +++
.../consensus/PipeConsensusConnectorMetrics.java | 19 +-
.../consensus/ProgressIndexDataNodeManager.java | 20 +-
.../schema/PipeSchemaRegionSnapshotEvent.java | 4 +-
.../tablet/PipeInsertNodeTabletInsertionEvent.java | 24 +-
.../common/tablet/PipeRawTabletInsertionEvent.java | 4 +-
.../tablet/TabletInsertionDataContainer.java | 6 +-
.../common/tsfile/PipeTsFileInsertionEvent.java | 50 +-
.../common/tsfile/TsFileInsertionPointCounter.java | 24 +-
.../container/TsFileInsertionDataContainer.java | 78 +
.../TsFileInsertionDataContainerProvider.java | 122 ++
.../query/TsFileInsertionQueryDataContainer.java} | 121 +-
.../TsFileInsertionQueryDataTabletIterator.java} | 22 +-
.../scan/AlignedSinglePageWholeChunkReader.java | 170 ++
.../container/scan/SinglePageWholeChunkReader.java | 116 ++
.../scan/TsFileInsertionScanDataContainer.java | 386 ++++
.../PipeHistoricalDataRegionTsFileExtractor.java | 71 +-
.../realtime/PipeRealtimeDataRegionExtractor.java | 43 +-
.../PipeRealtimeDataRegionHybridExtractor.java | 6 +-
.../realtime/assigner/DisruptorQueue.java | 12 +-
.../PipeDataNodeRemainingEventAndTimeMetrics.java | 20 +
.../PipeDataNodeRemainingEventAndTimeOperator.java | 29 +-
.../iotdb/db/pipe/metric/PipeProcessorMetrics.java | 37 +-
.../iotdb/db/pipe/metric/PipeResourceMetrics.java | 14 +-
.../downsampling/PartialPathLastObjectCache.java | 4 +-
.../pipeconsensus/PipeConsensusReceiver.java | 154 +-
.../pipeconsensus/PipeConsensusReceiverAgent.java | 53 +-
...nager.java => PipeDataNodeResourceManager.java} | 8 +-
.../db/pipe/resource/memory/PipeMemoryBlock.java | 18 +-
.../db/pipe/resource/memory/PipeMemoryManager.java | 63 +-
...oryWeighUtil.java => PipeMemoryWeightUtil.java} | 63 +-
.../pipe/resource/tsfile/PipeTsFileResource.java | 102 +-
.../resource/tsfile/PipeTsFileResourceManager.java | 10 +-
.../pipe/resource/wal/PipeWALResourceManager.java | 34 +-
.../hardlink/PipeWALHardlinkResourceManager.java | 23 +-
.../selfhost/PipeWALSelfHostResourceManager.java | 8 +-
.../pipe/task/builder/PipeDataNodeTaskBuilder.java | 23 +-
.../pipe/task/connection/PipeEventCollector.java | 29 +-
.../db/pipe/task/stage/PipeTaskProcessorStage.java | 20 +-
.../subtask/connector/PipeConnectorSubtask.java | 62 +-
.../PipeRealtimePriorityBlockingQueue.java | 11 +-
.../subtask/processor/PipeProcessorSubtask.java | 28 +-
.../iotdb/db/protocol/client/ConfigNodeClient.java | 58 +-
.../iotdb/db/protocol/client/ConfigNodeInfo.java | 2 +-
.../DnToCnInternalServiceAsyncRequestManager.java | 10 +
.../impl/DataNodeInternalRPCServiceImpl.java | 49 +-
.../db/queryengine/common/MPPQueryContext.java | 51 +-
.../common/header/ColumnHeaderConstant.java | 10 +-
.../fragment/FragmentInstanceContext.java | 19 +
.../fragment/FragmentInstanceExecution.java | 2 +
.../execution/load/LoadTsFileManager.java | 72 +-
.../operator/schema/source/DeviceSchemaSource.java | 2 +-
.../AbstractSeriesAggregationScanOperator.java | 2 +-
.../operator/source/AlignedSeriesScanOperator.java | 4 +-
.../operator/source/SeriesScanOperator.java | 3 +-
.../execution/operator/source/SeriesScanUtil.java | 3 +-
.../execution/schedule/AbstractDriverThread.java | 12 +-
.../schedule/DriverTaskAbortedException.java | 3 +
.../metric/load/LoadTsFileCostMetricsSet.java | 10 +-
.../iotdb/db/queryengine/plan/Coordinator.java | 2 +-
.../queryengine/plan/analyze/AnalyzeVisitor.java | 4 +-
.../plan/analyze/LoadTsfileAnalyzer.java | 4 +-
.../analyze/cache/partition/PartitionCache.java | 10 +-
.../analyze/cache/schema/DataNodeTTLCache.java | 43 +-
.../cache/schema/TimeSeriesSchemaCache.java | 2 +-
.../cache/schema/dualkeycache/impl/CacheStats.java | 2 -
.../schema/dualkeycache/impl/DualKeyCacheImpl.java | 2 +-
.../schema/lastcache/DataNodeLastCacheManager.java | 3 -
.../execution/config/TableConfigTaskVisitor.java | 30 +-
.../config/executor/ClusterConfigTaskExecutor.java | 82 +-
.../config/metadata/GetTimeSlotListTask.java | 6 +-
.../config/metadata/ShowVariablesTask.java | 10 +-
.../config/metadata/relational/ShowTablesTask.java | 37 +-
.../plan/optimization/AggregationPushDown.java | 28 +-
.../plan/planner/LocalExecutionPlanner.java | 13 +-
.../planner/memory/MemoryReservationManager.java | 46 +
.../NotThreadSafeMemoryReservationManager.java | 94 +
.../ThreadSafeMemoryReservationManager.java} | 33 +-
.../plan/planner/plan/node/PlanGraphPrinter.java | 5 +
.../plan/planner/plan/node/PlanNode.java | 10 -
.../plan/node/pipe/PipeEnrichedInsertNode.java | 5 -
.../planner/plan/node/write/DeleteDataNode.java | 18 +-
.../plan/planner/plan/node/write/InsertNode.java | 44 +-
.../planner/plan/node/write/InsertRowsNode.java | 11 +
.../plan/planner/plan/node/write/SearchNode.java | 51 +
.../plan/relational/planner/LogicalPlanner.java | 9 +-
.../plan/scheduler/load/LoadTsFileScheduler.java | 28 +-
.../statement/metadata/ShowDatabaseStatement.java | 24 +-
.../mtree/impl/pbtree/schemafile/SchemaFile.java | 10 +-
.../db/schemaengine/table/DataNodeTableCache.java | 140 +-
.../iotdb/db/schemaengine/table/ITableCache.java | 22 +-
.../java/org/apache/iotdb/db/service/DataNode.java | 3 +-
.../iotdb/db/service/metrics/WritingMetrics.java | 186 +-
.../iotdb/db/storageengine/StorageEngine.java | 6 -
.../db/storageengine/dataregion/DataRegion.java | 230 ++-
.../fast/reader/CompactionChunkReader.java | 17 +-
.../readchunk/loader/InstantChunkLoader.java | 12 +-
.../readchunk/loader/InstantPageLoader.java | 7 +-
.../compaction/repair/RepairDataFileScanUtil.java | 23 +-
.../compaction/schedule/CompactionScheduler.java | 5 +-
.../compaction/schedule/CompactionTaskManager.java | 8 +
.../estimator/AbstractCompactionEstimator.java | 6 +-
.../FastCompactionInnerCompactionEstimator.java | 15 +-
.../FastCrossSpaceCompactionEstimator.java | 15 +-
.../ReadChunkInnerCompactionEstimator.java | 19 +-
.../RepairUnsortedFileCompactionEstimator.java | 11 +-
.../dataregion/flush/CompressionRatio.java | 7 +
.../dataregion/memtable/AbstractMemTable.java | 32 +-
.../dataregion/memtable/TsFileProcessor.java | 15 +-
.../read/reader/common/PriorityMergeReader.java | 34 +-
.../storageengine/dataregion/wal/WALManager.java | 32 +-
.../dataregion/wal/buffer/WALBuffer.java | 53 +-
.../dataregion/wal/io/CheckpointWriter.java | 2 +-
.../storageengine/dataregion/wal/io/LogWriter.java | 53 +-
.../dataregion/wal/io/WALFileVersion.java | 61 +
.../dataregion/wal/io/WALInputStream.java | 166 +-
.../dataregion/wal/io/WALMetaData.java | 21 +-
.../storageengine/dataregion/wal/io/WALWriter.java | 36 +-
.../storageengine/dataregion/wal/node/WALNode.java | 18 +-
.../dataregion/wal/recover/WALNodeRecoverTask.java | 76 +-
...{WALRecoverWriter.java => WALRepairWriter.java} | 39 +-
.../file/UnsealedTsFileRecoverPerformer.java | 19 +
.../dataregion/wal/utils/WALEntryPosition.java | 1 -
.../dataregion/wal/utils/WALInsertNodeCache.java | 4 +-
.../agent/SubscriptionBrokerAgent.java | 3 +-
.../agent/SubscriptionConsumerAgent.java | 2 +-
.../subscription/agent/SubscriptionTopicAgent.java | 17 +
.../db/subscription/broker/SubscriptionBroker.java | 27 +-
.../broker/SubscriptionPrefetchingQueue.java | 243 ++-
.../broker/SubscriptionPrefetchingTabletQueue.java | 192 ++
.../SubscriptionPrefetchingTabletsQueue.java | 265 ---
.../broker/SubscriptionPrefetchingTsFileQueue.java | 388 ++--
.../db/subscription/event/SubscriptionEvent.java | 295 +++-
.../event/SubscriptionEventBinaryCache.java | 59 +-
.../event/SubscriptionTsFileEvent.java | 247 ---
.../batch/SubscriptionPipeTabletEventBatch.java | 149 ++
.../batch/SubscriptionPipeTsFileEventBatch.java | 61 +
.../event/pipe/SubscriptionPipeEmptyEvent.java} | 25 +-
.../event/pipe/SubscriptionPipeEvents.java} | 20 +-
.../pipe/SubscriptionPipeTabletBatchEvents.java} | 35 +-
.../pipe/SubscriptionPipeTsFileBatchEvents.java | 82 +
.../pipe/SubscriptionPipeTsFilePlainEvent.java | 56 +
.../receiver/SubscriptionReceiverV1.java | 39 +-
.../java/org/apache/iotdb/db/utils/MmapUtil.java | 8 +
.../pipe/connector/PipeTabletEventSorterTest.java | 218 +++
.../event/TsFileInsertionDataContainerTest.java | 232 ++-
.../PipeDataNodeSnapshotResourceManagerTest.java | 20 +-
.../execution/operator/OperatorMemoryTest.java | 19 +-
.../plan/planner/node/PlanGraphPrinterTest.java | 2 +-
.../compaction/AbstractCompactionTest.java | 3 +
...eCompactionWithFastPerformerValidationTest.java | 4 +
...actionWithReadPointPerformerValidationTest.java | 4 +
...eCrossSpaceCompactionWithFastPerformerTest.java | 5 +
.../storageengine/dataregion/wal/WALTestUtils.java | 4 +-
.../wal/compression/WALCompressionTest.java | 39 +-
.../dataregion/wal/node/WALNodeTest.java | 2 +-
...verWriterTest.java => WALRepairWriterTest.java} | 31 +-
.../wal/utils/WALInsertNodeCacheTest.java | 2 +-
.../iotdb/metrics/core/type/IoTDBCounter.java | 4 +-
.../apache/iotdb/metrics/config/MetricConfig.java | 8 +-
iotdb-core/node-commons/pom.xml | 69 +-
.../resources/conf/generate_properties.bat | 65 -
.../assembly/resources/conf/generate_properties.sh | 59 -
.../resources/conf/iotdb-system.properties | 1855 +-------------------
...properties => iotdb-system.properties.template} | 1112 ++++++------
.../AsyncConfigNodeInternalServiceClient.java | 28 +
.../async/AsyncDataNodeInternalServiceClient.java | 28 +
.../client/request/AsyncRequestManager.java | 9 +-
.../{Utils.java => TestConnectionUtils.java} | 37 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 79 +-
.../iotdb/commons/conf/CommonDescriptor.java | 57 +-
.../iotdb/commons/conf/ConfigurationFileUtils.java | 124 +-
.../apache/iotdb/commons/conf/IoTDBConstant.java | 1 -
.../consensus/index/impl/RecoverProgressIndex.java | 5 +-
.../commons/file/SystemPropertiesHandler.java | 62 +-
.../iotdb/commons/pipe/config/PipeConfig.java | 5 +
.../config/constant/PipeConnectorConstant.java | 12 +-
.../config/constant/PipeExtractorConstant.java | 6 +-
.../request/PipeConsensusTransferFileSealReq.java | 11 +
.../PipeConsensusTransferFileSealWithModReq.java | 19 +
.../pipe/connector/protocol/IoTDBConnector.java | 40 +-
.../connector/protocol/IoTDBSslSyncConnector.java | 16 +-
.../iotdb/commons/pipe/event/EnrichedEvent.java | 2 +-
.../commons/pipe/pattern/IoTDBPipePattern.java | 4 +
.../commons}/pipe/resource/log/PipeLogManager.java | 7 +-
.../commons}/pipe/resource/log/PipeLogStatus.java | 6 +-
.../pipe/task/connection/BlockingPendingQueue.java | 22 +-
.../iotdb/commons/pipe/task/meta/PipeTaskMeta.java | 19 +-
.../task/subtask/PipeAbstractConnectorSubtask.java | 56 +-
.../apache/iotdb/commons/schema/table/TsTable.java | 18 +-
.../schema/table/TsTableInternalRPCType.java | 30 +-
.../schema/table/TsTableInternalRPCUtil.java | 12 +-
.../table/column/TsTableColumnSchemaUtil.java | 4 +-
.../apache/iotdb/commons/schema/ttl/TTLCache.java | 41 +-
.../iotdb/commons/service/metric/enums/Metric.java | 11 +-
.../subscription/config/SubscriptionConfig.java | 31 +-
.../iotdb/commons/utils/CommonDateTimeUtils.java | 10 +
.../iotdb/commons/utils/TimePartitionUtils.java | 35 +-
.../commons/utils/TimePartitionUtilsTest.java | 92 +
iotdb-doap.rdf | 29 +-
.../src/main/thrift/confignode.thrift | 48 +-
pom.xml | 14 +-
362 files changed, 10803 insertions(+), 7327 deletions(-)
(iotdb) 02/03: merge
Posted by ca...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
caogaofei pushed a commit to branch beyyes/sort_transform_elimate_topk
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 173d6eeb3e13e70f08bcd869ec7d429405829f9b
Merge: 22e003e9c31 3145d189530
Author: Beyyes <cg...@foxmail.com>
AuthorDate: Tue Jul 16 14:19:22 2024 +0800
merge
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 2 +-
.../confignode/persistence/schema/ConfigMTree.java | 17 +-
.../persistence/schema/ConfigMTreeTest.java | 81 ++
.../plan/planner/plan/node/PlanNode.java | 4 +
.../plan/planner/plan/node/PlanVisitor.java | 5 +
.../querystats/PlanOptimizersStatsCollector.java | 75 ++
.../querystats/QueryPlanOptimizerStatistics.java | 57 ++
.../querystats/QueryPlanOptimizerStats.java | 84 ++
.../plan/relational/planner/Assignments.java | 20 +
.../relational/planner/ExpressionExtractor.java | 117 +++
.../planner/ExpressionSymbolInliner.java | 65 ++
.../plan/relational/planner/LogicalPlanner.java | 79 +-
.../plan/relational/planner/PlanBuilder.java | 4 +-
.../plan/relational/planner/PlanNodeSearcher.java | 226 +++++
.../plan/relational/planner/QueryPlanner.java | 4 +-
.../plan/relational/planner/SymbolsExtractor.java | 200 ++++
.../plan/relational/planner/ir/IrUtils.java | 5 +
.../GroupReference.java} | 72 +-
.../planner/iterative/IterativeOptimizer.java | 365 +++++++
.../plan/relational/planner/iterative/Lookup.java | 65 ++
.../plan/relational/planner/iterative/Memo.java | 225 +++++
.../plan/relational/planner/iterative/Plans.java | 54 ++
.../plan/relational/planner/iterative/Rule.java | 87 ++
.../relational/planner/iterative/RuleIndex.java | 76 ++
.../relational/planner/iterative/RuleStats.java | 53 +
.../planner/iterative/RuleStatsRecorder.java | 45 +
.../planner/iterative/rule/InlineProjections.java | 220 +++++
.../iterative/rule/ProjectOffPushDownRule.java | 67 ++
.../planner/iterative/rule/PruneFilterColumns.java | 46 +
.../planner/iterative/rule/PruneLimitColumns.java | 36 +
.../planner/iterative/rule/PruneOffsetColumns.java | 36 +
.../iterative/rule/PruneOutputSourceColumns.java | 43 +
.../iterative/rule/PruneProjectColumns.java | 39 +
.../planner/iterative/rule/PruneSortColumns.java | 44 +
.../iterative/rule/PruneTableScanColumns.java | 102 ++
.../rule/RemoveRedundantIdentityProjections.java | 48 +
.../relational/planner/iterative/rule/Util.java | 137 +++
.../relational/planner/node/ChildReplacer.java | 32 +
.../plan/relational/planner/node/CollectNode.java | 12 +
.../plan/relational/planner/node/FilterNode.java | 6 +
.../plan/relational/planner/node/LimitNode.java | 11 +
.../relational/planner/node/MergeSortNode.java | 18 +
.../plan/relational/planner/node/OffsetNode.java | 6 +
.../plan/relational/planner/node/OutputNode.java | 6 +
.../plan/relational/planner/node/Patterns.java | 429 +++++++++
.../plan/relational/planner/node/ProjectNode.java | 10 +
.../plan/relational/planner/node/SortNode.java | 6 +
.../relational/planner/node/TableScanNode.java | 12 +
.../plan/relational/planner/node/TopKNode.java | 22 +
.../optimizations/AdaptivePlanOptimizer.java | 56 ++
.../planner/optimizations/PlanOptimizer.java | 77 ++
.../planner/optimizations/PruneUnUsedColumns.java | 151 ---
.../RemoveRedundantIdentityProjections.java | 97 --
.../plan/relational/utils/MoreLists.java | 53 +
.../plan/relational/utils/matching/Capture.java | 43 +
.../plan/relational/utils/matching/Captures.java | 86 ++
.../relational/utils/matching/DefaultPrinter.java | 77 ++
.../plan/relational/utils/matching/Match.java | 66 ++
.../plan/relational/utils/matching/Pattern.java | 120 +++
.../relational/utils/matching/PatternVisitor.java | 46 +
.../plan/relational/utils/matching/Property.java | 89 ++
.../relational/utils/matching/PropertyPattern.java | 51 +
.../utils/matching/pattern/CapturePattern.java | 53 +
.../utils/matching/pattern/EqualsPattern.java | 53 +
.../utils/matching/pattern/FilterPattern.java | 55 ++
.../utils/matching/pattern/TypeOfPattern.java | 59 ++
.../utils/matching/pattern/WithPattern.java | 65 ++
.../plan/relational/analyzer/AnalyzerTest.java | 82 +-
.../analyzer/LimitOffsetPushDownTest.java | 42 +-
.../plan/relational/analyzer/SortTest.java | 1015 +++++++-------------
70 files changed, 4868 insertions(+), 1043 deletions(-)
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/LogicalPlanner.java
index 6bd88486b48,8d34326e066..98d260e1ed9
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/LogicalPlanner.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/LogicalPlanner.java
@@@ -30,13 -31,24 +31,25 @@@ import org.apache.iotdb.db.queryengine.
import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis;
import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Field;
import org.apache.iotdb.db.queryengine.plan.relational.analyzer.RelationType;
+ import org.apache.iotdb.db.queryengine.plan.relational.execution.querystats.PlanOptimizersStatsCollector;
import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata;
+ import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.IterativeOptimizer;
+ import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Rule;
+ import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.RuleStatsRecorder;
+ import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.InlineProjections;
+ import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneFilterColumns;
+ import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneLimitColumns;
+ import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneOffsetColumns;
+ import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneOutputSourceColumns;
+ import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneProjectColumns;
+ import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneSortColumns;
+ import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneTableScanColumns;
+ import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.RemoveRedundantIdentityProjections;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CreateTableDeviceNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OutputNode;
+import org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.AddStreamSort;
- import org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.PruneUnUsedColumns;
+ import org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.PlanOptimizer;
import org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.PushPredicateIntoTableScan;
- import org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.RemoveRedundantIdentityProjections;
import org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.SimplifyExpressions;
import org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.TablePlanOptimizer;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateDevice;
@@@ -79,16 -96,29 +97,30 @@@ public class LogicalPlanner
this.metadata = metadata;
this.sessionInfo = requireNonNull(sessionInfo, "session is null");
this.warningCollector = requireNonNull(warningCollector, "warningCollector is null");
-
+ PlannerContext plannerContext = new PlannerContext(metadata, new InternalTypeManager());
this.tablePlanOptimizers =
- Arrays.asList(new SimplifyExpressions(), new PushPredicateIntoTableScan());
+ Arrays.asList(
- new SimplifyExpressions(),
- new PruneUnUsedColumns(),
- new PushPredicateIntoTableScan(),
- new RemoveRedundantIdentityProjections(),
- new AddStreamSort());
++ new SimplifyExpressions(), new PushPredicateIntoTableScan(), new AddStreamSort());
+
+ Set<Rule<?>> pruneRules =
+ ImmutableSet.of(
+ new PruneFilterColumns(),
+ new PruneLimitColumns(),
+ new PruneOffsetColumns(),
+ new PruneOutputSourceColumns(),
+ new PruneProjectColumns(),
+ new PruneSortColumns(),
+ new PruneTableScanColumns(metadata));
+ Set<Rule<?>> inlineProjections =
+ ImmutableSet.of(
+ new InlineProjections(plannerContext), new RemoveRedundantIdentityProjections());
+ this.planOptimizers =
+ ImmutableList.of(
+ new IterativeOptimizer(plannerContext, new RuleStatsRecorder(), pruneRules),
+ new IterativeOptimizer(plannerContext, new RuleStatsRecorder(), inlineProjections));
}
+ @TestOnly
public LogicalPlanner(
MPPQueryContext context,
Metadata metadata,
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java
index 8456c03a52f,ae7f47f3058..a9c094d7d9a
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java
@@@ -152,6 -152,6 +152,7 @@@ public class QueryPlanner
outputs.stream().map(builder::translate).forEach(newFields::add);
builder = builder.withScope(analysis.getScope(node.getOrderBy().orElse(null)), newFields);
++ analysis.setSortNode(true);
}
List<Expression> orderBy = analysis.getOrderByExpressions(node);
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/LimitNode.java
index f5f4b93a6ff,a0862384d87..8e7b0292b0a
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/LimitNode.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/LimitNode.java
@@@ -46,9 -47,14 +47,14 @@@ public class LimitNode extends SingleCh
this.tiesResolvingScheme = tiesResolvingScheme;
}
+ public boolean requiresPreSortedInputs() {
+ // TODO
+ return false;
+ }
+
@Override
public PlanNode clone() {
- return new LimitNode(id, child, count, tiesResolvingScheme);
+ return new LimitNode(id, null, count, tiesResolvingScheme);
}
@Override
diff --cc iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/LimitOffsetPushDownTest.java
index 3cdaa96bb16,da2368a0ab3..31b33a772b4
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/LimitOffsetPushDownTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/LimitOffsetPushDownTest.java
@@@ -263,7 -257,7 +257,7 @@@ public class LimitOffsetPushDownTest
assertTrue(tableScanNode.getPushDownLimit() == 0 && tableScanNode.getPushDownOffset() == 0);
}
-- private PlanNode getChildrenNode(PlanNode root, int idx) {
++ static PlanNode getChildrenNode(PlanNode root, int idx) {
PlanNode result = root;
for (int i = 1; i <= idx; i++) {
result = result.getChildren().get(0);
diff --cc iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/SortTest.java
index 4fe3ae62137,c36e40d2bb8..3286313881a
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/SortTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/SortTest.java
@@@ -41,12 -40,8 +40,10 @@@ import org.apache.iotdb.db.queryengine.
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OutputNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode;
+import org.apache.iotdb.db.queryengine.plan.relational.planner.node.StreamSortNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode;
+import org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.AddStreamSort;
- import org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.PruneUnUsedColumns;
import org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.PushPredicateIntoTableScan;
- import org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.RemoveRedundantIdentityProjections;
import org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.SimplifyExpressions;
import org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.TablePlanOptimizer;
@@@ -58,6 -53,6 +55,7 @@@ import java.util.List
import java.util.stream.Collectors;
import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.AnalyzerTest.analyzeSQL;
++import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.LimitOffsetPushDownTest.getChildrenNode;
import static org.apache.iotdb.db.queryengine.plan.statement.component.Ordering.ASC;
import static org.apache.iotdb.db.queryengine.plan.statement.component.Ordering.DESC;
import static org.junit.Assert.assertEquals;
@@@ -85,12 -80,7 +83,8 @@@ public class SortTest
DistributedQueryPlan distributedQueryPlan;
TableScanNode tableScanNode;
List<TablePlanOptimizer> planOptimizerList =
- Arrays.asList(new SimplifyExpressions(), new PushPredicateIntoTableScan());
+ Arrays.asList(
- new SimplifyExpressions(),
- new PruneUnUsedColumns(),
- new PushPredicateIntoTableScan(),
- new RemoveRedundantIdentityProjections(),
- new AddStreamSort());
++ new SimplifyExpressions(), new PushPredicateIntoTableScan(), new AddStreamSort());
/*
* order by time, others, some_ids
@@@ -847,27 -700,17 +704,15 @@@
.plan(actualAnalysis);
rootNode = logicalQueryPlan.getRootNode();
- // OutputNode - LimitNode - OffsetNode - SortNode - ProjectNode - ProjectNode - FilterNode -
- // OutputNode - LimitNode - OffsetNode - ProjectNode - SortNode - ProjectNode - FilterNode -
-- // TableScanNode
++ // LogicalPlan: `Output - Limit - Offset - Project - StreamSort - Project - Filter - TableScan`
assertTrue(rootNode instanceof OutputNode);
-- assertTrue(rootNode.getChildren().get(0) instanceof LimitNode);
-- assertTrue(rootNode.getChildren().get(0).getChildren().get(0) instanceof OffsetNode);
-- assertTrue(
-- rootNode.getChildren().get(0).getChildren().get(0).getChildren().get(0)
- instanceof StreamSortNode);
- StreamSortNode sortNode =
- (StreamSortNode) rootNode.getChildren().get(0).getChildren().get(0).getChildren().get(0);
- assertTrue(sortNode.getChildren().get(0) instanceof ProjectNode);
- assertTrue(sortNode.getChildren().get(0).getChildren().get(0) instanceof ProjectNode);
- assertTrue(
- sortNode.getChildren().get(0).getChildren().get(0).getChildren().get(0)
- instanceof FilterNode);
- assertTrue(
- sortNode.getChildren().get(0).getChildren().get(0).getChildren().get(0).getChildren().get(0)
- instanceof TableScanNode);
- tableScanNode =
- (TableScanNode)
- sortNode
- instanceof ProjectNode);
- SortNode sortNode =
- (SortNode)
++ assertTrue(getChildrenNode(rootNode, 1) instanceof LimitNode);
++ assertTrue(getChildrenNode(rootNode, 2) instanceof OffsetNode);
++ assertTrue(getChildrenNode(rootNode, 3) instanceof ProjectNode);
++ assertTrue(getChildrenNode(rootNode, 4) instanceof StreamSortNode);
++ StreamSortNode streamSortNode =
++ (StreamSortNode)
+ rootNode
.getChildren()
.get(0)
.getChildren()
@@@ -876,14 -719,21 +721,21 @@@
.get(0)
.getChildren()
.get(0);
- assertTrue(sortNode.getChildren().get(0) instanceof ProjectNode);
- assertTrue(sortNode.getChildren().get(0).getChildren().get(0) instanceof FilterNode);
++ assertTrue(streamSortNode.getChildren().get(0) instanceof ProjectNode);
++ assertTrue(streamSortNode.getChildren().get(0).getChildren().get(0) instanceof FilterNode);
+ assertTrue(
- sortNode.getChildren().get(0).getChildren().get(0).getChildren().get(0)
++ streamSortNode.getChildren().get(0).getChildren().get(0).getChildren().get(0)
+ instanceof TableScanNode);
+ tableScanNode =
- (TableScanNode) sortNode.getChildren().get(0).getChildren().get(0).getChildren().get(0);
++ (TableScanNode)
++ streamSortNode.getChildren().get(0).getChildren().get(0).getChildren().get(0);
assertEquals("testdb.table1", tableScanNode.getQualifiedObjectName().toString());
- assertEquals(9, tableScanNode.getAssignments().size());
+ assertEquals(8, tableScanNode.getAssignments().size());
assertEquals(6, tableScanNode.getDeviceEntries().size());
assertEquals(5, tableScanNode.getIdAndAttributeIndexMap().size());
- // OutputNode - LimitNode - OffsetNode - MergeSortNode - SortNode - ProjectNode - ProjectNode -
- // OutputNode - LimitNode - OffsetNode - ProjectNode - MergeSortNode - SortNode - ProjectNode -
-- // FilterNode -
-- // TableScanNode
++ // DistributePlan: `Output - Limit - Offset - Project - MergeSort - StreamSort - Project -
++ // Filter - TableScan`
distributionPlanner = new TableDistributionPlanner(actualAnalysis, logicalQueryPlan, context);
distributedQueryPlan = distributionPlanner.plan();
assertEquals(3, distributedQueryPlan.getFragments().size());
@@@ -928,6 -759,14 +761,15 @@@
.get(0)
.getChildren()
.get(0);
+ assertTrue(mergeSortNode.getChildren().get(0) instanceof ExchangeNode);
- assertTrue(mergeSortNode.getChildren().get(1) instanceof SortNode);
++ assertTrue(mergeSortNode.getChildren().get(1) instanceof StreamSortNode);
+ assertTrue(mergeSortNode.getChildren().get(2) instanceof ExchangeNode);
- sortNode = (SortNode) mergeSortNode.getChildren().get(1);
- assertTrue(sortNode.getChildren().get(0) instanceof ProjectNode);
- assertTrue(sortNode.getChildren().get(0).getChildren().get(0) instanceof FilterNode);
++ streamSortNode = (StreamSortNode) mergeSortNode.getChildren().get(1);
++ assertTrue(streamSortNode.getChildren().get(0) instanceof ProjectNode);
++ assertTrue(streamSortNode.getChildren().get(0).getChildren().get(0) instanceof FilterNode);
+ tableScanNode =
- (TableScanNode) sortNode.getChildren().get(0).getChildren().get(0).getChildren().get(0);
++ (TableScanNode)
++ streamSortNode.getChildren().get(0).getChildren().get(0).getChildren().get(0);
assertEquals(4, tableScanNode.getDeviceEntries().size());
assertEquals(
Arrays.asList(
@@@ -940,7 -779,7 +782,7 @@@
.collect(Collectors.toList()));
assertEquals(DESC, tableScanNode.getScanOrder());
- // IdentitySinkNode - SortNode - ProjectNode - ProjectNode - FilterNode - TableScanNode
- // IdentitySinkNode - SortNode - ProjectNode - FilterNode - TableScanNode
++ // DistributePlan: `IdentitySink - StreamSort - Project - Filter - TableScan`
assertTrue(
distributedQueryPlan.getFragments().get(1).getPlanNodeTree() instanceof IdentitySinkNode);
assertTrue(