You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2025/03/03 03:01:12 UTC

(iotdb) branch force_ci/alter_column_datatype updated (5575a3d8b11 -> f677dda1655)

This is an automated email from the ASF dual-hosted git repository.

jiangtian pushed a change to branch force_ci/alter_column_datatype
in repository https://gitbox.apache.org/repos/asf/iotdb.git


    from 5575a3d8b11 fix it
     add 5d38f0c251e Fix the missing deletion when concurrent deletion and insertion compaction (#14955)
     add dbe2d37c389 enhance wal compatibility (#14954)
     add de10ea78911 [AINode] Optimize the error information when dropping built-in model (#14943)
     add fa9711e1146 IoTConsensusV2: Use custom commitIndex to indicate synchronization progress instead of pipe's commitIndex (#14624)
     add 29cc71b599f Add timestamp suffix to vulnerability check report. (#14960)
     add fb2cb7a394b Fix state setting of AbstractOperatePipeProcedureV2 & Enhance state checking of StateMachineProcedure #14959
     add 9e79d85b7a8 peformance regression of sequential inserting (#14951)
     add 95972550d8c Introduce flushing on specific regions & IoTConsensusV2: Flush old leader when leader transfer & Delay the execution of invalidateSchemaCache after leader change (#14910)
     add f6c3c5da617 Adapt to Mybatis generator (#14936)
     add 1e6513b1fc5 IoTConsensusV2: Multi-Dirs balance with sequence rolling strategy in receiver (#14962)
     add e1840d0ea7a [IoTConsensus] More accurate statistics on IoTConsensus memory management (#14965)
     add 56080ee2f58 Revert "Allow to write time column only (#14703)" (#14967)
     add b638a70a4cc Pipe: Reduce the frequency of printing logs when AirGapConnector connection fails (#14949)
     add a94ae439dde Fix the support of MQTT with java8 (#14968)
     add f1a224e56c1 Subscription: unify tablet push consumer builder return type (#14971)
     add ede623a27d8 Pipe: Implemented OPC DA Sink for local COM & Fixed the newest value of OPC UA Sink (#14964)
     add 2a77b7ab162 Pipe: Optimize the TsFile memory control mechanism of the disassembly table model (#14890)
     add aad2caa9f22 Fix flush old leader NPE when rebanlce #14981
     add 3384bd69a92 Subscription: add parameter to the clean up method to distinguish the force close situation & fix reference count management for iteration snapshot (#14966)
     add a1b9f7938af [Table Model Subquery] Support Exists subquery
     add 9ffef99a2ea Pipe: Make client-server the default setting of opc-ua-sink (#14979)
     add c1723158ebc [To Master] Adds extensibility to MQTT's table (#14976)
     add 662de3115c5 Pipe: Fixed the NPE caused by creating a new Region during the execution of Alter Pipe Procedure. (#14982)
     add b3e57d29409 Shuffle nodeurls to make sure all connections aren't to the first node by default #14983
     add ee7e8f9d349 [RTO/RPO] Add Phi Accrual for Node failure detection (#14866)
     add 21ad376a2a3 Add -lpf param in ExportData (#14957)
     add 5e1bd035dc6 Improved the calculation accuracy of table schema fetch + partial query & Fixed the bug that the attributes' memory is not calculated when schema region recovers (#14975)
     add 9c66a3cdc83 Fix group by hash when process TsBlock more than 1024 lines (#14989)
     add 82e9361c6a6 Implemented max ttl under specific database getter for table model (#14977)
     add 174b2cb968f IoTConsensusV2: Transfer table deletion without any parse or filter (#14988)
     new f677dda1655 Merge branch 'master' into force_ci/alter_column_datatype

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .github/workflows/vulnerability-check.yml          |   7 +-
 dependencies.json                                  |   1 +
 .../server/CustomizedJsonPayloadFormatter.java     |   8 +-
 .../org/apache/iotdb/mqtt/server/MyMessage.java    |  80 ---
 .../java/org/apache/iotdb/mqtt/MQTTClient.java     |   2 +-
 .../org/apache/iotdb/ainode/it/AINodeBasicIT.java  |   2 +
 .../relational/it/db/it/IoTDBInsertTableIT.java    |  11 +-
 .../db/it/IoTDBMultiTAGsWithAttributesTableIT.java |  12 +-
 .../relational/it/mqtt/IoTDBMQTTServiceIT.java     |   2 +-
 .../IoTDBCorrelatedExistsSubqueryIT.java           | 428 +++++++++++++
 .../IoTDBUncorrelatedExistsSubqueryIT.java         | 391 ++++++++++++
 .../it/session/IoTDBSessionRelationalIT.java       |  34 -
 .../iotdb/session/it/IoTDBSessionSimpleIT.java     |  48 --
 .../org/apache/iotdb/tool/common/OptionsUtil.java  |  54 +-
 .../org/apache/iotdb/tool/data/ExportData.java     |   4 +-
 .../org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java  |  25 +-
 .../apache/iotdb/jdbc/IoTDBPreparedStatement.java  |  31 +-
 .../IoTDBRelationalDatabaseMetadata.java           |  52 +-
 .../java/org/apache/iotdb/session/Session.java     |  17 +-
 .../SubscriptionTablePushConsumerBuilder.java      |   3 +-
 iotdb-core/confignode/pom.xml                      |   4 +
 .../iotdb/confignode/conf/ConfigNodeConfig.java    |  45 ++
 .../confignode/conf/ConfigNodeDescriptor.java      |  30 +
 .../iotdb/confignode/manager/ConfigManager.java    |   9 +
 .../apache/iotdb/confignode/manager/IManager.java  |   3 +
 .../iotdb/confignode/manager/ModelManager.java     |   5 +
 .../manager/load/balancer/RouteBalancer.java       | 154 +++--
 .../manager/load/cache/AbstractLoadCache.java      |  26 +-
 .../manager/load/cache/IFailureDetector.java}      |  24 +-
 .../manager/load/cache/detector/FixedDetector.java |  58 ++
 .../load/cache/detector/PhiAccrualDetector.java    | 181 ++++++
 .../load/cache/node/AINodeHeartbeatCache.java      |  15 +-
 .../load/cache/node/ConfigNodeHeartbeatCache.java  |  12 +-
 .../load/cache/node/DataNodeHeartbeatCache.java    |  11 +-
 .../manager/load/cache/region/RegionCache.java     |  12 +-
 .../iotdb/confignode/manager/node/NodeManager.java |   8 +
 .../manager/schema/ClusterSchemaManager.java       |  12 +-
 .../iotdb/confignode/persistence/ModelInfo.java    |   2 +-
 .../iotdb/confignode/persistence/TTLInfo.java      |   2 +-
 .../persistence/schema/ClusterSchemaInfo.java      |  20 +
 .../procedure/PartitionTableAutoCleaner.java       |  14 +-
 .../procedure/impl/StateMachineProcedure.java      |   9 +-
 .../impl/pipe/AbstractOperatePipeProcedureV2.java  |   2 +
 .../impl/pipe/task/AlterPipeProcedureV2.java       |   1 +
 .../manager/load/cache/detector/DetectorTest.java  | 179 ++++++
 .../common/request/IConsensusRequest.java          |   5 +
 .../common/request/IndexedConsensusRequest.java    |   8 +-
 .../consensus/config/PipeConsensusConfig.java      |  21 +-
 .../consensus/iot/logdispatcher/LogDispatcher.java |  12 +-
 .../consensus/pipe/PipeConsensusServerImpl.java    |  21 +-
 ...xManager.java => ReplicateProgressManager.java} |   8 +-
 .../pipe/metric/PipeConsensusSyncLagManager.java   |  71 ++-
 iotdb-core/datanode/pom.xml                        |   8 +
 .../java/org/apache/iotdb/db/conf/IoTDBConfig.java |   2 +-
 .../db/consensus/DataRegionConsensusImpl.java      |   4 +-
 .../PipeDataRegionConnectorConstructor.java        |   4 +
 .../agent/task/connection/PipeEventCollector.java  |  13 +-
 .../agent/task/stage/PipeTaskProcessorStage.java   |   5 +-
 .../connector/protocol/opcda/OpcDaConnector.java   | 138 ++++
 .../pipe/connector/protocol/opcda/OpcDaHeader.java | 200 ++++++
 .../protocol/opcda/OpcDaServerHandle.java          | 390 ++++++++++++
 .../connector/protocol/opcua/OpcUaConnector.java   |  50 +-
 .../connector/protocol/opcua/OpcUaNameSpace.java   |   2 +-
 .../pipeconsensus/PipeConsensusAsyncConnector.java |  16 +-
 .../pipeconsensus/PipeConsensusSyncConnector.java  |   9 +-
 .../PipeConsensusTabletInsertionEventHandler.java  |   8 +-
 .../PipeConsensusTsFileInsertionEventHandler.java  |   8 +-
 .../PipeConsensusTransferBatchReqBuilder.java      |   5 +-
 ....java => ReplicateProgressDataNodeManager.java} |  36 +-
 .../deletion/DeletionResourceManager.java          |   4 +-
 .../deletion/persist/PageCacheDeletionBuffer.java  |   5 +-
 .../common/tsfile/PipeTsFileInsertionEvent.java    |  17 +-
 .../scan/TsFileInsertionEventScanParser.java       |   3 -
 .../table/TsFileInsertionEventTableParser.java     |  95 ++-
 ...ileInsertionEventTableParserTabletIterator.java | 421 ++++++++++---
 .../event/realtime/PipeRealtimeEventFactory.java   |  71 ++-
 ...oricalDataRegionTsFileAndDeletionExtractor.java |  17 +
 .../realtime/assigner/PipeDataRegionAssigner.java  |   2 +
 .../listener/PipeInsertionDataNodeListener.java    |  16 +-
 .../pipeconsensus/PipeConsensusProcessor.java      |  43 +-
 .../pipeconsensus/PipeConsensusReceiver.java       | 338 +++++-----
 .../resource/memory/InsertNodeMemoryEstimator.java |   2 +-
 .../pipe/resource/memory/PipeMemoryWeightUtil.java |  65 ++
 .../db/protocol/mqtt/JSONPayloadFormatter.java     |   7 +-
 .../db/protocol/mqtt/LinePayloadFormatter.java     |   7 +-
 .../iotdb/db/protocol/mqtt/MPPPublishHandler.java  |  11 +-
 .../iotdb/db/protocol/mqtt/PayloadFormatter.java   |   6 +
 .../protocol/thrift/impl/ClientRPCServiceImpl.java |   2 +-
 ...owOperator.java => AssignUniqueIdOperator.java} |  75 ++-
 .../operator/schema/SchemaQueryScanOperator.java   |  12 +-
 .../operator/schema/source/ISchemaSource.java      |   5 +
 .../schema/source/TableDeviceFetchSource.java      |  21 +
 .../schema/source/TableDeviceQuerySource.java      |  36 +-
 ...perator.java => MergeSortLeftJoinOperator.java} |  57 +-
 .../aggregation/grouped/hash/HashStrategy.java     |   2 +-
 .../plan/planner/TableOperatorGenerator.java       |  53 ++
 .../plan/planner/plan/node/PlanGraphPrinter.java   |  10 +
 .../plan/planner/plan/node/PlanNodeType.java       |   4 +
 .../plan/planner/plan/node/PlanVisitor.java        |   5 +
 .../plan/planner/plan/node/write/InsertNode.java   |   3 +-
 .../planner/plan/node/write/InsertRowNode.java     |   3 -
 .../PredicatePushIntoMetadataChecker.java          |   7 +-
 .../fetcher/TableHeaderSchemaValidator.java        |   4 +
 .../distribute/TableDistributedPlanGenerator.java  |  58 +-
 .../iterative/rule/AggregationDecorrelation.java   |  83 +++
 ...olumns.java => PruneAssignUniqueIdColumns.java} |  20 +-
 .../iterative/rule/RemoveRedundantExists.java      | 104 +++
 ...orrelatedDistinctAggregationWithProjection.java | 186 ++++++
 ...elatedDistinctAggregationWithoutProjection.java | 167 +++++
 ...mCorrelatedGlobalAggregationWithProjection.java | 318 ++++++++++
 ...rrelatedGlobalAggregationWithoutProjection.java | 302 +++++++++
 ...CorrelatedGroupedAggregationWithProjection.java | 252 ++++++++
 ...relatedGroupedAggregationWithoutProjection.java | 233 +++++++
 .../rule/TransformCorrelatedJoinToJoin.java        |  99 +++
 .../rule/TransformExistsApplyToCorrelatedJoin.java | 214 +++++++
 ...forceSingleRowNode.java => AssignUniqueId.java} |  55 +-
 .../plan/relational/planner/node/Patterns.java     |  27 +-
 .../planner/optimizations/JoinUtils.java           | 249 +++++---
 .../optimizations/LogicalOptimizeFactory.java      |  40 +-
 .../optimizations/PlanNodeDecorrelator.java        | 699 +++++++++++++++++++++
 .../optimizations/PushPredicateIntoTableScan.java  | 143 ++++-
 .../optimizations/UnaliasSymbolReferences.java     |  13 +
 .../relational/planner/optimizations/Util.java     |  19 +
 .../plan/relational/sql/ast/InsertRows.java        |   3 -
 .../plan/statement/crud/InsertBaseStatement.java   |  28 +-
 .../plan/statement/crud/InsertRowStatement.java    |   3 -
 .../rescon/ISchemaRegionStatistics.java            |   2 +
 .../rescon/MemSchemaRegionStatistics.java          |  16 +
 .../attribute/DeviceAttributeStore.java            |  48 +-
 .../attribute/IDeviceAttributeStore.java           |  16 +-
 .../schemaregion/impl/SchemaRegionMemoryImpl.java  |  36 +-
 .../iotdb/db/storageengine/StorageEngine.java      |  19 +-
 .../dataregion/memtable/AbstractMemTable.java      |  85 ++-
 .../memtable/AlignedWritableMemChunk.java          |  13 +
 .../memtable/AlignedWritableMemChunkGroup.java     |  15 +-
 .../dataregion/memtable/IMemTable.java             |   2 +
 .../memtable/IWritableMemChunkGroup.java           |   2 +
 .../dataregion/memtable/TsFileProcessor.java       |  41 +-
 .../dataregion/memtable/WritableMemChunk.java      |   8 +
 .../dataregion/memtable/WritableMemChunkGroup.java |  20 +
 .../dataregion/tsfile/TsFileResource.java          |   8 +-
 .../broker/SubscriptionPrefetchingQueue.java       |  10 +-
 .../broker/SubscriptionPrefetchingTabletQueue.java |   2 +-
 .../broker/SubscriptionPrefetchingTsFileQueue.java |   2 +-
 .../db/subscription/event/SubscriptionEvent.java   |   4 +-
 .../event/batch/SubscriptionPipeEventBatch.java    |   2 +-
 .../event/batch/SubscriptionPipeEventBatches.java  |   2 +-
 .../batch/SubscriptionPipeTabletEventBatch.java    |   6 +-
 .../SubscriptionPipeTabletIterationSnapshot.java   |  18 +-
 .../batch/SubscriptionPipeTsFileEventBatch.java    |   2 +-
 .../event/pipe/SubscriptionPipeEmptyEvent.java     |   2 +-
 .../event/pipe/SubscriptionPipeEvents.java         |   2 +-
 .../pipe/SubscriptionPipeTabletBatchEvents.java    |  11 +-
 .../pipe/SubscriptionPipeTsFileBatchEvents.java    |   4 +-
 .../pipe/SubscriptionPipeTsFilePlainEvent.java     |   2 +-
 .../db/tools/schema/SRStatementGenerator.java      |   2 +
 .../db/utils/datastructure/AlignedTVList.java      |  44 +-
 .../iotdb/db/utils/datastructure/BinaryTVList.java |  38 +-
 .../db/utils/datastructure/BooleanTVList.java      |  38 +-
 .../iotdb/db/utils/datastructure/DoubleTVList.java |  38 +-
 .../iotdb/db/utils/datastructure/FloatTVList.java  |  38 +-
 .../iotdb/db/utils/datastructure/IntTVList.java    |  38 +-
 .../iotdb/db/utils/datastructure/LongTVList.java   |  38 +-
 .../iotdb/db/utils/datastructure/TVList.java       |  70 ++-
 .../db/protocol/mqtt/PayloadFormatManagerTest.java |   2 +-
 .../execution/operator/OperatorMemoryTest.java     |   2 -
 .../relational/analyzer/GroupByLargeDataTest.java  | 179 ++++++
 .../relational/planner/CorrelatedSubqueryTest.java | 153 +++++
 ...ueryTest.java => UncorrelatedSubqueryTest.java} | 104 ++-
 ...liasPresent.java => AssignUniqueIdMatcher.java} |  24 +-
 .../planner/assertions/PlanMatchPattern.java       |   6 +
 .../dataregion/memtable/TsFileProcessorTest.java   |  36 +-
 .../conf/iotdb-system.properties.template          |  25 +-
 .../iotdb/commons/path/ExtendedPartialPath.java    |   4 +
 .../agent/plugin/builtin/BuiltinPipePlugin.java    |   5 +
 .../OpcDaConnector.java}                           |  10 +-
 .../config/constant/PipeConnectorConstant.java     |   8 +-
 .../connector/protocol/IoTDBAirGapConnector.java   |  21 +-
 .../iotdb/commons/pipe/event/EnrichedEvent.java    |  17 +-
 .../apache/iotdb/commons/schema/table/TsTable.java |   2 +-
 .../thrift-commons/src/main/thrift/common.thrift   |   1 +
 .../src/main/thrift/pipeconsensus.thrift           |   5 +-
 pom.xml                                            |   8 +-
 183 files changed, 7946 insertions(+), 1248 deletions(-)
 delete mode 100644 example/mqtt-customize/src/main/java/org/apache/iotdb/mqtt/server/MyMessage.java
 create mode 100644 integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedExistsSubqueryIT.java
 create mode 100644 integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedExistsSubqueryIT.java
 copy iotdb-core/{datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/IFragInstanceDispatcher.java => confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/IFailureDetector.java} (60%)
 create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/detector/FixedDetector.java
 create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/detector/PhiAccrualDetector.java
 create mode 100644 iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/detector/DetectorTest.java
 rename iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/consensuspipe/{ProgressIndexManager.java => ReplicateProgressManager.java} (81%)
 create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/opcda/OpcDaConnector.java
 create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/opcda/OpcDaHeader.java
 create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/opcda/OpcDaServerHandle.java
 rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/{ProgressIndexDataNodeManager.java => ReplicateProgressDataNodeManager.java} (78%)
 copy iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/{EnforceSingleRowOperator.java => AssignUniqueIdOperator.java} (55%)
 copy iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/{MergeSortInnerJoinOperator.java => MergeSortLeftJoinOperator.java} (80%)
 create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/AggregationDecorrelation.java
 copy iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/{PruneFillColumns.java => PruneAssignUniqueIdColumns.java} (71%)
 create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/RemoveRedundantExists.java
 create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformCorrelatedDistinctAggregationWithProjection.java
 create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformCorrelatedDistinctAggregationWithoutProjection.java
 create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformCorrelatedGlobalAggregationWithProjection.java
 create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformCorrelatedGlobalAggregationWithoutProjection.java
 create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformCorrelatedGroupedAggregationWithProjection.java
 create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformCorrelatedGroupedAggregationWithoutProjection.java
 create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformCorrelatedJoinToJoin.java
 create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformExistsApplyToCorrelatedJoin.java
 copy iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/{EnforceSingleRowNode.java => AssignUniqueId.java} (62%)
 create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PlanNodeDecorrelator.java
 create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/GroupByLargeDataTest.java
 create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CorrelatedSubqueryTest.java
 rename iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/{SubqueryTest.java => UncorrelatedSubqueryTest.java} (85%)
 copy iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/{AliasPresent.java => AssignUniqueIdMatcher.java} (72%)
 copy iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/connector/{opcua/OpcUaConnector.java => opcda/OpcDaConnector.java} (86%)


(iotdb) 01/01: Merge branch 'master' into force_ci/alter_column_datatype

Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jiangtian pushed a commit to branch force_ci/alter_column_datatype
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit f677dda16551c9a0e19b9b0acc7036158ef41274
Merge: 5575a3d8b11 174b2cb968f
Author: Tian Jiang <jt...@163.com>
AuthorDate: Mon Mar 3 11:01:01 2025 +0800

    Merge branch 'master' into force_ci/alter_column_datatype
    
    # Conflicts:
    #       iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java
    #       iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java
    #       iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java
    #       iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java
    #       iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java

 .github/workflows/vulnerability-check.yml          |   7 +-
 dependencies.json                                  |   1 +
 .../server/CustomizedJsonPayloadFormatter.java     |   8 +-
 .../org/apache/iotdb/mqtt/server/MyMessage.java    |  80 ---
 .../java/org/apache/iotdb/mqtt/MQTTClient.java     |   2 +-
 .../org/apache/iotdb/ainode/it/AINodeBasicIT.java  |   2 +
 .../relational/it/db/it/IoTDBInsertTableIT.java    |  11 +-
 .../db/it/IoTDBMultiTAGsWithAttributesTableIT.java |  12 +-
 .../relational/it/mqtt/IoTDBMQTTServiceIT.java     |   2 +-
 .../IoTDBCorrelatedExistsSubqueryIT.java           | 428 +++++++++++++
 .../IoTDBUncorrelatedExistsSubqueryIT.java         | 391 ++++++++++++
 .../it/session/IoTDBSessionRelationalIT.java       |  34 -
 .../iotdb/session/it/IoTDBSessionSimpleIT.java     |  48 --
 .../org/apache/iotdb/tool/common/OptionsUtil.java  |  54 +-
 .../org/apache/iotdb/tool/data/ExportData.java     |   4 +-
 .../org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java  |  25 +-
 .../apache/iotdb/jdbc/IoTDBPreparedStatement.java  |  31 +-
 .../IoTDBRelationalDatabaseMetadata.java           |  52 +-
 .../java/org/apache/iotdb/session/Session.java     |  17 +-
 .../SubscriptionTablePushConsumerBuilder.java      |   3 +-
 iotdb-core/confignode/pom.xml                      |   4 +
 .../iotdb/confignode/conf/ConfigNodeConfig.java    |  45 ++
 .../confignode/conf/ConfigNodeDescriptor.java      |  30 +
 .../iotdb/confignode/manager/ConfigManager.java    |   9 +
 .../apache/iotdb/confignode/manager/IManager.java  |   3 +
 .../iotdb/confignode/manager/ModelManager.java     |   5 +
 .../manager/load/balancer/RouteBalancer.java       | 154 +++--
 .../manager/load/cache/AbstractLoadCache.java      |  26 +-
 .../manager/load/cache/IFailureDetector.java}      |  24 +-
 .../manager/load/cache/detector/FixedDetector.java |  58 ++
 .../load/cache/detector/PhiAccrualDetector.java    | 181 ++++++
 .../load/cache/node/AINodeHeartbeatCache.java      |  15 +-
 .../load/cache/node/ConfigNodeHeartbeatCache.java  |  12 +-
 .../load/cache/node/DataNodeHeartbeatCache.java    |  11 +-
 .../manager/load/cache/region/RegionCache.java     |  12 +-
 .../iotdb/confignode/manager/node/NodeManager.java |   8 +
 .../manager/schema/ClusterSchemaManager.java       |  12 +-
 .../iotdb/confignode/persistence/ModelInfo.java    |   2 +-
 .../iotdb/confignode/persistence/TTLInfo.java      |   2 +-
 .../persistence/schema/ClusterSchemaInfo.java      |  20 +
 .../procedure/PartitionTableAutoCleaner.java       |  14 +-
 .../procedure/impl/StateMachineProcedure.java      |   9 +-
 .../impl/pipe/AbstractOperatePipeProcedureV2.java  |   2 +
 .../impl/pipe/task/AlterPipeProcedureV2.java       |   1 +
 .../manager/load/cache/detector/DetectorTest.java  | 179 ++++++
 .../common/request/IConsensusRequest.java          |   5 +
 .../common/request/IndexedConsensusRequest.java    |   8 +-
 .../consensus/config/PipeConsensusConfig.java      |  21 +-
 .../consensus/iot/logdispatcher/LogDispatcher.java |  12 +-
 .../consensus/pipe/PipeConsensusServerImpl.java    |  21 +-
 ...xManager.java => ReplicateProgressManager.java} |   8 +-
 .../pipe/metric/PipeConsensusSyncLagManager.java   |  71 ++-
 iotdb-core/datanode/pom.xml                        |   8 +
 .../java/org/apache/iotdb/db/conf/IoTDBConfig.java |   2 +-
 .../db/consensus/DataRegionConsensusImpl.java      |   4 +-
 .../PipeDataRegionConnectorConstructor.java        |   4 +
 .../agent/task/connection/PipeEventCollector.java  |  13 +-
 .../agent/task/stage/PipeTaskProcessorStage.java   |   5 +-
 .../connector/protocol/opcda/OpcDaConnector.java   | 138 ++++
 .../pipe/connector/protocol/opcda/OpcDaHeader.java | 200 ++++++
 .../protocol/opcda/OpcDaServerHandle.java          | 390 ++++++++++++
 .../connector/protocol/opcua/OpcUaConnector.java   |  50 +-
 .../connector/protocol/opcua/OpcUaNameSpace.java   |   2 +-
 .../pipeconsensus/PipeConsensusAsyncConnector.java |  16 +-
 .../pipeconsensus/PipeConsensusSyncConnector.java  |   9 +-
 .../PipeConsensusTabletInsertionEventHandler.java  |   8 +-
 .../PipeConsensusTsFileInsertionEventHandler.java  |   8 +-
 .../PipeConsensusTransferBatchReqBuilder.java      |   5 +-
 ....java => ReplicateProgressDataNodeManager.java} |  36 +-
 .../deletion/DeletionResourceManager.java          |   4 +-
 .../deletion/persist/PageCacheDeletionBuffer.java  |   5 +-
 .../common/tsfile/PipeTsFileInsertionEvent.java    |  17 +-
 .../scan/TsFileInsertionEventScanParser.java       |   3 -
 .../table/TsFileInsertionEventTableParser.java     |  95 ++-
 ...ileInsertionEventTableParserTabletIterator.java | 421 ++++++++++---
 .../event/realtime/PipeRealtimeEventFactory.java   |  71 ++-
 ...oricalDataRegionTsFileAndDeletionExtractor.java |  17 +
 .../realtime/assigner/PipeDataRegionAssigner.java  |   2 +
 .../listener/PipeInsertionDataNodeListener.java    |  16 +-
 .../pipeconsensus/PipeConsensusProcessor.java      |  43 +-
 .../pipeconsensus/PipeConsensusReceiver.java       | 338 +++++-----
 .../resource/memory/InsertNodeMemoryEstimator.java |   2 +-
 .../pipe/resource/memory/PipeMemoryWeightUtil.java |  65 ++
 .../db/protocol/mqtt/JSONPayloadFormatter.java     |   7 +-
 .../db/protocol/mqtt/LinePayloadFormatter.java     |   7 +-
 .../iotdb/db/protocol/mqtt/MPPPublishHandler.java  |  11 +-
 .../iotdb/db/protocol/mqtt/PayloadFormatter.java   |   6 +
 .../protocol/thrift/impl/ClientRPCServiceImpl.java |   2 +-
 .../operator/process/AssignUniqueIdOperator.java   | 142 +++++
 .../operator/schema/SchemaQueryScanOperator.java   |  12 +-
 .../operator/schema/source/ISchemaSource.java      |   5 +
 .../schema/source/TableDeviceFetchSource.java      |  21 +
 .../schema/source/TableDeviceQuerySource.java      |  36 +-
 .../relational/MergeSortLeftJoinOperator.java      | 163 +++++
 .../aggregation/grouped/hash/HashStrategy.java     |   2 +-
 .../plan/planner/TableOperatorGenerator.java       |  53 ++
 .../plan/planner/plan/node/PlanGraphPrinter.java   |  10 +
 .../plan/planner/plan/node/PlanNodeType.java       |   4 +
 .../plan/planner/plan/node/PlanVisitor.java        |   5 +
 .../plan/planner/plan/node/write/InsertNode.java   |   3 +-
 .../planner/plan/node/write/InsertRowNode.java     |   3 -
 .../PredicatePushIntoMetadataChecker.java          |   7 +-
 .../fetcher/TableHeaderSchemaValidator.java        |   4 +
 .../distribute/TableDistributedPlanGenerator.java  |  58 +-
 .../iterative/rule/AggregationDecorrelation.java   |  83 +++
 .../iterative/rule/PruneAssignUniqueIdColumns.java |  47 ++
 .../iterative/rule/RemoveRedundantExists.java      | 104 +++
 ...orrelatedDistinctAggregationWithProjection.java | 186 ++++++
 ...elatedDistinctAggregationWithoutProjection.java | 167 +++++
 ...mCorrelatedGlobalAggregationWithProjection.java | 318 ++++++++++
 ...rrelatedGlobalAggregationWithoutProjection.java | 302 +++++++++
 ...CorrelatedGroupedAggregationWithProjection.java | 252 ++++++++
 ...relatedGroupedAggregationWithoutProjection.java | 233 +++++++
 .../rule/TransformCorrelatedJoinToJoin.java        |  99 +++
 .../rule/TransformExistsApplyToCorrelatedJoin.java | 214 +++++++
 .../relational/planner/node/AssignUniqueId.java    |  97 +++
 .../plan/relational/planner/node/Patterns.java     |  27 +-
 .../planner/optimizations/JoinUtils.java           | 249 +++++---
 .../optimizations/LogicalOptimizeFactory.java      |  40 +-
 .../optimizations/PlanNodeDecorrelator.java        | 699 +++++++++++++++++++++
 .../optimizations/PushPredicateIntoTableScan.java  | 143 ++++-
 .../optimizations/UnaliasSymbolReferences.java     |  13 +
 .../relational/planner/optimizations/Util.java     |  19 +
 .../plan/relational/sql/ast/InsertRows.java        |   3 -
 .../plan/statement/crud/InsertBaseStatement.java   |  28 +-
 .../plan/statement/crud/InsertRowStatement.java    |   3 -
 .../rescon/ISchemaRegionStatistics.java            |   2 +
 .../rescon/MemSchemaRegionStatistics.java          |  16 +
 .../attribute/DeviceAttributeStore.java            |  48 +-
 .../attribute/IDeviceAttributeStore.java           |  16 +-
 .../schemaregion/impl/SchemaRegionMemoryImpl.java  |  36 +-
 .../iotdb/db/storageengine/StorageEngine.java      |  19 +-
 .../dataregion/memtable/AbstractMemTable.java      |  85 ++-
 .../memtable/AlignedWritableMemChunk.java          |  13 +
 .../memtable/AlignedWritableMemChunkGroup.java     |  15 +-
 .../dataregion/memtable/IMemTable.java             |   2 +
 .../memtable/IWritableMemChunkGroup.java           |   2 +
 .../dataregion/memtable/TsFileProcessor.java       |  41 +-
 .../dataregion/memtable/WritableMemChunk.java      |   8 +
 .../dataregion/memtable/WritableMemChunkGroup.java |  20 +
 .../dataregion/tsfile/TsFileResource.java          |   8 +-
 .../broker/SubscriptionPrefetchingQueue.java       |  10 +-
 .../broker/SubscriptionPrefetchingTabletQueue.java |   2 +-
 .../broker/SubscriptionPrefetchingTsFileQueue.java |   2 +-
 .../db/subscription/event/SubscriptionEvent.java   |   4 +-
 .../event/batch/SubscriptionPipeEventBatch.java    |   2 +-
 .../event/batch/SubscriptionPipeEventBatches.java  |   2 +-
 .../batch/SubscriptionPipeTabletEventBatch.java    |   6 +-
 .../SubscriptionPipeTabletIterationSnapshot.java   |  18 +-
 .../batch/SubscriptionPipeTsFileEventBatch.java    |   2 +-
 .../event/pipe/SubscriptionPipeEmptyEvent.java     |   2 +-
 .../event/pipe/SubscriptionPipeEvents.java         |   2 +-
 .../pipe/SubscriptionPipeTabletBatchEvents.java    |  11 +-
 .../pipe/SubscriptionPipeTsFileBatchEvents.java    |   4 +-
 .../pipe/SubscriptionPipeTsFilePlainEvent.java     |   2 +-
 .../db/tools/schema/SRStatementGenerator.java      |   2 +
 .../db/utils/datastructure/AlignedTVList.java      |  44 +-
 .../iotdb/db/utils/datastructure/BinaryTVList.java |  38 +-
 .../db/utils/datastructure/BooleanTVList.java      |  38 +-
 .../iotdb/db/utils/datastructure/DoubleTVList.java |  38 +-
 .../iotdb/db/utils/datastructure/FloatTVList.java  |  38 +-
 .../iotdb/db/utils/datastructure/IntTVList.java    |  38 +-
 .../iotdb/db/utils/datastructure/LongTVList.java   |  38 +-
 .../iotdb/db/utils/datastructure/TVList.java       |  70 ++-
 .../db/protocol/mqtt/PayloadFormatManagerTest.java |   2 +-
 .../execution/operator/OperatorMemoryTest.java     |   2 -
 .../relational/analyzer/GroupByLargeDataTest.java  | 179 ++++++
 .../relational/planner/CorrelatedSubqueryTest.java | 153 +++++
 ...ueryTest.java => UncorrelatedSubqueryTest.java} | 104 ++-
 .../planner/assertions/AssignUniqueIdMatcher.java} |  32 +-
 .../planner/assertions/PlanMatchPattern.java       |   6 +
 .../dataregion/memtable/TsFileProcessorTest.java   |  36 +-
 .../conf/iotdb-system.properties.template          |  25 +-
 .../iotdb/commons/path/ExtendedPartialPath.java    |   4 +
 .../agent/plugin/builtin/BuiltinPipePlugin.java    |   5 +
 .../builtin/connector/opcda/OpcDaConnector.java}   |  21 +-
 .../config/constant/PipeConnectorConstant.java     |   8 +-
 .../connector/protocol/IoTDBAirGapConnector.java   |  21 +-
 .../iotdb/commons/pipe/event/EnrichedEvent.java    |  17 +-
 .../apache/iotdb/commons/schema/table/TsTable.java |   2 +-
 .../thrift-commons/src/main/thrift/common.thrift   |   1 +
 .../src/main/thrift/pipeconsensus.thrift           |   5 +-
 pom.xml                                            |   8 +-
 183 files changed, 8293 insertions(+), 1162 deletions(-)

diff --cc iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java
index 3748980d2ad,ab4b88fb78c..fe0289176ed
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java
@@@ -105,7 -105,7 +105,8 @@@ import org.apache.iotdb.mpp.rpc.thrift.
  import org.apache.iotdb.rpc.RpcUtils;
  import org.apache.iotdb.rpc.TSStatusCode;
  
+ import org.apache.tsfile.annotations.TableModel;
 +import org.apache.tsfile.enums.TSDataType;
  import org.apache.tsfile.file.metadata.IDeviceID;
  import org.apache.tsfile.utils.Pair;
  import org.slf4j.Logger;
diff --cc iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java
index b4fec9efc5e,e313c6d93a0..01969873a7f
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java
@@@ -93,7 -91,7 +93,8 @@@ import org.apache.iotdb.db.schemaengine
  import org.apache.iotdb.rpc.RpcUtils;
  import org.apache.iotdb.rpc.TSStatusCode;
  
+ import org.apache.tsfile.annotations.TableModel;
 +import org.apache.tsfile.enums.TSDataType;
  import org.apache.tsfile.utils.Pair;
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java
index c5b91f4dcdd,36c8470af2c..7d06d1f7e32
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java
@@@ -170,8 -173,11 +175,16 @@@ public class AlignedWritableMemChunkGro
      return memChunkGroup;
    }
  
 +  @Override
 +  public void checkDataType(InsertNode node) throws DataTypeInconsistentException {
 +    memChunk.checkDataType(node);
 +  }
++
+   protected static AlignedWritableMemChunkGroup deserializeSingleTVListMemChunks(
+       DataInputStream stream, boolean isTableModel) throws IOException {
+     AlignedWritableMemChunkGroup memChunkGroup = new AlignedWritableMemChunkGroup();
+     memChunkGroup.memChunk =
+         AlignedWritableMemChunk.deserializeSingleTVListMemChunks(stream, isTableModel);
+     return memChunkGroup;
+   }
  }
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java
index a37475e2e0b,382573e7450..a61407d4e48
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java
@@@ -60,7 -58,7 +60,9 @@@ public interface IWritableMemChunkGrou
  
    long getMeasurementSize(String measurement);
  
+   IWritableMemChunk getWritableMemChunk(String measurement);
+ 
    long getMaxTime();
 +
 +  void checkDataType(InsertNode node) throws DataTypeInconsistentException;
  }
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java
index deb73559b19,e47debd9d73..824b5c7a958
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java
@@@ -197,19 -202,15 +205,31 @@@ public class WritableMemChunkGroup impl
      return memChunkGroup;
    }
  
 +  @Override
 +  public void checkDataType(InsertNode node) throws DataTypeInconsistentException {
 +    for (MeasurementSchema incomingSchema : node.getMeasurementSchemas()) {
 +      if (incomingSchema == null) {
 +        continue;
 +      }
 +
 +      IWritableMemChunk memChunk = memChunkMap.get(incomingSchema.getMeasurementName());
 +      if (memChunk != null
 +          && memChunk.getWorkingTVList().getDataType() != incomingSchema.getType()) {
 +        throw new DataTypeInconsistentException(
 +            memChunk.getWorkingTVList().getDataType(), incomingSchema.getType());
 +      }
 +    }
 +  }
++
+   public static WritableMemChunkGroup deserializeSingleTVListMemChunks(DataInputStream stream)
+       throws IOException {
+     WritableMemChunkGroup memChunkGroup = new WritableMemChunkGroup();
+     int memChunkMapSize = stream.readInt();
+     for (int i = 0; i < memChunkMapSize; ++i) {
+       String measurement = ReadWriteIOUtils.readString(stream);
+       IWritableMemChunk memChunk = WritableMemChunk.deserializeSingleTVListMemChunks(stream);
+       memChunkGroup.memChunkMap.put(measurement, memChunk);
+     }
+     return memChunkGroup;
+   }
  }