You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by yc...@apache.org on 2025/03/04 09:44:55 UTC

(iotdb) 07/07: Merge remote-tracking branch 'origin/master' into ycy/AINodeTraining

This is an automated email from the ASF dual-hosted git repository.

ycycse pushed a commit to branch ycy/AINodeTraining
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit 49b6c0b29412c42b406668b096fc71bdede40a86
Merge: dc468e1824c e8d07c156da
Author: YangCaiyin <yc...@gmail.com>
AuthorDate: Tue Mar 4 17:44:14 2025 +0800

    Merge remote-tracking branch 'origin/master' into ycy/AINodeTraining

 .github/workflows/vulnerability-check.yml          |   7 +-
 dependencies.json                                  |   1 +
 .../server/CustomizedJsonPayloadFormatter.java     |   8 +-
 .../org/apache/iotdb/mqtt/server/MyMessage.java    |  80 ---
 .../java/org/apache/iotdb/mqtt/MQTTClient.java     |   2 +-
 .../relational/it/db/it/IoTDBInsertTableIT.java    |  11 +-
 .../db/it/IoTDBMultiTAGsWithAttributesTableIT.java |  12 +-
 .../relational/it/mqtt/IoTDBMQTTServiceIT.java     |   2 +-
 .../IoTDBCorrelatedExistsSubqueryIT.java           | 428 +++++++++++++
 .../IoTDBUncorrelatedExistsSubqueryIT.java         | 391 ++++++++++++
 .../it/session/IoTDBSessionRelationalIT.java       |  34 -
 .../iotdb/session/it/IoTDBSessionSimpleIT.java     |  48 --
 .../org/apache/iotdb/tool/common/OptionsUtil.java  |  54 +-
 .../org/apache/iotdb/tool/data/ExportData.java     |   4 +-
 .../iotdb/jdbc/IoTDBAbstractDatabaseMetadata.java  |  88 +--
 .../org/apache/iotdb/jdbc/IoTDBConnection.java     |   7 +
 .../apache/iotdb/jdbc/IoTDBDatabaseMetadata.java   |  64 ++
 .../org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java  |  25 +-
 .../apache/iotdb/jdbc/IoTDBPreparedStatement.java  |  31 +-
 .../IoTDBRelationalDatabaseMetadata.java           | 146 ++++-
 .../java/org/apache/iotdb/session/Session.java     |  17 +-
 .../SubscriptionTablePushConsumerBuilder.java      |   3 +-
 iotdb-core/confignode/pom.xml                      |   4 +
 .../iotdb/confignode/conf/ConfigNodeConfig.java    |  45 ++
 .../confignode/conf/ConfigNodeDescriptor.java      |  30 +
 .../iotdb/confignode/manager/ConfigManager.java    |   9 +
 .../apache/iotdb/confignode/manager/IManager.java  |   3 +
 .../manager/load/balancer/RouteBalancer.java       | 154 +++--
 .../manager/load/cache/AbstractLoadCache.java      |  26 +-
 .../cache/IFailureDetector.java}                   |  26 +-
 .../manager/load/cache/detector/FixedDetector.java |  58 ++
 .../load/cache/detector/PhiAccrualDetector.java    | 181 ++++++
 .../load/cache/node/AINodeHeartbeatCache.java      |  15 +-
 .../load/cache/node/ConfigNodeHeartbeatCache.java  |  12 +-
 .../load/cache/node/DataNodeHeartbeatCache.java    |  11 +-
 .../manager/load/cache/region/RegionCache.java     |  12 +-
 .../iotdb/confignode/manager/node/NodeManager.java |   8 +
 .../pipe/agent/task/PipeConfigNodeSubtask.java     |   2 +-
 .../pipe/agent/task/PipeConfigNodeTaskAgent.java   |   4 +-
 .../pipe/extractor/IoTDBConfigRegionExtractor.java |   4 +-
 .../manager/pipe/metric/PipeConfigNodeMetrics.java |   9 +
 .../PipeConfigNodeRemainingTimeMetrics.java        |   2 +-
 .../PipeConfigNodeRemainingTimeOperator.java       |   2 +-
 .../PipeConfigNodeResourceMetrics.java             |   2 +-
 .../{ => overview}/PipeProcedureMetrics.java       |   2 +-
 .../metric/{ => overview}/PipeTaskInfoMetrics.java |   2 +-
 .../PipeTemporaryMetaInCoordinatorMetrics.java     |   2 +-
 .../PipeConfigNodeReceiverMetrics.java             |   2 +-
 .../PipeConfigRegionConnectorMetrics.java          |   2 +-
 .../PipeConfigNodeListenerMetrics.java             |   2 +-
 .../PipeConfigRegionExtractorMetrics.java          |   2 +-
 .../receiver/protocol/IoTDBConfigNodeReceiver.java |   2 +-
 .../resource/PipeConfigNodeResourceManager.java    |   1 +
 .../PipeConfigNodeSnapshotResourceManager.java     |   2 +-
 .../manager/schema/ClusterSchemaManager.java       |  12 +-
 .../iotdb/confignode/persistence/TTLInfo.java      |   2 +-
 .../confignode/persistence/pipe/PipeInfo.java      |   2 +-
 .../persistence/schema/ClusterSchemaInfo.java      |  20 +
 .../procedure/PartitionTableAutoCleaner.java       |  14 +-
 .../procedure/impl/StateMachineProcedure.java      |   9 +-
 .../impl/pipe/AbstractOperatePipeProcedureV2.java  |   4 +-
 .../impl/pipe/task/AlterPipeProcedureV2.java       |   1 +
 .../manager/load/cache/detector/DetectorTest.java  | 179 ++++++
 .../common/request/IConsensusRequest.java          |   5 +
 .../common/request/IndexedConsensusRequest.java    |   8 +-
 .../consensus/config/PipeConsensusConfig.java      |  21 +-
 .../consensus/iot/logdispatcher/LogDispatcher.java |  12 +-
 .../consensus/pipe/PipeConsensusServerImpl.java    |  21 +-
 ...xManager.java => ReplicateProgressManager.java} |   8 +-
 .../pipe/metric/PipeConsensusSyncLagManager.java   |  71 ++-
 iotdb-core/datanode/pom.xml                        |   8 +
 .../java/org/apache/iotdb/db/conf/IoTDBConfig.java |   2 +-
 .../db/consensus/DataRegionConsensusImpl.java      |   4 +-
 .../PipeDataRegionConnectorConstructor.java        |   4 +
 .../runtime/PipeSchemaRegionListenerManager.java   |   2 +-
 .../db/pipe/agent/task/PipeDataNodeTaskAgent.java  |   4 +-
 .../pipe/agent/task/connection/EnrichedDeque.java  |   2 +-
 .../agent/task/connection/PipeEventCollector.java  |  14 +-
 .../agent/task/stage/PipeTaskProcessorStage.java   |   5 +-
 .../subtask/connector/PipeConnectorSubtask.java    |   4 +-
 .../connector/PipeConnectorSubtaskManager.java     |   2 +-
 .../PipeRealtimePriorityBlockingQueue.java         |   2 +-
 .../subtask/processor/PipeProcessorSubtask.java    |   4 +-
 .../connector/protocol/opcda/OpcDaConnector.java   | 138 ++++
 .../pipe/connector/protocol/opcda/OpcDaHeader.java | 200 ++++++
 .../protocol/opcda/OpcDaServerHandle.java          | 390 ++++++++++++
 .../connector/protocol/opcua/OpcUaConnector.java   |  50 +-
 .../connector/protocol/opcua/OpcUaNameSpace.java   |   2 +-
 .../pipeconsensus/PipeConsensusAsyncConnector.java |  16 +-
 .../pipeconsensus/PipeConsensusSyncConnector.java  |   9 +-
 .../PipeConsensusTabletInsertionEventHandler.java  |   8 +-
 .../PipeConsensusTsFileInsertionEventHandler.java  |   8 +-
 .../PipeConsensusTransferBatchReqBuilder.java      |   5 +-
 ....java => ReplicateProgressDataNodeManager.java} |  36 +-
 .../deletion/DeletionResourceManager.java          |   4 +-
 .../deletion/persist/PageCacheDeletionBuffer.java  |   5 +-
 .../event/common/heartbeat/PipeHeartbeatEvent.java |   4 +-
 .../tablet/PipeInsertNodeTabletInsertionEvent.java |  67 +-
 .../common/tablet/PipeRawTabletInsertionEvent.java |  30 +-
 .../common/tsfile/PipeTsFileInsertionEvent.java    |  19 +-
 .../scan/TsFileInsertionEventScanParser.java       |   3 -
 .../table/TsFileInsertionEventTableParser.java     |  95 ++-
 ...ileInsertionEventTableParserTabletIterator.java | 421 ++++++++++---
 .../event/realtime/PipeRealtimeEventFactory.java   |  71 ++-
 .../dataregion/IoTDBDataRegionExtractor.java       |   4 +-
 ...oricalDataRegionTsFileAndDeletionExtractor.java |  17 +
 .../realtime/PipeRealtimeDataRegionExtractor.java  |   2 +-
 .../PipeRealtimeDataRegionHybridExtractor.java     |   2 +-
 .../realtime/assigner/PipeDataRegionAssigner.java  |   6 +-
 .../dataregion/realtime/epoch/TsFileEpoch.java     |   2 +-
 .../listener/PipeInsertionDataNodeListener.java    |  16 +-
 .../schemaregion/IoTDBSchemaRegionExtractor.java   |   4 +-
 .../iotdb/db/pipe/metric/PipeDataNodeMetrics.java  |  12 +
 .../PipeDataNodeRemainingEventAndTimeMetrics.java  |   2 +-
 .../PipeDataNodeRemainingEventAndTimeOperator.java |   2 +-
 .../{ => overview}/PipeHeartbeatEventMetrics.java  |   2 +-
 .../metric/{ => overview}/PipeResourceMetrics.java |   2 +-
 .../PipeWALInsertNodeCacheMetrics.java             |   2 +-
 .../{ => processor}/PipeProcessorMetrics.java      |   2 +-
 .../PipeDataNodeReceiverMetrics.java               |   2 +-
 .../PipeSchemaRegionConnectorMetrics.java          |   2 +-
 .../PipeSchemaRegionExtractorMetrics.java          |   2 +-
 .../PipeSchemaRegionListenerMetrics.java           |   2 +-
 .../{ => sink}/PipeDataRegionConnectorMetrics.java |   2 +-
 .../metric/{ => source}/PipeAssignerMetrics.java   |   2 +-
 .../{ => source}/PipeDataRegionEventCounter.java   |   2 +-
 .../PipeDataRegionExtractorMetrics.java            |   2 +-
 .../pipeconsensus/PipeConsensusProcessor.java      |  43 +-
 .../pipeconsensus/PipeConsensusReceiver.java       | 338 +++++-----
 .../protocol/thrift/IoTDBDataNodeReceiver.java     |   2 +-
 .../resource/memory/InsertNodeMemoryEstimator.java |   2 +-
 .../pipe/resource/memory/PipeMemoryWeightUtil.java |  65 ++
 .../db/protocol/mqtt/JSONPayloadFormatter.java     |   7 +-
 .../db/protocol/mqtt/LinePayloadFormatter.java     |   7 +-
 .../iotdb/db/protocol/mqtt/MPPPublishHandler.java  |  11 +-
 .../iotdb/db/protocol/mqtt/PayloadFormatter.java   |   6 +
 .../protocol/thrift/impl/ClientRPCServiceImpl.java |   2 +-
 .../operator/process/AssignUniqueIdOperator.java   | 142 +++++
 .../operator/schema/SchemaQueryScanOperator.java   |  12 +-
 .../operator/schema/source/ISchemaSource.java      |   5 +
 .../schema/source/TableDeviceFetchSource.java      |  21 +
 .../schema/source/TableDeviceQuerySource.java      |  36 +-
 .../InformationSchemaContentSupplierFactory.java   |   2 +-
 .../relational/MergeSortLeftJoinOperator.java      | 163 +++++
 .../grouped/StreamingHashAggregationOperator.java  |   3 +-
 .../aggregation/grouped/hash/HashStrategy.java     |   2 +-
 .../plan/analyze/ClusterPartitionFetcher.java      |   5 +
 .../plan/analyze/IPartitionFetcher.java            |   4 +
 .../execution/config/sys/pipe/ShowPipeTask.java    |   2 +-
 .../plan/planner/TableOperatorGenerator.java       |  53 ++
 .../plan/planner/plan/node/PlanGraphPrinter.java   |  10 +
 .../plan/planner/plan/node/PlanNodeType.java       |   4 +
 .../plan/planner/plan/node/PlanVisitor.java        |   5 +
 .../plan/planner/plan/node/write/InsertNode.java   |   3 +-
 .../planner/plan/node/write/InsertRowNode.java     |   3 -
 .../node/write/RelationalInsertTabletNode.java     |   8 -
 .../PredicatePushIntoMetadataChecker.java          |   7 +-
 .../fetcher/TableHeaderSchemaValidator.java        |   4 +
 .../distribute/TableDistributedPlanGenerator.java  |  58 +-
 .../iterative/rule/AggregationDecorrelation.java   |  83 +++
 .../iterative/rule/PruneAssignUniqueIdColumns.java |  47 ++
 .../iterative/rule/RemoveRedundantExists.java      | 104 +++
 ...orrelatedDistinctAggregationWithProjection.java | 186 ++++++
 ...elatedDistinctAggregationWithoutProjection.java | 167 +++++
 ...mCorrelatedGlobalAggregationWithProjection.java | 318 ++++++++++
 ...rrelatedGlobalAggregationWithoutProjection.java | 302 +++++++++
 ...CorrelatedGroupedAggregationWithProjection.java | 252 ++++++++
 ...relatedGroupedAggregationWithoutProjection.java | 233 +++++++
 .../rule/TransformCorrelatedJoinToJoin.java        |  99 +++
 .../rule/TransformExistsApplyToCorrelatedJoin.java | 214 +++++++
 .../relational/planner/node/AssignUniqueId.java    |  97 +++
 .../plan/relational/planner/node/Patterns.java     |  27 +-
 .../planner/optimizations/JoinUtils.java           | 249 +++++---
 .../optimizations/LogicalOptimizeFactory.java      |  40 +-
 .../optimizations/PlanNodeDecorrelator.java        | 699 +++++++++++++++++++++
 .../optimizations/PushPredicateIntoTableScan.java  | 158 ++++-
 .../optimizations/UnaliasSymbolReferences.java     |  13 +
 .../relational/planner/optimizations/Util.java     |  19 +
 .../plan/relational/sql/ast/InsertRows.java        |   3 -
 .../plan/scheduler/load/LoadTsFileScheduler.java   |  25 +
 .../plan/statement/crud/InsertBaseStatement.java   |  28 +-
 .../plan/statement/crud/InsertRowStatement.java    |   3 -
 .../rescon/ISchemaRegionStatistics.java            |   2 +
 .../rescon/MemSchemaRegionStatistics.java          |  16 +
 .../attribute/DeviceAttributeStore.java            |  48 +-
 .../attribute/IDeviceAttributeStore.java           |  16 +-
 .../schemaregion/impl/SchemaRegionMemoryImpl.java  |  36 +-
 .../iotdb/db/storageengine/StorageEngine.java      |  19 +-
 .../dataregion/memtable/AbstractMemTable.java      |  30 +-
 .../memtable/AlignedWritableMemChunkGroup.java     |   4 +-
 .../dataregion/memtable/IMemTable.java             |   2 +-
 .../memtable/IWritableMemChunkGroup.java           |   2 +-
 .../dataregion/memtable/ReadOnlyMemChunk.java      |  10 +-
 .../dataregion/memtable/TsFileProcessor.java       |  43 +-
 .../dataregion/memtable/WritableMemChunk.java      |   4 +-
 .../dataregion/memtable/WritableMemChunkGroup.java |   6 +-
 .../read/reader/chunk/MemChunkReader.java          |   4 +-
 .../read/reader/chunk/MemPageReader.java           |   6 +-
 .../dataregion/wal/utils/WALInsertNodeCache.java   |   2 +-
 .../broker/SubscriptionPrefetchingQueue.java       |  10 +-
 .../broker/SubscriptionPrefetchingTabletQueue.java |   2 +-
 .../broker/SubscriptionPrefetchingTsFileQueue.java |   2 +-
 .../db/subscription/event/SubscriptionEvent.java   |   4 +-
 .../event/batch/SubscriptionPipeEventBatch.java    |   2 +-
 .../event/batch/SubscriptionPipeEventBatches.java  |   2 +-
 .../batch/SubscriptionPipeTabletEventBatch.java    |  50 +-
 .../SubscriptionPipeTabletIterationSnapshot.java   |  44 +-
 .../batch/SubscriptionPipeTsFileEventBatch.java    |   2 +-
 .../event/pipe/SubscriptionPipeEmptyEvent.java     |   2 +-
 .../event/pipe/SubscriptionPipeEvents.java         |   2 +-
 .../pipe/SubscriptionPipeTabletBatchEvents.java    |  11 +-
 .../pipe/SubscriptionPipeTsFileBatchEvents.java    |   4 +-
 .../pipe/SubscriptionPipeTsFilePlainEvent.java     |   2 +-
 .../SubscriptionConnectorSubtaskManager.java       |   2 +-
 .../db/tools/schema/SRStatementGenerator.java      |   2 +
 .../db/utils/datastructure/AlignedTVList.java      |  44 +-
 .../iotdb/db/utils/datastructure/BinaryTVList.java |  25 +-
 .../db/utils/datastructure/BooleanTVList.java      |  25 +-
 .../iotdb/db/utils/datastructure/DoubleTVList.java |  25 +-
 .../iotdb/db/utils/datastructure/FloatTVList.java  |  25 +-
 .../iotdb/db/utils/datastructure/IntTVList.java    |  25 +-
 .../iotdb/db/utils/datastructure/LongTVList.java   |  25 +-
 ...tIterator.java => MergeSortTVListIterator.java} |  12 +-
 .../iotdb/db/utils/datastructure/TVList.java       |  47 +-
 .../db/protocol/mqtt/PayloadFormatManagerTest.java |   2 +-
 .../execution/operator/OperatorMemoryTest.java     |   2 -
 .../plan/analyze/FakePartitionFetcherImpl.java     |   5 +
 .../plan/planner/distribution/Util.java            |   5 +
 .../plan/planner/distribution/Util2.java           |   5 +
 .../analyzer/AggregationCornerCaseTest.java        | 323 ++++++++++
 .../plan/relational/analyzer/TSBSMetadata.java     |   7 +
 .../plan/relational/analyzer/TestMatadata.java     |   7 +
 .../relational/planner/CorrelatedSubqueryTest.java | 153 +++++
 ...ueryTest.java => UncorrelatedSubqueryTest.java} | 104 ++-
 .../planner/assertions/AssignUniqueIdMatcher.java} |  32 +-
 .../planner/assertions/PlanMatchPattern.java       |   6 +
 .../dataregion/memtable/TsFileProcessorTest.java   |  36 +-
 .../read/reader/chunk/MemChunkLoaderTest.java      |  26 +-
 .../conf/iotdb-system.properties.template          |  25 +-
 .../apache/iotdb/commons/conf/CommonConfig.java    |  34 +
 .../iotdb/commons/conf/CommonDescriptor.java       |  16 +
 .../iotdb/commons/path/ExtendedPartialPath.java    |   4 +
 .../agent/plugin/builtin/BuiltinPipePlugin.java    |   5 +
 .../builtin/connector/opcda/OpcDaConnector.java}   |  23 +-
 .../config/constant/PipeConnectorConstant.java     |   8 +-
 .../connector/protocol/IoTDBAirGapConnector.java   |  21 +-
 .../iotdb/commons/pipe/event/EnrichedEvent.java    |  17 +-
 .../apache/iotdb/commons/schema/table/TsTable.java |   2 +-
 .../subscription/config/SubscriptionConfig.java    |  21 +
 .../thrift-commons/src/main/thrift/common.thrift   |   1 +
 .../src/main/thrift/pipeconsensus.thrift           |   5 +-
 pom.xml                                            |   8 +-
 252 files changed, 8790 insertions(+), 1381 deletions(-)