You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by yc...@apache.org on 2025/03/04 09:44:55 UTC
(iotdb) 07/07: Merge remote-tracking branch 'origin/master' into ycy/AINodeTraining
This is an automated email from the ASF dual-hosted git repository.
ycycse pushed a commit to branch ycy/AINodeTraining
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 49b6c0b29412c42b406668b096fc71bdede40a86
Merge: dc468e1824c e8d07c156da
Author: YangCaiyin <yc...@gmail.com>
AuthorDate: Tue Mar 4 17:44:14 2025 +0800
Merge remote-tracking branch 'origin/master' into ycy/AINodeTraining
.github/workflows/vulnerability-check.yml | 7 +-
dependencies.json | 1 +
.../server/CustomizedJsonPayloadFormatter.java | 8 +-
.../org/apache/iotdb/mqtt/server/MyMessage.java | 80 ---
.../java/org/apache/iotdb/mqtt/MQTTClient.java | 2 +-
.../relational/it/db/it/IoTDBInsertTableIT.java | 11 +-
.../db/it/IoTDBMultiTAGsWithAttributesTableIT.java | 12 +-
.../relational/it/mqtt/IoTDBMQTTServiceIT.java | 2 +-
.../IoTDBCorrelatedExistsSubqueryIT.java | 428 +++++++++++++
.../IoTDBUncorrelatedExistsSubqueryIT.java | 391 ++++++++++++
.../it/session/IoTDBSessionRelationalIT.java | 34 -
.../iotdb/session/it/IoTDBSessionSimpleIT.java | 48 --
.../org/apache/iotdb/tool/common/OptionsUtil.java | 54 +-
.../org/apache/iotdb/tool/data/ExportData.java | 4 +-
.../iotdb/jdbc/IoTDBAbstractDatabaseMetadata.java | 88 +--
.../org/apache/iotdb/jdbc/IoTDBConnection.java | 7 +
.../apache/iotdb/jdbc/IoTDBDatabaseMetadata.java | 64 ++
.../org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java | 25 +-
.../apache/iotdb/jdbc/IoTDBPreparedStatement.java | 31 +-
.../IoTDBRelationalDatabaseMetadata.java | 146 ++++-
.../java/org/apache/iotdb/session/Session.java | 17 +-
.../SubscriptionTablePushConsumerBuilder.java | 3 +-
iotdb-core/confignode/pom.xml | 4 +
.../iotdb/confignode/conf/ConfigNodeConfig.java | 45 ++
.../confignode/conf/ConfigNodeDescriptor.java | 30 +
.../iotdb/confignode/manager/ConfigManager.java | 9 +
.../apache/iotdb/confignode/manager/IManager.java | 3 +
.../manager/load/balancer/RouteBalancer.java | 154 +++--
.../manager/load/cache/AbstractLoadCache.java | 26 +-
.../cache/IFailureDetector.java} | 26 +-
.../manager/load/cache/detector/FixedDetector.java | 58 ++
.../load/cache/detector/PhiAccrualDetector.java | 181 ++++++
.../load/cache/node/AINodeHeartbeatCache.java | 15 +-
.../load/cache/node/ConfigNodeHeartbeatCache.java | 12 +-
.../load/cache/node/DataNodeHeartbeatCache.java | 11 +-
.../manager/load/cache/region/RegionCache.java | 12 +-
.../iotdb/confignode/manager/node/NodeManager.java | 8 +
.../pipe/agent/task/PipeConfigNodeSubtask.java | 2 +-
.../pipe/agent/task/PipeConfigNodeTaskAgent.java | 4 +-
.../pipe/extractor/IoTDBConfigRegionExtractor.java | 4 +-
.../manager/pipe/metric/PipeConfigNodeMetrics.java | 9 +
.../PipeConfigNodeRemainingTimeMetrics.java | 2 +-
.../PipeConfigNodeRemainingTimeOperator.java | 2 +-
.../PipeConfigNodeResourceMetrics.java | 2 +-
.../{ => overview}/PipeProcedureMetrics.java | 2 +-
.../metric/{ => overview}/PipeTaskInfoMetrics.java | 2 +-
.../PipeTemporaryMetaInCoordinatorMetrics.java | 2 +-
.../PipeConfigNodeReceiverMetrics.java | 2 +-
.../PipeConfigRegionConnectorMetrics.java | 2 +-
.../PipeConfigNodeListenerMetrics.java | 2 +-
.../PipeConfigRegionExtractorMetrics.java | 2 +-
.../receiver/protocol/IoTDBConfigNodeReceiver.java | 2 +-
.../resource/PipeConfigNodeResourceManager.java | 1 +
.../PipeConfigNodeSnapshotResourceManager.java | 2 +-
.../manager/schema/ClusterSchemaManager.java | 12 +-
.../iotdb/confignode/persistence/TTLInfo.java | 2 +-
.../confignode/persistence/pipe/PipeInfo.java | 2 +-
.../persistence/schema/ClusterSchemaInfo.java | 20 +
.../procedure/PartitionTableAutoCleaner.java | 14 +-
.../procedure/impl/StateMachineProcedure.java | 9 +-
.../impl/pipe/AbstractOperatePipeProcedureV2.java | 4 +-
.../impl/pipe/task/AlterPipeProcedureV2.java | 1 +
.../manager/load/cache/detector/DetectorTest.java | 179 ++++++
.../common/request/IConsensusRequest.java | 5 +
.../common/request/IndexedConsensusRequest.java | 8 +-
.../consensus/config/PipeConsensusConfig.java | 21 +-
.../consensus/iot/logdispatcher/LogDispatcher.java | 12 +-
.../consensus/pipe/PipeConsensusServerImpl.java | 21 +-
...xManager.java => ReplicateProgressManager.java} | 8 +-
.../pipe/metric/PipeConsensusSyncLagManager.java | 71 ++-
iotdb-core/datanode/pom.xml | 8 +
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 2 +-
.../db/consensus/DataRegionConsensusImpl.java | 4 +-
.../PipeDataRegionConnectorConstructor.java | 4 +
.../runtime/PipeSchemaRegionListenerManager.java | 2 +-
.../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 4 +-
.../pipe/agent/task/connection/EnrichedDeque.java | 2 +-
.../agent/task/connection/PipeEventCollector.java | 14 +-
.../agent/task/stage/PipeTaskProcessorStage.java | 5 +-
.../subtask/connector/PipeConnectorSubtask.java | 4 +-
.../connector/PipeConnectorSubtaskManager.java | 2 +-
.../PipeRealtimePriorityBlockingQueue.java | 2 +-
.../subtask/processor/PipeProcessorSubtask.java | 4 +-
.../connector/protocol/opcda/OpcDaConnector.java | 138 ++++
.../pipe/connector/protocol/opcda/OpcDaHeader.java | 200 ++++++
.../protocol/opcda/OpcDaServerHandle.java | 390 ++++++++++++
.../connector/protocol/opcua/OpcUaConnector.java | 50 +-
.../connector/protocol/opcua/OpcUaNameSpace.java | 2 +-
.../pipeconsensus/PipeConsensusAsyncConnector.java | 16 +-
.../pipeconsensus/PipeConsensusSyncConnector.java | 9 +-
.../PipeConsensusTabletInsertionEventHandler.java | 8 +-
.../PipeConsensusTsFileInsertionEventHandler.java | 8 +-
.../PipeConsensusTransferBatchReqBuilder.java | 5 +-
....java => ReplicateProgressDataNodeManager.java} | 36 +-
.../deletion/DeletionResourceManager.java | 4 +-
.../deletion/persist/PageCacheDeletionBuffer.java | 5 +-
.../event/common/heartbeat/PipeHeartbeatEvent.java | 4 +-
.../tablet/PipeInsertNodeTabletInsertionEvent.java | 67 +-
.../common/tablet/PipeRawTabletInsertionEvent.java | 30 +-
.../common/tsfile/PipeTsFileInsertionEvent.java | 19 +-
.../scan/TsFileInsertionEventScanParser.java | 3 -
.../table/TsFileInsertionEventTableParser.java | 95 ++-
...ileInsertionEventTableParserTabletIterator.java | 421 ++++++++++---
.../event/realtime/PipeRealtimeEventFactory.java | 71 ++-
.../dataregion/IoTDBDataRegionExtractor.java | 4 +-
...oricalDataRegionTsFileAndDeletionExtractor.java | 17 +
.../realtime/PipeRealtimeDataRegionExtractor.java | 2 +-
.../PipeRealtimeDataRegionHybridExtractor.java | 2 +-
.../realtime/assigner/PipeDataRegionAssigner.java | 6 +-
.../dataregion/realtime/epoch/TsFileEpoch.java | 2 +-
.../listener/PipeInsertionDataNodeListener.java | 16 +-
.../schemaregion/IoTDBSchemaRegionExtractor.java | 4 +-
.../iotdb/db/pipe/metric/PipeDataNodeMetrics.java | 12 +
.../PipeDataNodeRemainingEventAndTimeMetrics.java | 2 +-
.../PipeDataNodeRemainingEventAndTimeOperator.java | 2 +-
.../{ => overview}/PipeHeartbeatEventMetrics.java | 2 +-
.../metric/{ => overview}/PipeResourceMetrics.java | 2 +-
.../PipeWALInsertNodeCacheMetrics.java | 2 +-
.../{ => processor}/PipeProcessorMetrics.java | 2 +-
.../PipeDataNodeReceiverMetrics.java | 2 +-
.../PipeSchemaRegionConnectorMetrics.java | 2 +-
.../PipeSchemaRegionExtractorMetrics.java | 2 +-
.../PipeSchemaRegionListenerMetrics.java | 2 +-
.../{ => sink}/PipeDataRegionConnectorMetrics.java | 2 +-
.../metric/{ => source}/PipeAssignerMetrics.java | 2 +-
.../{ => source}/PipeDataRegionEventCounter.java | 2 +-
.../PipeDataRegionExtractorMetrics.java | 2 +-
.../pipeconsensus/PipeConsensusProcessor.java | 43 +-
.../pipeconsensus/PipeConsensusReceiver.java | 338 +++++-----
.../protocol/thrift/IoTDBDataNodeReceiver.java | 2 +-
.../resource/memory/InsertNodeMemoryEstimator.java | 2 +-
.../pipe/resource/memory/PipeMemoryWeightUtil.java | 65 ++
.../db/protocol/mqtt/JSONPayloadFormatter.java | 7 +-
.../db/protocol/mqtt/LinePayloadFormatter.java | 7 +-
.../iotdb/db/protocol/mqtt/MPPPublishHandler.java | 11 +-
.../iotdb/db/protocol/mqtt/PayloadFormatter.java | 6 +
.../protocol/thrift/impl/ClientRPCServiceImpl.java | 2 +-
.../operator/process/AssignUniqueIdOperator.java | 142 +++++
.../operator/schema/SchemaQueryScanOperator.java | 12 +-
.../operator/schema/source/ISchemaSource.java | 5 +
.../schema/source/TableDeviceFetchSource.java | 21 +
.../schema/source/TableDeviceQuerySource.java | 36 +-
.../InformationSchemaContentSupplierFactory.java | 2 +-
.../relational/MergeSortLeftJoinOperator.java | 163 +++++
.../grouped/StreamingHashAggregationOperator.java | 3 +-
.../aggregation/grouped/hash/HashStrategy.java | 2 +-
.../plan/analyze/ClusterPartitionFetcher.java | 5 +
.../plan/analyze/IPartitionFetcher.java | 4 +
.../execution/config/sys/pipe/ShowPipeTask.java | 2 +-
.../plan/planner/TableOperatorGenerator.java | 53 ++
.../plan/planner/plan/node/PlanGraphPrinter.java | 10 +
.../plan/planner/plan/node/PlanNodeType.java | 4 +
.../plan/planner/plan/node/PlanVisitor.java | 5 +
.../plan/planner/plan/node/write/InsertNode.java | 3 +-
.../planner/plan/node/write/InsertRowNode.java | 3 -
.../node/write/RelationalInsertTabletNode.java | 8 -
.../PredicatePushIntoMetadataChecker.java | 7 +-
.../fetcher/TableHeaderSchemaValidator.java | 4 +
.../distribute/TableDistributedPlanGenerator.java | 58 +-
.../iterative/rule/AggregationDecorrelation.java | 83 +++
.../iterative/rule/PruneAssignUniqueIdColumns.java | 47 ++
.../iterative/rule/RemoveRedundantExists.java | 104 +++
...orrelatedDistinctAggregationWithProjection.java | 186 ++++++
...elatedDistinctAggregationWithoutProjection.java | 167 +++++
...mCorrelatedGlobalAggregationWithProjection.java | 318 ++++++++++
...rrelatedGlobalAggregationWithoutProjection.java | 302 +++++++++
...CorrelatedGroupedAggregationWithProjection.java | 252 ++++++++
...relatedGroupedAggregationWithoutProjection.java | 233 +++++++
.../rule/TransformCorrelatedJoinToJoin.java | 99 +++
.../rule/TransformExistsApplyToCorrelatedJoin.java | 214 +++++++
.../relational/planner/node/AssignUniqueId.java | 97 +++
.../plan/relational/planner/node/Patterns.java | 27 +-
.../planner/optimizations/JoinUtils.java | 249 +++++---
.../optimizations/LogicalOptimizeFactory.java | 40 +-
.../optimizations/PlanNodeDecorrelator.java | 699 +++++++++++++++++++++
.../optimizations/PushPredicateIntoTableScan.java | 158 ++++-
.../optimizations/UnaliasSymbolReferences.java | 13 +
.../relational/planner/optimizations/Util.java | 19 +
.../plan/relational/sql/ast/InsertRows.java | 3 -
.../plan/scheduler/load/LoadTsFileScheduler.java | 25 +
.../plan/statement/crud/InsertBaseStatement.java | 28 +-
.../plan/statement/crud/InsertRowStatement.java | 3 -
.../rescon/ISchemaRegionStatistics.java | 2 +
.../rescon/MemSchemaRegionStatistics.java | 16 +
.../attribute/DeviceAttributeStore.java | 48 +-
.../attribute/IDeviceAttributeStore.java | 16 +-
.../schemaregion/impl/SchemaRegionMemoryImpl.java | 36 +-
.../iotdb/db/storageengine/StorageEngine.java | 19 +-
.../dataregion/memtable/AbstractMemTable.java | 30 +-
.../memtable/AlignedWritableMemChunkGroup.java | 4 +-
.../dataregion/memtable/IMemTable.java | 2 +-
.../memtable/IWritableMemChunkGroup.java | 2 +-
.../dataregion/memtable/ReadOnlyMemChunk.java | 10 +-
.../dataregion/memtable/TsFileProcessor.java | 43 +-
.../dataregion/memtable/WritableMemChunk.java | 4 +-
.../dataregion/memtable/WritableMemChunkGroup.java | 6 +-
.../read/reader/chunk/MemChunkReader.java | 4 +-
.../read/reader/chunk/MemPageReader.java | 6 +-
.../dataregion/wal/utils/WALInsertNodeCache.java | 2 +-
.../broker/SubscriptionPrefetchingQueue.java | 10 +-
.../broker/SubscriptionPrefetchingTabletQueue.java | 2 +-
.../broker/SubscriptionPrefetchingTsFileQueue.java | 2 +-
.../db/subscription/event/SubscriptionEvent.java | 4 +-
.../event/batch/SubscriptionPipeEventBatch.java | 2 +-
.../event/batch/SubscriptionPipeEventBatches.java | 2 +-
.../batch/SubscriptionPipeTabletEventBatch.java | 50 +-
.../SubscriptionPipeTabletIterationSnapshot.java | 44 +-
.../batch/SubscriptionPipeTsFileEventBatch.java | 2 +-
.../event/pipe/SubscriptionPipeEmptyEvent.java | 2 +-
.../event/pipe/SubscriptionPipeEvents.java | 2 +-
.../pipe/SubscriptionPipeTabletBatchEvents.java | 11 +-
.../pipe/SubscriptionPipeTsFileBatchEvents.java | 4 +-
.../pipe/SubscriptionPipeTsFilePlainEvent.java | 2 +-
.../SubscriptionConnectorSubtaskManager.java | 2 +-
.../db/tools/schema/SRStatementGenerator.java | 2 +
.../db/utils/datastructure/AlignedTVList.java | 44 +-
.../iotdb/db/utils/datastructure/BinaryTVList.java | 25 +-
.../db/utils/datastructure/BooleanTVList.java | 25 +-
.../iotdb/db/utils/datastructure/DoubleTVList.java | 25 +-
.../iotdb/db/utils/datastructure/FloatTVList.java | 25 +-
.../iotdb/db/utils/datastructure/IntTVList.java | 25 +-
.../iotdb/db/utils/datastructure/LongTVList.java | 25 +-
...tIterator.java => MergeSortTVListIterator.java} | 12 +-
.../iotdb/db/utils/datastructure/TVList.java | 47 +-
.../db/protocol/mqtt/PayloadFormatManagerTest.java | 2 +-
.../execution/operator/OperatorMemoryTest.java | 2 -
.../plan/analyze/FakePartitionFetcherImpl.java | 5 +
.../plan/planner/distribution/Util.java | 5 +
.../plan/planner/distribution/Util2.java | 5 +
.../analyzer/AggregationCornerCaseTest.java | 323 ++++++++++
.../plan/relational/analyzer/TSBSMetadata.java | 7 +
.../plan/relational/analyzer/TestMatadata.java | 7 +
.../relational/planner/CorrelatedSubqueryTest.java | 153 +++++
...ueryTest.java => UncorrelatedSubqueryTest.java} | 104 ++-
.../planner/assertions/AssignUniqueIdMatcher.java} | 32 +-
.../planner/assertions/PlanMatchPattern.java | 6 +
.../dataregion/memtable/TsFileProcessorTest.java | 36 +-
.../read/reader/chunk/MemChunkLoaderTest.java | 26 +-
.../conf/iotdb-system.properties.template | 25 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 34 +
.../iotdb/commons/conf/CommonDescriptor.java | 16 +
.../iotdb/commons/path/ExtendedPartialPath.java | 4 +
.../agent/plugin/builtin/BuiltinPipePlugin.java | 5 +
.../builtin/connector/opcda/OpcDaConnector.java} | 23 +-
.../config/constant/PipeConnectorConstant.java | 8 +-
.../connector/protocol/IoTDBAirGapConnector.java | 21 +-
.../iotdb/commons/pipe/event/EnrichedEvent.java | 17 +-
.../apache/iotdb/commons/schema/table/TsTable.java | 2 +-
.../subscription/config/SubscriptionConfig.java | 21 +
.../thrift-commons/src/main/thrift/common.thrift | 1 +
.../src/main/thrift/pipeconsensus.thrift | 5 +-
pom.xml | 8 +-
252 files changed, 8790 insertions(+), 1381 deletions(-)