You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2025/03/03 03:01:13 UTC
(iotdb) 01/01: Merge branch 'master' into force_ci/alter_column_datatype
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch force_ci/alter_column_datatype
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit f677dda16551c9a0e19b9b0acc7036158ef41274
Merge: 5575a3d8b11 174b2cb968f
Author: Tian Jiang <jt...@163.com>
AuthorDate: Mon Mar 3 11:01:01 2025 +0800
Merge branch 'master' into force_ci/alter_column_datatype
# Conflicts:
# iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java
# iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java
# iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java
# iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java
# iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java
.github/workflows/vulnerability-check.yml | 7 +-
dependencies.json | 1 +
.../server/CustomizedJsonPayloadFormatter.java | 8 +-
.../org/apache/iotdb/mqtt/server/MyMessage.java | 80 ---
.../java/org/apache/iotdb/mqtt/MQTTClient.java | 2 +-
.../org/apache/iotdb/ainode/it/AINodeBasicIT.java | 2 +
.../relational/it/db/it/IoTDBInsertTableIT.java | 11 +-
.../db/it/IoTDBMultiTAGsWithAttributesTableIT.java | 12 +-
.../relational/it/mqtt/IoTDBMQTTServiceIT.java | 2 +-
.../IoTDBCorrelatedExistsSubqueryIT.java | 428 +++++++++++++
.../IoTDBUncorrelatedExistsSubqueryIT.java | 391 ++++++++++++
.../it/session/IoTDBSessionRelationalIT.java | 34 -
.../iotdb/session/it/IoTDBSessionSimpleIT.java | 48 --
.../org/apache/iotdb/tool/common/OptionsUtil.java | 54 +-
.../org/apache/iotdb/tool/data/ExportData.java | 4 +-
.../org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java | 25 +-
.../apache/iotdb/jdbc/IoTDBPreparedStatement.java | 31 +-
.../IoTDBRelationalDatabaseMetadata.java | 52 +-
.../java/org/apache/iotdb/session/Session.java | 17 +-
.../SubscriptionTablePushConsumerBuilder.java | 3 +-
iotdb-core/confignode/pom.xml | 4 +
.../iotdb/confignode/conf/ConfigNodeConfig.java | 45 ++
.../confignode/conf/ConfigNodeDescriptor.java | 30 +
.../iotdb/confignode/manager/ConfigManager.java | 9 +
.../apache/iotdb/confignode/manager/IManager.java | 3 +
.../iotdb/confignode/manager/ModelManager.java | 5 +
.../manager/load/balancer/RouteBalancer.java | 154 +++--
.../manager/load/cache/AbstractLoadCache.java | 26 +-
.../manager/load/cache/IFailureDetector.java} | 24 +-
.../manager/load/cache/detector/FixedDetector.java | 58 ++
.../load/cache/detector/PhiAccrualDetector.java | 181 ++++++
.../load/cache/node/AINodeHeartbeatCache.java | 15 +-
.../load/cache/node/ConfigNodeHeartbeatCache.java | 12 +-
.../load/cache/node/DataNodeHeartbeatCache.java | 11 +-
.../manager/load/cache/region/RegionCache.java | 12 +-
.../iotdb/confignode/manager/node/NodeManager.java | 8 +
.../manager/schema/ClusterSchemaManager.java | 12 +-
.../iotdb/confignode/persistence/ModelInfo.java | 2 +-
.../iotdb/confignode/persistence/TTLInfo.java | 2 +-
.../persistence/schema/ClusterSchemaInfo.java | 20 +
.../procedure/PartitionTableAutoCleaner.java | 14 +-
.../procedure/impl/StateMachineProcedure.java | 9 +-
.../impl/pipe/AbstractOperatePipeProcedureV2.java | 2 +
.../impl/pipe/task/AlterPipeProcedureV2.java | 1 +
.../manager/load/cache/detector/DetectorTest.java | 179 ++++++
.../common/request/IConsensusRequest.java | 5 +
.../common/request/IndexedConsensusRequest.java | 8 +-
.../consensus/config/PipeConsensusConfig.java | 21 +-
.../consensus/iot/logdispatcher/LogDispatcher.java | 12 +-
.../consensus/pipe/PipeConsensusServerImpl.java | 21 +-
...xManager.java => ReplicateProgressManager.java} | 8 +-
.../pipe/metric/PipeConsensusSyncLagManager.java | 71 ++-
iotdb-core/datanode/pom.xml | 8 +
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 2 +-
.../db/consensus/DataRegionConsensusImpl.java | 4 +-
.../PipeDataRegionConnectorConstructor.java | 4 +
.../agent/task/connection/PipeEventCollector.java | 13 +-
.../agent/task/stage/PipeTaskProcessorStage.java | 5 +-
.../connector/protocol/opcda/OpcDaConnector.java | 138 ++++
.../pipe/connector/protocol/opcda/OpcDaHeader.java | 200 ++++++
.../protocol/opcda/OpcDaServerHandle.java | 390 ++++++++++++
.../connector/protocol/opcua/OpcUaConnector.java | 50 +-
.../connector/protocol/opcua/OpcUaNameSpace.java | 2 +-
.../pipeconsensus/PipeConsensusAsyncConnector.java | 16 +-
.../pipeconsensus/PipeConsensusSyncConnector.java | 9 +-
.../PipeConsensusTabletInsertionEventHandler.java | 8 +-
.../PipeConsensusTsFileInsertionEventHandler.java | 8 +-
.../PipeConsensusTransferBatchReqBuilder.java | 5 +-
....java => ReplicateProgressDataNodeManager.java} | 36 +-
.../deletion/DeletionResourceManager.java | 4 +-
.../deletion/persist/PageCacheDeletionBuffer.java | 5 +-
.../common/tsfile/PipeTsFileInsertionEvent.java | 17 +-
.../scan/TsFileInsertionEventScanParser.java | 3 -
.../table/TsFileInsertionEventTableParser.java | 95 ++-
...ileInsertionEventTableParserTabletIterator.java | 421 ++++++++++---
.../event/realtime/PipeRealtimeEventFactory.java | 71 ++-
...oricalDataRegionTsFileAndDeletionExtractor.java | 17 +
.../realtime/assigner/PipeDataRegionAssigner.java | 2 +
.../listener/PipeInsertionDataNodeListener.java | 16 +-
.../pipeconsensus/PipeConsensusProcessor.java | 43 +-
.../pipeconsensus/PipeConsensusReceiver.java | 338 +++++-----
.../resource/memory/InsertNodeMemoryEstimator.java | 2 +-
.../pipe/resource/memory/PipeMemoryWeightUtil.java | 65 ++
.../db/protocol/mqtt/JSONPayloadFormatter.java | 7 +-
.../db/protocol/mqtt/LinePayloadFormatter.java | 7 +-
.../iotdb/db/protocol/mqtt/MPPPublishHandler.java | 11 +-
.../iotdb/db/protocol/mqtt/PayloadFormatter.java | 6 +
.../protocol/thrift/impl/ClientRPCServiceImpl.java | 2 +-
.../operator/process/AssignUniqueIdOperator.java | 142 +++++
.../operator/schema/SchemaQueryScanOperator.java | 12 +-
.../operator/schema/source/ISchemaSource.java | 5 +
.../schema/source/TableDeviceFetchSource.java | 21 +
.../schema/source/TableDeviceQuerySource.java | 36 +-
.../relational/MergeSortLeftJoinOperator.java | 163 +++++
.../aggregation/grouped/hash/HashStrategy.java | 2 +-
.../plan/planner/TableOperatorGenerator.java | 53 ++
.../plan/planner/plan/node/PlanGraphPrinter.java | 10 +
.../plan/planner/plan/node/PlanNodeType.java | 4 +
.../plan/planner/plan/node/PlanVisitor.java | 5 +
.../plan/planner/plan/node/write/InsertNode.java | 3 +-
.../planner/plan/node/write/InsertRowNode.java | 3 -
.../PredicatePushIntoMetadataChecker.java | 7 +-
.../fetcher/TableHeaderSchemaValidator.java | 4 +
.../distribute/TableDistributedPlanGenerator.java | 58 +-
.../iterative/rule/AggregationDecorrelation.java | 83 +++
.../iterative/rule/PruneAssignUniqueIdColumns.java | 47 ++
.../iterative/rule/RemoveRedundantExists.java | 104 +++
...orrelatedDistinctAggregationWithProjection.java | 186 ++++++
...elatedDistinctAggregationWithoutProjection.java | 167 +++++
...mCorrelatedGlobalAggregationWithProjection.java | 318 ++++++++++
...rrelatedGlobalAggregationWithoutProjection.java | 302 +++++++++
...CorrelatedGroupedAggregationWithProjection.java | 252 ++++++++
...relatedGroupedAggregationWithoutProjection.java | 233 +++++++
.../rule/TransformCorrelatedJoinToJoin.java | 99 +++
.../rule/TransformExistsApplyToCorrelatedJoin.java | 214 +++++++
.../relational/planner/node/AssignUniqueId.java | 97 +++
.../plan/relational/planner/node/Patterns.java | 27 +-
.../planner/optimizations/JoinUtils.java | 249 +++++---
.../optimizations/LogicalOptimizeFactory.java | 40 +-
.../optimizations/PlanNodeDecorrelator.java | 699 +++++++++++++++++++++
.../optimizations/PushPredicateIntoTableScan.java | 143 ++++-
.../optimizations/UnaliasSymbolReferences.java | 13 +
.../relational/planner/optimizations/Util.java | 19 +
.../plan/relational/sql/ast/InsertRows.java | 3 -
.../plan/statement/crud/InsertBaseStatement.java | 28 +-
.../plan/statement/crud/InsertRowStatement.java | 3 -
.../rescon/ISchemaRegionStatistics.java | 2 +
.../rescon/MemSchemaRegionStatistics.java | 16 +
.../attribute/DeviceAttributeStore.java | 48 +-
.../attribute/IDeviceAttributeStore.java | 16 +-
.../schemaregion/impl/SchemaRegionMemoryImpl.java | 36 +-
.../iotdb/db/storageengine/StorageEngine.java | 19 +-
.../dataregion/memtable/AbstractMemTable.java | 85 ++-
.../memtable/AlignedWritableMemChunk.java | 13 +
.../memtable/AlignedWritableMemChunkGroup.java | 15 +-
.../dataregion/memtable/IMemTable.java | 2 +
.../memtable/IWritableMemChunkGroup.java | 2 +
.../dataregion/memtable/TsFileProcessor.java | 41 +-
.../dataregion/memtable/WritableMemChunk.java | 8 +
.../dataregion/memtable/WritableMemChunkGroup.java | 20 +
.../dataregion/tsfile/TsFileResource.java | 8 +-
.../broker/SubscriptionPrefetchingQueue.java | 10 +-
.../broker/SubscriptionPrefetchingTabletQueue.java | 2 +-
.../broker/SubscriptionPrefetchingTsFileQueue.java | 2 +-
.../db/subscription/event/SubscriptionEvent.java | 4 +-
.../event/batch/SubscriptionPipeEventBatch.java | 2 +-
.../event/batch/SubscriptionPipeEventBatches.java | 2 +-
.../batch/SubscriptionPipeTabletEventBatch.java | 6 +-
.../SubscriptionPipeTabletIterationSnapshot.java | 18 +-
.../batch/SubscriptionPipeTsFileEventBatch.java | 2 +-
.../event/pipe/SubscriptionPipeEmptyEvent.java | 2 +-
.../event/pipe/SubscriptionPipeEvents.java | 2 +-
.../pipe/SubscriptionPipeTabletBatchEvents.java | 11 +-
.../pipe/SubscriptionPipeTsFileBatchEvents.java | 4 +-
.../pipe/SubscriptionPipeTsFilePlainEvent.java | 2 +-
.../db/tools/schema/SRStatementGenerator.java | 2 +
.../db/utils/datastructure/AlignedTVList.java | 44 +-
.../iotdb/db/utils/datastructure/BinaryTVList.java | 38 +-
.../db/utils/datastructure/BooleanTVList.java | 38 +-
.../iotdb/db/utils/datastructure/DoubleTVList.java | 38 +-
.../iotdb/db/utils/datastructure/FloatTVList.java | 38 +-
.../iotdb/db/utils/datastructure/IntTVList.java | 38 +-
.../iotdb/db/utils/datastructure/LongTVList.java | 38 +-
.../iotdb/db/utils/datastructure/TVList.java | 70 ++-
.../db/protocol/mqtt/PayloadFormatManagerTest.java | 2 +-
.../execution/operator/OperatorMemoryTest.java | 2 -
.../relational/analyzer/GroupByLargeDataTest.java | 179 ++++++
.../relational/planner/CorrelatedSubqueryTest.java | 153 +++++
...ueryTest.java => UncorrelatedSubqueryTest.java} | 104 ++-
.../planner/assertions/AssignUniqueIdMatcher.java} | 32 +-
.../planner/assertions/PlanMatchPattern.java | 6 +
.../dataregion/memtable/TsFileProcessorTest.java | 36 +-
.../conf/iotdb-system.properties.template | 25 +-
.../iotdb/commons/path/ExtendedPartialPath.java | 4 +
.../agent/plugin/builtin/BuiltinPipePlugin.java | 5 +
.../builtin/connector/opcda/OpcDaConnector.java} | 21 +-
.../config/constant/PipeConnectorConstant.java | 8 +-
.../connector/protocol/IoTDBAirGapConnector.java | 21 +-
.../iotdb/commons/pipe/event/EnrichedEvent.java | 17 +-
.../apache/iotdb/commons/schema/table/TsTable.java | 2 +-
.../thrift-commons/src/main/thrift/common.thrift | 1 +
.../src/main/thrift/pipeconsensus.thrift | 5 +-
pom.xml | 8 +-
183 files changed, 8293 insertions(+), 1162 deletions(-)
diff --cc iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java
index 3748980d2ad,ab4b88fb78c..fe0289176ed
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java
@@@ -105,7 -105,7 +105,8 @@@ import org.apache.iotdb.mpp.rpc.thrift.
import org.apache.iotdb.rpc.RpcUtils;
import org.apache.iotdb.rpc.TSStatusCode;
+ import org.apache.tsfile.annotations.TableModel;
+import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.utils.Pair;
import org.slf4j.Logger;
diff --cc iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java
index b4fec9efc5e,e313c6d93a0..01969873a7f
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java
@@@ -93,7 -91,7 +93,8 @@@ import org.apache.iotdb.db.schemaengine
import org.apache.iotdb.rpc.RpcUtils;
import org.apache.iotdb.rpc.TSStatusCode;
+ import org.apache.tsfile.annotations.TableModel;
+import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.utils.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java
index c5b91f4dcdd,36c8470af2c..7d06d1f7e32
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java
@@@ -170,8 -173,11 +175,16 @@@ public class AlignedWritableMemChunkGro
return memChunkGroup;
}
+ @Override
+ public void checkDataType(InsertNode node) throws DataTypeInconsistentException {
+ memChunk.checkDataType(node);
+ }
++
+ protected static AlignedWritableMemChunkGroup deserializeSingleTVListMemChunks(
+ DataInputStream stream, boolean isTableModel) throws IOException {
+ AlignedWritableMemChunkGroup memChunkGroup = new AlignedWritableMemChunkGroup();
+ memChunkGroup.memChunk =
+ AlignedWritableMemChunk.deserializeSingleTVListMemChunks(stream, isTableModel);
+ return memChunkGroup;
+ }
}
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java
index a37475e2e0b,382573e7450..a61407d4e48
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java
@@@ -60,7 -58,7 +60,9 @@@ public interface IWritableMemChunkGrou
long getMeasurementSize(String measurement);
+ IWritableMemChunk getWritableMemChunk(String measurement);
+
long getMaxTime();
+
+ void checkDataType(InsertNode node) throws DataTypeInconsistentException;
}
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java
index deb73559b19,e47debd9d73..824b5c7a958
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java
@@@ -197,19 -202,15 +205,31 @@@ public class WritableMemChunkGroup impl
return memChunkGroup;
}
+ @Override
+ public void checkDataType(InsertNode node) throws DataTypeInconsistentException {
+ for (MeasurementSchema incomingSchema : node.getMeasurementSchemas()) {
+ if (incomingSchema == null) {
+ continue;
+ }
+
+ IWritableMemChunk memChunk = memChunkMap.get(incomingSchema.getMeasurementName());
+ if (memChunk != null
+ && memChunk.getWorkingTVList().getDataType() != incomingSchema.getType()) {
+ throw new DataTypeInconsistentException(
+ memChunk.getWorkingTVList().getDataType(), incomingSchema.getType());
+ }
+ }
+ }
++
+ public static WritableMemChunkGroup deserializeSingleTVListMemChunks(DataInputStream stream)
+ throws IOException {
+ WritableMemChunkGroup memChunkGroup = new WritableMemChunkGroup();
+ int memChunkMapSize = stream.readInt();
+ for (int i = 0; i < memChunkMapSize; ++i) {
+ String measurement = ReadWriteIOUtils.readString(stream);
+ IWritableMemChunk memChunk = WritableMemChunk.deserializeSingleTVListMemChunks(stream);
+ memChunkGroup.memChunkMap.put(measurement, memChunk);
+ }
+ return memChunkGroup;
+ }
}