You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2025/04/29 03:44:04 UTC
(iotdb) 01/04: Merge branch 'master' into preallocate_array_list
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch preallocate_array_list
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit f890cf073d0eb5a5916fa4abdd67ce9c4f36469b
Merge: eb46237e8b1 a0152af2b1d
Author: Tian Jiang <jt...@163.com>
AuthorDate: Tue Apr 29 10:20:13 2025 +0800
Merge branch 'master' into preallocate_array_list
distribution/src/assembly/ainode.xml | 6 +
distribution/src/assembly/all.xml | 50 +-
distribution/src/assembly/cli.xml | 17 +-
distribution/src/assembly/confignode.xml | 41 +-
distribution/src/assembly/datanode.xml | 45 +-
.../DockerCompose/docker-compose-host-3c3d.yml | 2 +
.../DockerCompose/docker-compose-standalone.yml | 4 +-
.../main/DockerCompose/replace-conf-from-env.sh | 11 +-
docker/src/main/Dockerfile-1c1d | 2 +-
.../TableModelSubscriptionSessionExample.java | 18 +-
.../iotdb/udf/table/ExcludeColumnExample.java | 11 +-
.../org/apache/iotdb/udf/table/RepeatExample.java | 21 +-
.../org/apache/iotdb/udf/table/SplitExample.java | 27 +-
integration-test/src/assembly/mpp-test.xml | 32 +-
.../example/relational/MyErrorTableFunction.java | 15 +-
.../udf/example/relational/MyExcludeColumn.java | 11 +-
.../udf/example/relational/MyRepeatWithIndex.java | 17 +-
.../example/relational/MyRepeatWithoutIndex.java | 17 +-
.../{MyExcludeColumn.java => MySelectColumn.java} | 21 +-
.../db/query/udf/example/relational/MySplit.java | 27 +-
.../it/env/cluster/config/MppConfigNodeConfig.java | 12 +
.../it/env/cluster/config/MppDataNodeConfig.java | 12 +
.../iotdb/it/env/cluster/env/AbstractEnv.java | 13 +-
.../env/remote/config/RemoteConfigNodeConfig.java | 10 +
.../it/env/remote/config/RemoteDataNodeConfig.java | 10 +
.../iotdb/it/env/remote/env/RemoteServerEnv.java | 13 +-
.../java/org/apache/iotdb/itbase/env/BaseEnv.java | 11 +-
.../apache/iotdb/itbase/env/ConfigNodeConfig.java | 5 +
.../apache/iotdb/itbase/env/DataNodeConfig.java | 4 +
.../apache/iotdb/cli/it/StartClientScriptIT.java | 4 +-
.../org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java | 54 ++
.../org/apache/iotdb/db/it/IoTDBRestartIT.java | 2 -
.../apache/iotdb/db/it/metric/IoTDBMetricIT.java | 92 ++-
.../org/apache/iotdb/db/it/utils/TestUtils.java | 29 +-
.../manual/basic/IoTDBPipeWithLoadIT.java | 107 +++
.../relational/it/db/it/IoTDBWindowTVFIT.java | 23 +-
.../db/it/udf/IoTDBUserDefinedTableFunctionIT.java | 16 +
.../it/query/recent/IoTDBTableAggregationIT.java | 69 ++
.../it/local/IoTDBSubscriptionPermissionIT.java | 339 ++++++++++
.../apache/iotdb/tools/it/ExportDataTestIT.java | 6 +-
.../apache/iotdb/tools/it/ExportSchemaTestIT.java | 10 +-
.../apache/iotdb/tools/it/ExportTsFileTestIT.java | 87 +--
.../apache/iotdb/tools/it/ImportDataTestIT.java | 2 +-
.../apache/iotdb/tools/it/ImportSchemaTestIT.java | 22 +-
.../api/relational/EmptyTableFunctionHandle.java | 18 +-
.../iotdb/udf/api/relational/TableFunction.java | 10 +-
.../relational/table/MapTableFunctionHandle.java | 186 ++++++
.../relational/table/TableFunctionAnalysis.java | 21 +-
.../api/relational/table/TableFunctionHandle.java | 27 +-
.../table/argument/ScalarArgumentChecker.java | 17 +-
.../processor/TableFunctionDataProcessor.java | 12 +-
.../ScalarParameterSpecification.java | 23 +-
iotdb-client/cli/src/assembly/cli.xml | 10 +
.../src/assembly/resources/tools/load-tsfile.bat | 50 --
.../src/assembly/resources/tools/load-tsfile.sh | 60 --
.../src/main/java/org/apache/iotdb/cli/Cli.java | 1 +
.../org/apache/iotdb/tool/common/Constants.java | 2 +-
.../org/apache/iotdb/tool/common/OptionsUtil.java | 5 +-
.../apache/iotdb/tool/data/AbstractDataTool.java | 12 +
.../org/apache/iotdb/tool/data/ImportDataTree.java | 57 +-
.../org/apache/iotdb/tool/tsfile/ExportTsFile.java | 5 +
.../iotdb/tool/tsfile/ImportTsFileRemotely.java | 6 +-
.../subscription/AbstractSubscriptionTsFile.java | 1 +
.../subscription/SubscriptionTableTsFile.java | 28 +-
.../subscription/SubscriptionTreeTsFile.java | 29 +-
iotdb-client/client-cpp/src/main/Session.cpp | 2 +-
.../client-cpp/src/main/SessionConnection.h | 2 +-
iotdb-client/client-py/README.md | 4 +-
iotdb-client/client-py/iotdb/Session.py | 18 +-
iotdb-client/client-py/iotdb/SessionPool.py | 4 +-
iotdb-client/client-py/iotdb/utils/Field.py | 29 +-
.../client-py/iotdb/utils/SessionDataSet.py | 19 +-
.../client-py/iotdb/utils/iotdb_rpc_dataset.py | 38 +-
iotdb-client/client-py/iotdb/utils/rpc_utils.py | 41 ++
iotdb-client/client-py/requirements.txt | 1 +
iotdb-client/client-py/resources/pyproject.toml | 3 +-
.../session_aligned_timeseries_example.py | 4 +-
iotdb-client/client-py/session_example.py | 4 +-
iotdb-client/client-py/session_pool_example.py | 2 +-
iotdb-client/client-py/session_ssl_example.py | 4 +-
.../client-py/table_model_session_example.py | 4 +-
.../integration/tablet_performance_comparison.py | 4 +-
.../tests/integration/test_new_data_types.py | 12 +-
.../tests/integration/test_tablemodel_query.py | 22 +-
.../org/apache/iotdb/jdbc/IoTDBConnection.java | 2 -
.../apache/iotdb/jdbc/IoTDBDatabaseMetadata.java | 4 +-
.../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 134 ++--
.../rpc/subscription/annotation/TableModel.java | 2 +-
.../rpc/subscription/config/ConsumerConfig.java | 8 +
.../response/PipeSubscribeHeartbeatResp.java | 48 +-
.../apache/iotdb/session/SessionConnection.java | 148 ++++-
.../SubscriptionSessionConnection.java | 32 -
.../base/AbstractSubscriptionConsumer.java | 2 +-
.../base/AbstractSubscriptionConsumerBuilder.java | 20 -
.../base/AbstractSubscriptionProvider.java | 13 +-
.../base/AbstractSubscriptionProviders.java | 6 +-
.../SubscriptionTablePullConsumerBuilder.java | 3 +-
.../SubscriptionTablePushConsumerBuilder.java | 3 +-
.../tree/SubscriptionTreePullConsumerBuilder.java | 3 +-
.../tree/SubscriptionTreePushConsumerBuilder.java | 3 +-
.../payload/SubscriptionSessionDataSet.java | 93 ++-
iotdb-core/ainode/ainode.xml | 27 +
iotdb-core/confignode/src/assembly/confignode.xml | 28 +-
.../iotdb/confignode/manager/ConfigManager.java | 2 +-
.../iotdb/confignode/manager/ProcedureManager.java | 6 +-
.../region/GreedyCopySetRegionGroupAllocator.java | 356 +++++++++-
.../region/GreedyRegionGroupAllocator.java | 13 +
.../balancer/region/IRegionGroupAllocator.java | 21 +
.../PartiteGraphPlacementRegionGroupAllocator.java | 13 +
.../persistence/schema/ClusterSchemaInfo.java | 15 +-
.../procedure/env/RemoveDataNodeHandler.java | 186 ++++++
.../procedure/impl/StateMachineProcedure.java | 11 +-
.../impl/node/RemoveDataNodesProcedure.java | 6 +-
.../consumer/CreateConsumerProcedure.java | 4 +-
.../subscription/CreateSubscriptionProcedure.java | 21 +-
.../GreedyCopySetRemoveNodeReplicaSelectTest.java | 342 ++++++++++
.../pipe/consensuspipe/ConsensusPipeReceiver.java | 3 +-
iotdb-core/datanode/src/assembly/server.xml | 44 +-
.../apache/iotdb/db/conf/DataNodeMemoryConfig.java | 12 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 11 -
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 14 +-
.../schemaregion/SchemaExecutionVisitor.java | 15 +
.../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 26 -
.../task/builder/PipeDataNodeTaskBuilder.java | 51 +-
.../task/execution/PipeSubtaskExecutorManager.java | 7 +
.../subtask/connector/PipeConnectorSubtask.java | 13 +-
.../PipeRealtimePriorityBlockingQueue.java | 26 +-
.../IoTDBDataNodeCacheLeaderClientManager.java | 12 +-
.../batch/PipeTransferBatchReqBuilder.java | 41 +-
.../pipeconsensus/PipeConsensusAsyncConnector.java | 157 +++--
.../pipeconsensus/PipeConsensusSyncConnector.java | 3 -
.../PipeConsensusTransferBatchReqBuilder.java | 19 +-
.../async/IoTDBDataRegionAsyncConnector.java | 31 +-
.../deletion/persist/PageCacheDeletionBuffer.java | 12 +-
.../common/tablet/PipeRawTabletInsertionEvent.java | 5 +
.../event/common/terminate/PipeTerminateEvent.java | 6 +-
.../scan/TsFileInsertionEventScanParser.java | 5 +-
...ileInsertionEventTableParserTabletIterator.java | 16 +-
...oricalDataRegionTsFileAndDeletionExtractor.java | 30 +-
.../PipeRealtimeDataRegionHybridExtractor.java | 5 +-
.../realtime/assigner/PipeDataRegionAssigner.java | 5 +-
.../pipe/processor/twostage/combiner/Combiner.java | 5 +-
.../pipeconsensus/PipeConsensusReceiver.java | 327 +++++-----
.../pipeconsensus/PipeConsensusReceiverAgent.java | 61 +-
.../protocol/thrift/IoTDBDataNodeReceiver.java | 10 +-
.../db/pipe/resource/memory/PipeMemoryManager.java | 54 +-
.../pipe/resource/tsfile/PipeTsFileResource.java | 14 +-
.../resource/tsfile/PipeTsFileResourceManager.java | 2 +-
.../rest/table/v1/handler/ExceptionHandler.java | 4 +
.../protocol/rest/v1/handler/ExceptionHandler.java | 4 +
.../protocol/rest/v2/handler/ExceptionHandler.java | 4 +
.../process/function/TableFunctionOperator.java | 18 +-
.../relational/aggregation/AccumulatorFactory.java | 5 +
.../ApproxCountDistinctAccumulator.java | 265 ++++++++
.../source/relational/aggregation/HyperLogLog.java | 246 +++++++
.../aggregation/HyperLogLogStateFactory.java | 88 +++
.../GroupedApproxCountDistinctAccumulator.java | 314 +++++++++
.../aggregation/grouped/array/BinaryBigArray.java | 4 -
.../grouped/array/HyperLogLogBigArray.java | 84 +++
.../aggregation/grouped/array/MapBigArray.java | 4 -
.../aggregation/grouped/array/ObjectBigArray.java | 12 +-
.../iotdb/db/queryengine/plan/Coordinator.java | 4 -
.../plan/analyze/load/LoadTsFileAnalyzer.java | 66 +-
.../analyze/load/LoadTsFileTableSchemaCache.java | 78 ++-
.../analyze/schema/AutoCreateSchemaExecutor.java | 14 +-
.../analyze/schema/ClusterSchemaFetchExecutor.java | 9 +-
.../plan/analyze/schema/ClusterSchemaFetcher.java | 1 -
.../config/executor/ClusterConfigTaskExecutor.java | 4 +-
.../plan/planner/TableOperatorGenerator.java | 5 +-
.../queryengine/plan/planner/TreeModelPlanner.java | 2 -
.../plan/planner/plan/node/PlanGraphPrinter.java | 30 +-
.../plan/node/metadata/write/MeasurementGroup.java | 25 +-
.../relational/analyzer/StatementAnalyzer.java | 9 +
.../TableFunctionInvocationAnalysis.java | 8 +
.../relational/metadata/TableMetadataImpl.java | 15 +
.../plan/relational/planner/RelationPlanner.java | 2 +-
.../plan/relational/planner/TableModelPlanner.java | 9 -
.../rule/ImplementTableFunctionSource.java | 4 +-
.../rule/PruneTableFunctionProcessorColumns.java | 2 +-
.../PruneTableFunctionProcessorSourceColumns.java | 2 +-
.../relational/planner/node/TableFunctionNode.java | 57 +-
.../planner/node/TableFunctionProcessorNode.java | 51 +-
.../optimizations/UnaliasSymbolReferences.java | 6 +-
.../plan/relational/sql/ast/LoadTsFile.java | 10 +-
.../plan/relational/sql/parser/AstBuilder.java | 17 +-
.../plan/scheduler/AsyncPlanNodeSender.java | 21 +-
.../plan/scheduler/ClusterScheduler.java | 5 -
.../FailedFragmentInstanceWithStatus.java | 26 +-
.../scheduler/FragmentInstanceDispatcherImpl.java | 248 ++++---
.../db/schemaengine/table/DataNodeTableCache.java | 18 +-
.../iotdb/db/service/DataNodeShutdownHook.java | 4 +
.../iotdb/db/storageengine/StorageEngine.java | 1 +
.../performer/ICrossCompactionPerformer.java | 6 +
...rformer.java => IInnerCompactionPerformer.java} | 11 +-
.../execute/performer/ISeqCompactionPerformer.java | 2 +-
.../performer/IUnseqCompactionPerformer.java | 2 +-
.../performer/impl/FastCompactionPerformer.java | 15 +
.../impl/ReadChunkCompactionPerformer.java | 8 +
.../impl/ReadPointCompactionPerformer.java | 8 +
.../execute/task/AbstractCompactionTask.java | 10 +
.../execute/task/InnerSpaceCompactionTask.java | 18 +-
.../estimator/AbstractCompactionEstimator.java | 124 +++-
.../estimator/AbstractCrossSpaceEstimator.java | 6 +-
.../estimator/AbstractInnerSpaceEstimator.java | 5 +-
.../estimator/CompactionEstimateUtils.java | 91 ++-
.../selector/estimator/CompactionTaskInfo.java | 6 -
...taInfo.java => CompactionTaskMetadataInfo.java} | 12 +-
.../FastCompactionInnerCompactionEstimator.java | 44 +-
.../FastCrossSpaceCompactionEstimator.java | 29 +-
.../compaction/selector/estimator/FileInfo.java | 39 +-
.../ReadChunkInnerCompactionEstimator.java | 25 +-
.../RepairUnsortedFileCompactionEstimator.java | 21 +-
.../impl/NewSizeTieredCompactionSelector.java | 76 ++-
.../impl/RewriteCrossSpaceCompactionSelector.java | 2 +-
.../dataregion/wal/buffer/WALBuffer.java | 9 +-
.../dataregion/wal/io/WALInputStream.java | 2 +
.../receiver/SubscriptionReceiverV1.java | 47 +-
.../apache/iotdb/db/utils/ErrorHandlingUtils.java | 4 +-
.../apache/iotdb/db/utils/ModificationUtils.java | 23 +-
.../iotdb/db/utils/constant/SqlConstant.java | 2 +
.../db/queryengine/plan/function/Exclude.java | 11 +-
.../iotdb/db/queryengine/plan/function/Repeat.java | 22 +-
.../iotdb/db/queryengine/plan/function/Split.java | 27 +-
.../relational/analyzer/TableFunctionTest.java | 92 +--
.../assertions/TableFunctionProcessorMatcher.java | 178 +----
.../org/apache/iotdb/db/script/EnvScriptIT.java | 12 +-
.../utils/CompactionTaskMemCostEstimatorTest.java | 87 ++-
.../wal/compression/WALCompressionTest.java | 45 +-
iotdb-core/metrics/interface/pom.xml | 5 +
.../apache/iotdb/metrics/config/MetricConfig.java | 38 ++
.../metrics/config/MetricConfigDescriptor.java | 19 +
.../reporter/prometheus/PrometheusReporter.java | 62 +-
.../conf/iotdb-system.properties.template | 11 +
.../apache/iotdb/commons/conf/CommonConfig.java | 445 +++++++++++++
.../iotdb/commons/conf/CommonDescriptor.java | 486 +-------------
.../exception/QuerySchemaFetchFailedException.java | 19 +-
.../task/connection/BlockingPendingQueue.java | 13 +-
.../agent/task/execution/PipeSubtaskExecutor.java | 2 +-
.../agent/task/execution/PipeSubtaskScheduler.java | 15 +-
.../agent/task/progress/PipeEventCommitter.java | 28 +-
.../iotdb/commons/pipe/config/PipeConfig.java | 5 +-
.../config/PipeDescriptor.java} | 726 +++++----------------
.../config/constant/PipeConnectorConstant.java | 3 +
.../pipe/connector/protocol/IoTDBConnector.java | 10 +
.../commons/pipe/receiver/IoTDBFileReceiver.java | 18 +-
.../meta/consumer/ConsumerGroupMeta.java | 49 ++
.../subscription/meta/consumer/ConsumerMeta.java | 12 +
.../commons/subscription/meta/topic/TopicMeta.java | 9 +-
.../TableBuiltinAggregationFunction.java | 2 +
.../relational/tvf/CapacityTableFunction.java | 27 +-
.../relational/tvf/CumulateTableFunction.java | 40 +-
.../builtin/relational/tvf/HOPTableFunction.java | 44 +-
.../relational/tvf/SessionTableFunction.java | 24 +-
.../relational/tvf/TumbleTableFunction.java | 32 +-
.../relational/tvf/VariationTableFunction.java | 25 +-
.../apache/iotdb/commons/utils/StatusUtils.java | 2 +-
.../thrift-commons/src/main/thrift/common.thrift | 3 +-
library-udf/src/assembly/tools/register-UDF.bat | 110 ++--
pom.xml | 2 +-
.../resources => scripts}/conf/ainode-env.sh | 0
.../resources => scripts}/conf/confignode-env.sh | 21 +-
.../resources => scripts}/conf/datanode-env.sh | 24 +-
.../sbin => scripts/conf}/iotdb-common.sh | 0
.../conf => scripts/conf/windows}/ainode-env.bat | 11 +-
.../conf/windows}/confignode-env.bat | 2 +-
.../conf => scripts/conf/windows}/datanode-env.bat | 2 +-
.../sbin => scripts/sbin/cluster}/start-all.sh | 2 +-
.../sbin => scripts/sbin/cluster}/stop-all.sh | 2 +-
.../resources => scripts}/sbin/start-ainode.sh | 17 +-
.../resources => scripts}/sbin/start-cli.sh | 0
.../resources => scripts}/sbin/start-confignode.sh | 2 +-
.../resources => scripts}/sbin/start-datanode.sh | 2 +-
.../resources => scripts}/sbin/start-standalone.sh | 0
.../resources => scripts}/sbin/stop-ainode.sh | 0
.../resources => scripts}/sbin/stop-confignode.sh | 2 +-
.../resources => scripts}/sbin/stop-datanode.sh | 2 +-
.../resources => scripts}/sbin/stop-standalone.sh | 0
.../sbin => scripts/sbin/windows}/start-ainode.bat | 12 +-
.../sbin/windows/start-cli-table.bat | 4 +-
.../sbin => scripts/sbin/windows}/start-cli.bat | 2 +-
.../sbin/windows}/start-confignode.bat | 8 +-
.../sbin/windows}/start-datanode.bat | 14 +-
.../sbin/windows}/start-standalone.bat | 12 +-
.../sbin => scripts/sbin/windows}/stop-ainode.bat | 8 +-
.../sbin/windows}/stop-confignode.bat | 13 +-
.../sbin/windows}/stop-datanode.bat | 13 +-
.../sbin/windows}/stop-standalone.bat | 10 +-
.../resources => scripts}/tools/export-data.sh | 0
.../tools/export-tsfile.sh | 8 +-
.../resources => scripts}/tools/import-data.sh | 0
.../tools => scripts/tools/ops}/backup.sh | 2 +-
.../tools => scripts/tools/ops}/collect-info.sh | 2 +-
.../tools/ops}/daemon-confignode.sh | 2 +-
.../sbin => scripts/tools/ops}/daemon-datanode.sh | 2 +-
.../sbin => scripts/tools/ops}/destroy-all.sh | 18 +-
.../tools/ops}/destroy-confignode.sh | 2 +-
.../sbin => scripts/tools/ops}/destroy-datanode.sh | 2 +-
.../sbin => scripts/tools/ops}/health_check.sh | 4 +-
.../sbin => scripts/tools/ops}/remove-ainode.sh | 23 +-
.../tools/schema}/export-schema.sh | 2 +-
.../tools/schema}/import-schema.sh | 2 +-
.../tools/schema/print-pb-tree-file.sh | 2 +-
.../tools/schema/print-schema-log.sh | 2 +-
.../check-overlap-sequence-files-and-repair.sh | 2 +-
.../tools/tsfile/overlap-statistic-tool.sh | 2 +-
.../tools/tsfile/print-iotdb-data-dir.sh | 2 +-
.../tools/tsfile/print-tsfile-resource-files.sh | 2 +-
.../tools/tsfile/print-tsfile.sh | 2 +-
.../tools/tsfile/settle-tsfile.sh | 0
.../tools/tsfile/split-tsfile-tool.sh | 2 +-
.../tools/tsfile/validate-tsfile.sh | 2 +-
.../tools/windows}/export-data.bat | 2 +-
.../tools/windows/export-tsfile.bat | 10 +-
.../tools/windows}/import-data.bat | 2 +-
.../tools => scripts/tools/windows/ops}/backup.bat | 6 +-
.../tools/windows/ops}/collect-info.bat | 8 +-
.../tools/windows/ops}/destroy-all.bat | 8 +-
.../tools/windows/ops}/destroy-confignode.bat | 4 +-
.../tools/windows/ops}/destroy-datanode.bat | 4 +-
.../tools/windows/ops}/health_check.bat | 14 +-
.../tools/windows/ops}/remove-ainode.bat | 20 +-
.../tools/windows/schema}/export-schema.bat | 2 +-
.../tools/windows/schema}/import-schema.bat | 2 +-
.../tools/windows}/schema/print-pb-tree-file.bat | 2 +-
.../tools/windows}/schema/print-schema-log.bat | 2 +-
.../check-overlap-sequence-files-and-repair.bat | 2 +-
.../windows}/tsfile/overlap-statistic-tool.bat | 2 +-
.../tools/windows}/tsfile/print-iotdb-data-dir.bat | 2 +-
.../tsfile/print-tsfile-resource-files.bat | 2 +-
.../tools/windows}/tsfile/print-tsfile.bat | 2 +-
.../tools/windows}/tsfile/settle-tsfile.bat | 2 +-
.../tools/windows}/tsfile/split-tsfile-tool.bat | 2 +-
.../tools/windows}/tsfile/validate-tsfile.bat | 2 +-
333 files changed, 7106 insertions(+), 3241 deletions(-)