You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2023/04/25 07:20:15 UTC
[iotdb] branch native_raft updated (eec4e1d22b -> 718d6bc869)
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a change to branch native_raft
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from eec4e1d22b fix close order
add e3e83fe8aa [IOTDB-5788] Built-in pipe plug-in management mechanism (#9680)
add 8b52e79bb3 [IOTDB-5791] Construct Cluster-LoadPublisher-Thread and IClusterStatusSubscriber (#9645)
add 22b04f1bc8 Remove check of closed state in ISink
add 6767a142ca Update Programming-Go-Native-API.md (#9688)
add 6a8d009a72 Remove unecessary synchronized on decrementCnt of SinkListener
add d92ea6e77c [IOTDB-5567] improve show cluster slots information (#9665)
add e8bf6ef1a8 [IOTDB-5810] Result handle in memory pool is not be cleaned properly (#9685)
add 104a8bf26f Fix concurrent state change bug in QueryStateMachine
add 969995276b Combine DataNodeSchemaCache of Template and Non-Template Scenarios (#9687)
add ce843c2c06 [IOTDB-5779] PipeConnector reuse strategy based on reference count mechanism (#9629)
add 4020214423 [IoTDB-5721] Streaming query DataPartition and Schema while loading TsFile (#9684)
add 29e7e1a851 [IOTDB-5662] Fix BufferedUnderflowException occurs in inner space compaction (#9322)
new 718d6bc869 Merge branch 'master' into native_raft
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 28 ++-
.../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 8 +-
.../consensus/request/ConfigPhysicalPlan.java | 6 +-
.../consensus/request/ConfigPhysicalPlanType.java | 2 +
...lotListPlan.java => CountTimeSlotListPlan.java} | 52 ++--
.../read/partition/GetSeriesSlotListPlan.java | 18 +-
.../read/partition/GetTimeSlotListPlan.java | 44 +++-
.../request/read/region/GetRegionIdPlan.java | 46 ++--
.../CountTimeSlotListResp.java} | 23 +-
.../response/partition/GetRegionIdResp.java | 2 +-
.../response/partition/GetSeriesSlotListResp.java | 2 +-
.../response/partition/GetTimeSlotListResp.java | 2 +-
.../confignode/manager/ClusterSchemaManager.java | 15 --
.../iotdb/confignode/manager/ConfigManager.java | 22 +-
.../apache/iotdb/confignode/manager/IManager.java | 35 ++-
.../manager/consensus/ConsensusManager.java | 12 +-
.../iotdb/confignode/manager/load/LoadManager.java | 8 +-
.../manager/load/balancer/RouteBalancer.java | 21 +-
.../load/balancer/router/RegionRouteMap.java | 5 +
.../confignode/manager/load/cache/LoadCache.java | 15 +-
.../load/cache/region/RegionGroupCache.java | 4 +
.../manager/load/service/StatisticsService.java | 115 +++++----
.../load/subscriber/IClusterStatusSubscriber.java | 19 +-
.../manager/load/subscriber/RouteChangeEvent.java | 74 ++++++
.../subscriber/StatisticsChangeEvent.java} | 26 +-
.../iotdb/confignode/manager/observer/IEvent.java | 21 --
.../manager/partition/PartitionManager.java | 60 +++--
.../manager/pipe/PipePluginCoordinator.java | 2 +-
.../persistence/executor/ConfigPlanExecutor.java | 3 +
.../partition/DatabasePartitionTable.java | 23 +-
.../persistence/partition/PartitionInfo.java | 128 ++++++++--
.../persistence/pipe/PipePluginInfo.java | 37 +--
.../impl/pipe/plugin/DropPipePluginProcedure.java | 6 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 48 ++--
.../request/ConfigPhysicalPlanSerDeTest.java | 19 +-
.../iotdb/confignode/persistence/PipeInfoTest.java | 2 +-
.../pipe/plugin/CreatePipePluginProcedureTest.java | 2 +-
docs/UserGuide/Cluster/Cluster-Maintenance.md | 176 ++++++++++---
docs/zh/UserGuide/API/Programming-Go-Native-API.md | 1 +
docs/zh/UserGuide/Cluster/Cluster-Maintenance.md | 177 ++++++++++---
.../iotdb/it/env/cluster/AbstractNodeWrapper.java | 1 +
.../it/partition/IoTDBPartitionGetterIT.java | 96 +++-----
.../resources/conf/iotdb-common.properties | 5 +
.../apache/iotdb/commons/conf/CommonConfig.java | 10 +
.../iotdb/commons/conf/CommonDescriptor.java | 8 +
.../commons/partition/DataPartitionTable.java | 45 +++-
.../commons/partition/SchemaPartitionTable.java | 10 +
.../commons/partition/SeriesPartitionTable.java | 32 +--
.../pipe/plugin/builtin/BuiltinPipePlugin.java | 55 +++++
.../builtin/connector/DoNothingConnector.java | 72 ++++++
.../builtin/processor/DoNothingProcessor.java | 68 +++++
.../meta/ConfigNodePipePluginMetaKeeper.java | 25 +-
.../plugin/meta/DataNodePipePluginMetaKeeper.java | 28 ++-
.../commons/pipe/plugin/meta/PipePluginMeta.java | 65 +++--
.../pipe/plugin/meta/PipePluginMetaKeeper.java | 62 ++++-
.../commons/pipe/task/meta/PipeStaticMeta.java | 24 +-
.../apache/iotdb/db/client/ConfigNodeClient.java | 22 ++
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 6 +
.../readchunk/SingleSeriesCompactionExecutor.java | 38 ++-
.../db/metadata/cache/DataNodeSchemaCache.java | 205 +++++----------
...he.java => DeviceUsingTemplateSchemaCache.java} | 45 +---
...SchemaCache.java => TimeSeriesSchemaCache.java} | 44 +---
.../db/mpp/common/header/ColumnHeaderConstant.java | 13 +-
.../db/mpp/common/header/DatasetHeaderFactory.java | 4 +
.../iotdb/db/mpp/execution/QueryStateMachine.java | 78 +++---
.../execution/exchange/MPPDataExchangeManager.java | 8 +-
.../execution/exchange/sink/LocalSinkChannel.java | 7 +
.../execution/exchange/sink/ShuffleSinkHandle.java | 7 +-
.../execution/exchange/source/SourceHandle.java | 25 +-
.../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 68 ++---
.../plan/analyze/schema/ClusterSchemaFetcher.java | 56 +----
.../plan/analyze/schema/NormalSchemaFetcher.java | 5 +-
.../plan/analyze/schema/TemplateSchemaFetcher.java | 10 +-
.../db/mpp/plan/execution/QueryExecution.java | 27 +-
.../plan/execution/config/ConfigTaskVisitor.java | 7 +
.../config/executor/ClusterConfigTaskExecutor.java | 74 ++++--
.../config/executor/IConfigTaskExecutor.java | 4 +
...egionIdTask.java => CountTimeSlotListTask.java} | 33 +--
.../config/metadata/GetTimeSlotListTask.java | 9 +-
.../config/metadata/ShowPipePluginsTask.java | 18 +-
.../iotdb/db/mpp/plan/parser/ASTVisitor.java | 86 ++++---
.../plan/node/load/LoadSingleTsFileNode.java | 59 ++---
.../planner/plan/node/load/LoadTsFileNode.java | 16 +-
.../scheduler/load/LoadTsFileDispatcherImpl.java | 20 +-
.../plan/scheduler/load/LoadTsFileScheduler.java | 201 +++++++++++----
.../db/mpp/plan/statement/StatementVisitor.java | 6 +
...tement.java => CountTimeSlotListStatement.java} | 59 +++--
.../statement/metadata/GetRegionIdStatement.java | 54 ++--
.../metadata/GetSeriesSlotListStatement.java | 20 +-
.../metadata/GetTimeSlotListStatement.java | 46 +++-
.../org/apache/iotdb/db/pipe/agent/PipeAgent.java | 9 +-
.../db/pipe/agent/plugin/PipePluginAgent.java | 67 +++--
.../db/pipe/agent/runtime/PipeRuntimeAgent.java | 17 +-
.../iotdb/db/pipe/agent/task/PipeTaskAgent.java | 19 +-
.../db/pipe/agent/task/PipeTaskRegionAgent.java | 22 --
.../core/connector/PipeConnectorContainer.java | 22 --
.../pipe/core/connector/PipeConnectorManager.java | 22 --
.../connector/PipeConnectorSubtaskLifeCycle.java | 98 ++++++++
.../connector/PipeConnectorSubtaskManager.java | 107 ++++++++
.../execution/executor/PipeSubtaskExecutor.java | 12 +-
.../pipe/task/callable/PipeConnectorSubtask.java | 37 ---
.../pipe/task/callable/PipeProcessorSubtask.java | 37 ---
.../db/pipe/task/stage/PipeTaskCollectorStage.java | 41 ++-
.../db/pipe/task/stage/PipeTaskConnectorStage.java | 109 +++++++-
.../db/pipe/task/stage/PipeTaskProcessorStage.java | 38 ++-
.../iotdb/db/pipe/task/stage/PipeTaskStage.java | 34 +--
.../task/{callable => subtask}/DecoratingLock.java | 2 +-
.../{callable => subtask}/PipeAssignerSubtask.java | 7 +-
.../subtask/PipeConnectorSubtask.java} | 44 +++-
.../subtask/PipeProcessorSubtask.java} | 38 ++-
.../task/{callable => subtask}/PipeSubtask.java | 8 +-
.../java/org/apache/iotdb/db/service/DataNode.java | 6 +
.../impl/DataNodeInternalRPCServiceImpl.java | 8 -
.../ReadChunkCompactionPerformerNoAlignedTest.java | 274 +++++++++++++++++++++
.../utils/CompactionFileGeneratorUtils.java | 60 +++++
.../db/mpp/execution/QueryStateMachineTest.java | 2 +
.../executor/PipeAssignerSubtaskExecutorTest.java | 2 +-
.../executor/PipeConnectorSubtaskExecutorTest.java | 6 +-
.../executor/PipeProcessorSubtaskExecutorTest.java | 12 +-
.../executor/PipeSubtaskExecutorTest.java | 2 +-
.../src/main/thrift/confignode.thrift | 36 ++-
121 files changed, 3030 insertions(+), 1463 deletions(-)
copy confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/partition/{GetTimeSlotListPlan.java => CountTimeSlotListPlan.java} (66%)
copy confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/{database/CountDatabaseResp.java => partition/CountTimeSlotListResp.java} (73%)
copy example/tsfile/src/main/java/org/apache/iotdb/tsfile/Constant.java => confignode/src/main/java/org/apache/iotdb/confignode/manager/load/subscriber/IClusterStatusSubscriber.java (66%)
create mode 100644 confignode/src/main/java/org/apache/iotdb/confignode/manager/load/subscriber/RouteChangeEvent.java
rename confignode/src/main/java/org/apache/iotdb/confignode/manager/{observer/NodeStatisticsEvent.java => load/subscriber/StatisticsChangeEvent.java} (51%)
delete mode 100644 confignode/src/main/java/org/apache/iotdb/confignode/manager/observer/IEvent.java
create mode 100644 node-commons/src/main/java/org/apache/iotdb/commons/pipe/plugin/builtin/BuiltinPipePlugin.java
create mode 100644 node-commons/src/main/java/org/apache/iotdb/commons/pipe/plugin/builtin/connector/DoNothingConnector.java
create mode 100644 node-commons/src/main/java/org/apache/iotdb/commons/pipe/plugin/builtin/processor/DoNothingProcessor.java
rename server/src/main/java/org/apache/iotdb/db/metadata/cache/{DataNodeTemplateSchemaCache.java => DeviceUsingTemplateSchemaCache.java} (81%)
copy server/src/main/java/org/apache/iotdb/db/metadata/cache/{DataNodeSchemaCache.java => TimeSeriesSchemaCache.java} (87%)
copy server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/metadata/{GetRegionIdTask.java => CountTimeSlotListTask.java} (70%)
copy server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/{GetTimeSlotListStatement.java => CountTimeSlotListStatement.java} (66%)
delete mode 100644 server/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeTaskRegionAgent.java
delete mode 100644 server/src/main/java/org/apache/iotdb/db/pipe/core/connector/PipeConnectorContainer.java
delete mode 100644 server/src/main/java/org/apache/iotdb/db/pipe/core/connector/PipeConnectorManager.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/pipe/core/connector/PipeConnectorSubtaskLifeCycle.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/pipe/core/connector/PipeConnectorSubtaskManager.java
delete mode 100644 server/src/main/java/org/apache/iotdb/db/pipe/task/callable/PipeConnectorSubtask.java
delete mode 100644 server/src/main/java/org/apache/iotdb/db/pipe/task/callable/PipeProcessorSubtask.java
rename server/src/main/java/org/apache/iotdb/db/pipe/task/{callable => subtask}/DecoratingLock.java (96%)
rename server/src/main/java/org/apache/iotdb/db/pipe/task/{callable => subtask}/PipeAssignerSubtask.java (91%)
rename server/src/main/java/org/apache/iotdb/db/pipe/{core/connector/PipeConnectorPluginRuntimeWrapper.java => task/subtask/PipeConnectorSubtask.java} (64%)
rename server/src/main/java/org/apache/iotdb/db/pipe/{core/processor/PipeProcessorPluginRuntimeWrapper.java => task/subtask/PipeProcessorSubtask.java} (71%)
rename server/src/main/java/org/apache/iotdb/db/pipe/task/{callable => subtask}/PipeSubtask.java (95%)
[iotdb] 01/01: Merge branch 'master' into native_raft
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch native_raft
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 718d6bc869bdb7e393863199099c106dba1375cd
Merge: eec4e1d22b 29e7e1a851
Author: Tian Jiang <jt...@163.com>
AuthorDate: Tue Apr 25 15:22:31 2023 +0800
Merge branch 'master' into native_raft
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 28 ++-
.../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 8 +-
.../consensus/request/ConfigPhysicalPlan.java | 6 +-
.../consensus/request/ConfigPhysicalPlanType.java | 2 +
...lotListPlan.java => CountTimeSlotListPlan.java} | 52 ++--
.../read/partition/GetSeriesSlotListPlan.java | 18 +-
.../read/partition/GetTimeSlotListPlan.java | 44 +++-
.../request/read/region/GetRegionIdPlan.java | 46 ++--
...egionIdResp.java => CountTimeSlotListResp.java} | 19 +-
.../response/partition/GetRegionIdResp.java | 2 +-
.../response/partition/GetSeriesSlotListResp.java | 2 +-
.../response/partition/GetTimeSlotListResp.java | 2 +-
.../confignode/manager/ClusterSchemaManager.java | 15 --
.../iotdb/confignode/manager/ConfigManager.java | 22 +-
.../apache/iotdb/confignode/manager/IManager.java | 35 ++-
.../manager/consensus/ConsensusManager.java | 12 +-
.../iotdb/confignode/manager/load/LoadManager.java | 8 +-
.../manager/load/balancer/RouteBalancer.java | 21 +-
.../load/balancer/router/RegionRouteMap.java | 5 +
.../confignode/manager/load/cache/LoadCache.java | 15 +-
.../load/cache/region/RegionGroupCache.java | 4 +
.../manager/load/service/StatisticsService.java | 115 +++++----
.../load/subscriber/IClusterStatusSubscriber.java | 19 +-
.../manager/load/subscriber/RouteChangeEvent.java | 74 ++++++
.../subscriber/StatisticsChangeEvent.java} | 26 +-
.../iotdb/confignode/manager/observer/IEvent.java | 21 --
.../manager/partition/PartitionManager.java | 60 +++--
.../manager/pipe/PipePluginCoordinator.java | 2 +-
.../persistence/executor/ConfigPlanExecutor.java | 3 +
.../partition/DatabasePartitionTable.java | 23 +-
.../persistence/partition/PartitionInfo.java | 128 ++++++++--
.../persistence/pipe/PipePluginInfo.java | 37 +--
.../impl/pipe/plugin/DropPipePluginProcedure.java | 6 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 48 ++--
.../request/ConfigPhysicalPlanSerDeTest.java | 19 +-
.../iotdb/confignode/persistence/PipeInfoTest.java | 2 +-
.../pipe/plugin/CreatePipePluginProcedureTest.java | 2 +-
docs/UserGuide/Cluster/Cluster-Maintenance.md | 176 ++++++++++---
docs/zh/UserGuide/API/Programming-Go-Native-API.md | 1 +
docs/zh/UserGuide/Cluster/Cluster-Maintenance.md | 177 ++++++++++---
.../iotdb/it/env/cluster/AbstractNodeWrapper.java | 1 +
.../it/partition/IoTDBPartitionGetterIT.java | 96 +++-----
.../resources/conf/iotdb-common.properties | 5 +
.../apache/iotdb/commons/conf/CommonConfig.java | 10 +
.../iotdb/commons/conf/CommonDescriptor.java | 8 +
.../commons/partition/DataPartitionTable.java | 45 +++-
.../commons/partition/SchemaPartitionTable.java | 10 +
.../commons/partition/SeriesPartitionTable.java | 32 +--
.../pipe/plugin/builtin/BuiltinPipePlugin.java | 55 +++++
.../builtin/connector/DoNothingConnector.java | 72 ++++++
.../builtin/processor/DoNothingProcessor.java | 68 +++++
.../meta/ConfigNodePipePluginMetaKeeper.java | 25 +-
.../plugin/meta/DataNodePipePluginMetaKeeper.java | 28 ++-
.../commons/pipe/plugin/meta/PipePluginMeta.java | 65 +++--
.../pipe/plugin/meta/PipePluginMetaKeeper.java | 62 ++++-
.../commons/pipe/task/meta/PipeStaticMeta.java | 24 +-
.../apache/iotdb/db/client/ConfigNodeClient.java | 22 ++
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 6 +
.../readchunk/SingleSeriesCompactionExecutor.java | 38 ++-
.../db/metadata/cache/DataNodeSchemaCache.java | 205 +++++----------
...he.java => DeviceUsingTemplateSchemaCache.java} | 45 +---
...SchemaCache.java => TimeSeriesSchemaCache.java} | 44 +---
.../db/mpp/common/header/ColumnHeaderConstant.java | 13 +-
.../db/mpp/common/header/DatasetHeaderFactory.java | 4 +
.../iotdb/db/mpp/execution/QueryStateMachine.java | 78 +++---
.../execution/exchange/MPPDataExchangeManager.java | 8 +-
.../execution/exchange/sink/LocalSinkChannel.java | 7 +
.../execution/exchange/sink/ShuffleSinkHandle.java | 7 +-
.../execution/exchange/source/SourceHandle.java | 25 +-
.../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 68 ++---
.../plan/analyze/schema/ClusterSchemaFetcher.java | 56 +----
.../plan/analyze/schema/NormalSchemaFetcher.java | 5 +-
.../plan/analyze/schema/TemplateSchemaFetcher.java | 10 +-
.../db/mpp/plan/execution/QueryExecution.java | 27 +-
.../plan/execution/config/ConfigTaskVisitor.java | 7 +
.../config/executor/ClusterConfigTaskExecutor.java | 74 ++++--
.../config/executor/IConfigTaskExecutor.java | 4 +
...lotListTask.java => CountTimeSlotListTask.java} | 38 +--
.../config/metadata/GetTimeSlotListTask.java | 9 +-
.../config/metadata/ShowPipePluginsTask.java | 18 +-
.../iotdb/db/mpp/plan/parser/ASTVisitor.java | 86 ++++---
.../plan/node/load/LoadSingleTsFileNode.java | 59 ++---
.../planner/plan/node/load/LoadTsFileNode.java | 16 +-
.../scheduler/load/LoadTsFileDispatcherImpl.java | 20 +-
.../plan/scheduler/load/LoadTsFileScheduler.java | 201 +++++++++++----
.../db/mpp/plan/statement/StatementVisitor.java | 6 +
...tement.java => CountTimeSlotListStatement.java} | 59 +++--
.../statement/metadata/GetRegionIdStatement.java | 54 ++--
.../metadata/GetSeriesSlotListStatement.java | 20 +-
.../metadata/GetTimeSlotListStatement.java | 46 +++-
.../org/apache/iotdb/db/pipe/agent/PipeAgent.java | 9 +-
.../db/pipe/agent/plugin/PipePluginAgent.java | 67 +++--
.../db/pipe/agent/runtime/PipeRuntimeAgent.java | 17 +-
.../iotdb/db/pipe/agent/task/PipeTaskAgent.java | 19 +-
.../db/pipe/agent/task/PipeTaskRegionAgent.java | 22 --
.../core/connector/PipeConnectorContainer.java | 22 --
.../pipe/core/connector/PipeConnectorManager.java | 22 --
.../connector/PipeConnectorSubtaskLifeCycle.java | 98 ++++++++
.../connector/PipeConnectorSubtaskManager.java | 107 ++++++++
.../execution/executor/PipeSubtaskExecutor.java | 12 +-
.../pipe/task/callable/PipeConnectorSubtask.java | 37 ---
.../pipe/task/callable/PipeProcessorSubtask.java | 37 ---
.../db/pipe/task/stage/PipeTaskCollectorStage.java | 41 ++-
.../db/pipe/task/stage/PipeTaskConnectorStage.java | 109 +++++++-
.../db/pipe/task/stage/PipeTaskProcessorStage.java | 38 ++-
.../iotdb/db/pipe/task/stage/PipeTaskStage.java | 34 +--
.../task/{callable => subtask}/DecoratingLock.java | 2 +-
.../{callable => subtask}/PipeAssignerSubtask.java | 7 +-
.../subtask/PipeConnectorSubtask.java} | 44 +++-
.../subtask/PipeProcessorSubtask.java} | 38 ++-
.../task/{callable => subtask}/PipeSubtask.java | 8 +-
.../java/org/apache/iotdb/db/service/DataNode.java | 6 +
.../impl/DataNodeInternalRPCServiceImpl.java | 8 -
.../ReadChunkCompactionPerformerNoAlignedTest.java | 274 +++++++++++++++++++++
.../utils/CompactionFileGeneratorUtils.java | 60 +++++
.../db/mpp/execution/QueryStateMachineTest.java | 2 +
.../executor/PipeAssignerSubtaskExecutorTest.java | 2 +-
.../executor/PipeConnectorSubtaskExecutorTest.java | 6 +-
.../executor/PipeProcessorSubtaskExecutorTest.java | 12 +-
.../executor/PipeSubtaskExecutorTest.java | 2 +-
.../src/main/thrift/confignode.thrift | 36 ++-
121 files changed, 3027 insertions(+), 1467 deletions(-)