You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ro...@apache.org on 2023/08/14 12:15:14 UTC
[iotdb] branch multi-way-sync updated (90153702157 -> fea17764486)
This is an automated email from the ASF dual-hosted git repository.
rong pushed a change to branch multi-way-sync
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 90153702157 backup multyi-way-sync
add 1f321df57b7 [To Master] Fixed the prompt after CSV file import (#10824)
add 2e6d9527c04 [IOTDB-6105] Load: NPE when analyzing tsfile (#10821)
add ff3239bf829 [IOTDB-6106] Fixed the timeout parameter not working in thrift asyncClient (#10822)
add 29d414b2937 session module add UT (#10815)
add 031495a6301 Pipe: refactor connector packages and rename v1/v2 to sync/async (#10840)
add da7e66db404 fix parameter schemaRegionPerNode load error (#10841)
add 80427a00c5f Enhance the event notification mechanism of StatisticsService (#10829)
add 59a0dcf11a1 Refactoring DeleteOutdatedFileTask in WalNode (#10765)
add ccd73310fac fix inaccurate statistics for monitoring the number of unseq Tsfile on cross compaction (#10651)
add 47a14313621 add memory estimator on inner space compaction (#10688)
add 7dd7cf78956 Fix a concurrent bug when recover last flush time and compact (#10839)
add 3dc1490cfc8 [IOTDB-6108] Fix AlignedTVList memory calculation is imprecise (#10827)
add 11af323b79d Pipe: Remove parameters from iotdb-common.properties that users will not modify (#10851)
new fea17764486 Merge branch 'master' of github.com:apache/iotdb into multi-way-sync
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../it/env/cluster/config/MppCommonConfig.java | 6 +
.../env/cluster/config/MppSharedCommonConfig.java | 7 +
.../it/env/remote/config/RemoteCommonConfig.java | 4 +
.../org/apache/iotdb/itbase/env/CommonConfig.java | 2 +
.../it/partition/IoTDBPartitionDurableIT.java | 6 +-
.../main/java/org/apache/iotdb/tool/ImportCsv.java | 2 +-
iotdb-client/session/pom.xml | 5 +
.../apache/iotdb/session/pool/SessionPoolTest.java | 90 ++--
.../confignode/conf/ConfigNodeDescriptor.java | 2 +-
.../iotdb/confignode/manager/load/LoadManager.java | 8 +-
.../manager/load/balancer/RouteBalancer.java | 4 +-
.../manager/load/service/StatisticsService.java | 16 +-
.../iot/client/AsyncIoTConsensusServiceClient.java | 1 +
.../resources/conf/iotdb-datanode.properties | 2 +-
.../org/apache/iotdb/db/audit/AuditLogger.java | 2 +-
.../db/pipe/agent/receiver/PipeReceiverAgent.java | 5 +-
.../{v1 => payload/evolvable}/PipeRequestType.java | 2 +-
.../reponse/PipeTransferFilePieceResp.java | 2 +-
.../request/PipeTransferFilePieceReq.java | 6 +-
.../request/PipeTransferFileSealReq.java | 6 +-
.../request/PipeTransferHandshakeReq.java | 6 +-
.../request/PipeTransferInsertNodeReq.java | 6 +-
.../evolvable}/request/PipeTransferTabletReq.java | 6 +-
.../legacy}/DeletionPipeData.java | 6 +-
.../pipedata => payload/legacy}/PipeData.java | 4 +-
.../legacy}/TsFilePipeData.java | 6 +-
.../protocol/airgap/IoTDBAirGapConnector.java | 57 +++
.../legacy/IoTDBLegacyPipeConnector.java} | 20 +-
.../thrift}/IoTDBThriftConnector.java | 2 +-
.../IoTDBThriftConnectorRequestVersion.java | 3 +-
.../thrift/async/IoTDBThriftAsyncConnector.java} | 44 +-
...nsferInsertNodeTabletInsertionEventHandler.java | 6 +-
...PipeTransferRawTabletInsertionEventHandler.java | 6 +-
.../PipeTransferTabletInsertionEventHandler.java | 11 +-
.../PipeTransferTsFileInsertionEventHandler.java | 14 +-
.../thrift/sync/IoTDBThriftSyncConnector.java} | 47 +-
.../sync/IoTDBThriftSyncConnectorClient.java} | 6 +-
.../airgap/IoTDBAirGapReceiver.java} | 18 +-
.../legacy/IoTDBLegacyPipeReceiver.java} | 20 +-
.../legacy/loader/DeletionLoader.java | 2 +-
.../legacy/loader/ILoader.java | 2 +-
.../legacy/loader/TsFileLoader.java | 4 +-
.../thrift}/IoTDBThriftReceiver.java | 4 +-
.../thrift}/IoTDBThriftReceiverV1.java | 18 +-
.../connector/PipeConnectorSubtaskManager.java | 20 +-
.../protocol/thrift/impl/ClientRPCServiceImpl.java | 10 +-
.../queryengine/plan/analyze/AnalyzeVisitor.java | 397 +--------------
.../plan/analyze/LoadTsfileAnalyzer.java | 543 +++++++++++++++++++++
.../db/queryengine/plan/parser/ASTVisitor.java | 2 +-
.../plan/statement/crud/LoadTsFileStatement.java | 29 +-
.../dataregion/HashLastFlushTimeMap.java | 15 +-
.../exception/CompactionExceptionHandler.java | 11 +-
.../execute/task/AbstractCompactionTask.java | 2 +
.../execute/task/CrossSpaceCompactionTask.java | 24 +-
.../execute/task/InnerSpaceCompactionTask.java | 58 ++-
.../compaction/execute/utils/CompactionUtils.java | 53 ++
.../estimator/AbstractCompactionEstimator.java | 3 +-
.../estimator/AbstractInnerSpaceEstimator.java | 74 ++-
.../estimator/CompactionEstimateUtils.java | 96 ++++
.../FastCompactionInnerCompactionEstimator.java | 52 ++
.../compaction/selector/estimator/FileInfo.java | 48 ++
.../ReadChunkInnerCompactionEstimator.java | 59 +++
.../ReadPointCrossCompactionEstimator.java | 81 +--
.../dataregion/memtable/TsFileProcessor.java | 146 +++---
.../dataregion/tsfile/TsFileManager.java | 17 +
.../wal/checkpoint/CheckpointManager.java | 2 +-
.../storageengine/dataregion/wal/node/WALNode.java | 243 ++++++---
.../db/utils/datastructure/AlignedTVList.java | 46 ++
.../iotdb/db/pipe/connector/PipeConnectorTest.java | 24 +-
.../iotdb/db/pipe/connector/PipeReceiverTest.java | 6 +-
.../db/pipe/connector/PipeThriftRequestTest.java | 12 +-
.../compaction/CompactionSchedulerTest.java | 1 +
.../utils/CompactionTaskMemCostEstimatorTest.java | 94 ++++
.../dataregion/memtable/TsFileProcessorTest.java | 97 ++++
.../resources/conf/iotdb-common.properties | 82 +---
.../iotdb/commons/client/ClientPoolFactory.java | 2 +-
.../async/AsyncConfigNodeIServiceClient.java | 1 +
.../async/AsyncDataNodeInternalServiceClient.java | 1 +
.../AsyncDataNodeMPPDataExchangeServiceClient.java | 1 +
.../async/AsyncPipeDataTransferServiceClient.java | 1 +
.../iotdb/commons/concurrent/ThreadName.java | 3 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 23 +-
.../iotdb/commons/conf/CommonDescriptor.java | 8 +-
.../iotdb/commons/pipe/config/PipeConfig.java | 4 +-
.../pipe/plugin/builtin/BuiltinPipePlugin.java | 12 +-
...onnector.java => IoTDBLegacyPipeConnector.java} | 10 +-
...ectorV1.java => IoTDBThriftAsyncConnector.java} | 10 +-
...nectorV2.java => IoTDBThriftSyncConnector.java} | 10 +-
.../iotdb/commons/client/ClientManagerTest.java | 128 ++++-
.../pipe/plugin/builtin/BuiltinPipePluginTest.java | 4 +-
.../iotdb/tsfile/read/TsFileSequenceReader.java | 12 +-
91 files changed, 2023 insertions(+), 1050 deletions(-)
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/{v1 => payload/evolvable}/PipeRequestType.java (96%)
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/{v1 => payload/evolvable}/reponse/PipeTransferFilePieceResp.java (97%)
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/{v1 => payload/evolvable}/request/PipeTransferFilePieceReq.java (94%)
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/{v1 => payload/evolvable}/request/PipeTransferFileSealReq.java (93%)
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/{v1 => payload/evolvable}/request/PipeTransferHandshakeReq.java (92%)
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/{v1 => payload/evolvable}/request/PipeTransferInsertNodeReq.java (94%)
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/{v1 => payload/evolvable}/request/PipeTransferTabletReq.java (97%)
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/{legacy/pipedata => payload/legacy}/DeletionPipeData.java (93%)
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/{legacy/pipedata => payload/legacy}/PipeData.java (96%)
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/{legacy/pipedata => payload/legacy}/TsFilePipeData.java (95%)
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBAirGapConnector.java
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/{legacy/IoTDBSyncConnector.java => protocol/legacy/IoTDBLegacyPipeConnector.java} (94%)
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/{base => protocol/thrift}/IoTDBThriftConnector.java (98%)
copy iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/{ => protocol/thrift}/IoTDBThriftConnectorRequestVersion.java (94%)
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/{v2/IoTDBThriftConnectorV2.java => protocol/thrift/async/IoTDBThriftAsyncConnector.java} (90%)
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/{v2 => protocol/thrift/async}/handler/PipeTransferInsertNodeTabletInsertionEventHandler.java (88%)
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/{v2 => protocol/thrift/async}/handler/PipeTransferRawTabletInsertionEventHandler.java (88%)
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/{v2 => protocol/thrift/async}/handler/PipeTransferTabletInsertionEventHandler.java (90%)
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/{v2 => protocol/thrift/async}/handler/PipeTransferTsFileInsertionEventHandler.java (91%)
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/{v1/IoTDBThriftConnectorV1.java => protocol/thrift/sync/IoTDBThriftSyncConnector.java} (86%)
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/{v1/IoTDBThriftConnectorClient.java => protocol/thrift/sync/IoTDBThriftSyncConnectorClient.java} (90%)
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/{connector/IoTDBThriftConnectorRequestVersion.java => receiver/airgap/IoTDBAirGapReceiver.java} (72%)
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/{connector/legacy/IoTDBSyncReceiver.java => receiver/legacy/IoTDBLegacyPipeReceiver.java} (96%)
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/{connector => receiver}/legacy/loader/DeletionLoader.java (98%)
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/{connector => receiver}/legacy/loader/ILoader.java (96%)
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/{connector => receiver}/legacy/loader/TsFileLoader.java (96%)
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/{agent/receiver => receiver/thrift}/IoTDBThriftReceiver.java (90%)
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/{connector/v1 => receiver/thrift}/IoTDBThriftReceiverV1.java (96%)
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsfileAnalyzer.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/CompactionEstimateUtils.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCompactionInnerCompactionEstimator.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FileInfo.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/ReadChunkInnerCompactionEstimator.java
create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionTaskMemCostEstimatorTest.java
rename iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/plugin/builtin/connector/{IoTDBSyncConnector.java => IoTDBLegacyPipeConnector.java} (88%)
rename iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/plugin/builtin/connector/{IoTDBThriftConnectorV1.java => IoTDBThriftAsyncConnector.java} (73%)
rename iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/plugin/builtin/connector/{IoTDBThriftConnectorV2.java => IoTDBThriftSyncConnector.java} (73%)
[iotdb] 01/01: Merge branch 'master' of github.com:apache/iotdb into multi-way-sync
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch multi-way-sync
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit fea17764486aeab3e7fd5bdac231e544ab559f8c
Merge: 90153702157 11af323b79d
Author: Steve Yurong Su <ro...@apache.org>
AuthorDate: Mon Aug 14 20:06:18 2023 +0800
Merge branch 'master' of github.com:apache/iotdb into multi-way-sync
.../it/env/cluster/config/MppCommonConfig.java | 6 +
.../env/cluster/config/MppSharedCommonConfig.java | 7 +
.../it/env/remote/config/RemoteCommonConfig.java | 4 +
.../org/apache/iotdb/itbase/env/CommonConfig.java | 2 +
.../it/partition/IoTDBPartitionDurableIT.java | 6 +-
.../main/java/org/apache/iotdb/tool/ImportCsv.java | 2 +-
iotdb-client/session/pom.xml | 5 +
.../apache/iotdb/session/pool/SessionPoolTest.java | 90 ++--
.../confignode/conf/ConfigNodeDescriptor.java | 2 +-
.../iotdb/confignode/manager/load/LoadManager.java | 8 +-
.../manager/load/balancer/RouteBalancer.java | 4 +-
.../manager/load/service/StatisticsService.java | 16 +-
.../iot/client/AsyncIoTConsensusServiceClient.java | 1 +
.../resources/conf/iotdb-datanode.properties | 2 +-
.../org/apache/iotdb/db/audit/AuditLogger.java | 2 +-
.../db/pipe/agent/receiver/PipeReceiverAgent.java | 5 +-
.../{v1 => payload/evolvable}/PipeRequestType.java | 2 +-
.../reponse/PipeTransferFilePieceResp.java | 2 +-
.../request/PipeTransferFilePieceReq.java | 6 +-
.../request/PipeTransferFileSealReq.java | 6 +-
.../request/PipeTransferHandshakeReq.java | 6 +-
.../request/PipeTransferInsertNodeReq.java | 6 +-
.../evolvable}/request/PipeTransferTabletReq.java | 6 +-
.../legacy}/DeletionPipeData.java | 6 +-
.../pipedata => payload/legacy}/PipeData.java | 4 +-
.../legacy}/TsFilePipeData.java | 6 +-
.../protocol/airgap/IoTDBAirGapConnector.java | 57 +++
.../legacy/IoTDBLegacyPipeConnector.java} | 20 +-
.../thrift}/IoTDBThriftConnector.java | 2 +-
.../IoTDBThriftConnectorRequestVersion.java | 3 +-
.../thrift/async/IoTDBThriftAsyncConnector.java} | 44 +-
...nsferInsertNodeTabletInsertionEventHandler.java | 6 +-
...PipeTransferRawTabletInsertionEventHandler.java | 6 +-
.../PipeTransferTabletInsertionEventHandler.java | 11 +-
.../PipeTransferTsFileInsertionEventHandler.java | 14 +-
.../thrift/sync/IoTDBThriftSyncConnector.java} | 47 +-
.../sync/IoTDBThriftSyncConnectorClient.java} | 6 +-
.../airgap/IoTDBAirGapReceiver.java} | 18 +-
.../legacy/IoTDBLegacyPipeReceiver.java} | 20 +-
.../legacy/loader/DeletionLoader.java | 2 +-
.../legacy/loader/ILoader.java | 2 +-
.../legacy/loader/TsFileLoader.java | 4 +-
.../thrift}/IoTDBThriftReceiver.java | 4 +-
.../thrift}/IoTDBThriftReceiverV1.java | 18 +-
.../connector/PipeConnectorSubtaskManager.java | 20 +-
.../protocol/thrift/impl/ClientRPCServiceImpl.java | 10 +-
.../queryengine/plan/analyze/AnalyzeVisitor.java | 397 +--------------
.../plan/analyze/LoadTsfileAnalyzer.java | 543 +++++++++++++++++++++
.../db/queryengine/plan/parser/ASTVisitor.java | 2 +-
.../plan/statement/crud/LoadTsFileStatement.java | 29 +-
.../dataregion/HashLastFlushTimeMap.java | 15 +-
.../exception/CompactionExceptionHandler.java | 11 +-
.../execute/task/AbstractCompactionTask.java | 2 +
.../execute/task/CrossSpaceCompactionTask.java | 24 +-
.../execute/task/InnerSpaceCompactionTask.java | 58 ++-
.../compaction/execute/utils/CompactionUtils.java | 53 ++
.../estimator/AbstractCompactionEstimator.java | 3 +-
.../estimator/AbstractInnerSpaceEstimator.java | 74 ++-
.../estimator/CompactionEstimateUtils.java | 96 ++++
.../FastCompactionInnerCompactionEstimator.java | 52 ++
.../compaction/selector/estimator/FileInfo.java | 48 ++
.../ReadChunkInnerCompactionEstimator.java | 59 +++
.../ReadPointCrossCompactionEstimator.java | 81 +--
.../dataregion/memtable/TsFileProcessor.java | 146 +++---
.../dataregion/tsfile/TsFileManager.java | 17 +
.../wal/checkpoint/CheckpointManager.java | 2 +-
.../storageengine/dataregion/wal/node/WALNode.java | 243 ++++++---
.../db/utils/datastructure/AlignedTVList.java | 46 ++
.../iotdb/db/pipe/connector/PipeConnectorTest.java | 24 +-
.../iotdb/db/pipe/connector/PipeReceiverTest.java | 6 +-
.../db/pipe/connector/PipeThriftRequestTest.java | 12 +-
.../compaction/CompactionSchedulerTest.java | 1 +
.../utils/CompactionTaskMemCostEstimatorTest.java | 94 ++++
.../dataregion/memtable/TsFileProcessorTest.java | 97 ++++
.../resources/conf/iotdb-common.properties | 82 +---
.../iotdb/commons/client/ClientPoolFactory.java | 2 +-
.../async/AsyncConfigNodeIServiceClient.java | 1 +
.../async/AsyncDataNodeInternalServiceClient.java | 1 +
.../AsyncDataNodeMPPDataExchangeServiceClient.java | 1 +
.../async/AsyncPipeDataTransferServiceClient.java | 1 +
.../iotdb/commons/concurrent/ThreadName.java | 3 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 23 +-
.../iotdb/commons/conf/CommonDescriptor.java | 8 +-
.../iotdb/commons/pipe/config/PipeConfig.java | 4 +-
.../pipe/plugin/builtin/BuiltinPipePlugin.java | 12 +-
...onnector.java => IoTDBLegacyPipeConnector.java} | 10 +-
...ectorV1.java => IoTDBThriftAsyncConnector.java} | 10 +-
...nectorV2.java => IoTDBThriftSyncConnector.java} | 10 +-
.../iotdb/commons/client/ClientManagerTest.java | 128 ++++-
.../pipe/plugin/builtin/BuiltinPipePluginTest.java | 4 +-
.../iotdb/tsfile/read/TsFileSequenceReader.java | 12 +-
91 files changed, 2023 insertions(+), 1050 deletions(-)