You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2024/01/23 10:32:03 UTC
(iotdb) branch ty/HA updated (28b276bf130 -> 5493b32b60e)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a change to branch ty/HA
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 28b276bf130 support all write operation retry
add 7dd570d991a Load: Introduce LoadTsFileManager.CleanupTask to force close writer manager after exception occurring (#11924)
add 4cc5b2ba48d Fix query result is empty in order by time align by device
add 771c096b478 Pipe: let non enriched event forever retry when exception occurred to avoid subtask fake running status recorded at data node task agent (#11929)
add dd48a57ddb2 Improve some descriptions of daemon-script
add 63605be3186 Fix the issue where ConfigNode reports 'ConsensusGroupAlreadyExist' error during startup in some scenarios (#11928)
add c3597b6ae12 Fixed the issue that IoTConsensus synchronization is stuck when WAL file is corrupted (#11930)
add 0cb14994659 Enable SchemaRegion Leader Auto Balance (#11926)
add f94f99a4828 Change partial aggregation symbols for variance and standard deviation aggregators
add 3464e95f495 Load: Support configure load_clean_up_task_execution_delay_time_seconds of a load task (#11936)
add db7d115204a Skip broken wal files when iterate the PlanNodeIterator (#11935)
add 4ec79560c60 fix remaining issues when upgrading to ratis 3.0.1 (#11940)
add f1db2923dfb Pipe: decrease the default value of pipe_sink_selector_number to 4 for better performance (#11942)
add 85e71a43543 Fix update time index after compact aligned series (#11943)
add 8885df31144 fix PointPriorityReader calculation of aligned series null value num (#11945)
add 9e05b262559 Add the ability for ClientManager to periodically clean up idle objects
add 69202a94a51 reduce stack message of last time check exception for read chunk compaction performer (#11927)
add 6f37cf83b1d [IOTDB-6293] Use nanotime in cluster heartbeat (#11916)
add 4cad6af6715 Fix compile error caused by concurrent pr (#11951)
add 3846bb7f2bc Enhance the robustness of Ratis linearizable reads for node offline scenarios (#11954)
add 070de203710 PBTree: Decouple PageManager (#11953)
add 827735d1d77 fixed collection iotdb info unable to obtain information (#11938)
new 5493b32b60e resolve conflicts
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../src/assembly/resources/tools/collect-info.bat | 146 ++++---
.../resources/conf/iotdb-confignode.properties | 5 -
.../assembly/resources/sbin/daemon-confignode.sh | 18 +-
.../heartbeat/DataNodeHeartbeatHandler.java | 2 +-
.../iotdb/confignode/conf/ConfigNodeConfig.java | 17 +-
.../confignode/conf/ConfigNodeDescriptor.java | 8 -
.../manager/consensus/ConsensusManager.java | 2 -
.../confignode/manager/load/cache/LoadCache.java | 2 +-
.../manager/load/cache/node/BaseNodeCache.java | 2 +-
.../load/cache/node/ConfigNodeHeartbeatCache.java | 2 +-
.../load/cache/node/DataNodeHeartbeatCache.java | 2 +-
.../load/cache/node/NodeHeartbeatSample.java | 6 +-
.../manager/load/cache/node/NodeStatistics.java | 2 +-
.../manager/load/cache/region/RegionCache.java | 5 +-
.../load/cache/region/RegionHeartbeatSample.java | 2 +-
.../manager/load/service/HeartbeatService.java | 4 +-
.../procedure/env/ConfigNodeProcedureEnv.java | 2 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 2 +-
.../router/priority/GreedyPriorityTest.java | 6 +-
.../priority/LeaderPriorityBalancerTest.java | 6 +-
.../manager/load/cache/LoadCacheTest.java | 2 +-
.../manager/load/cache/NodeCacheTest.java | 6 +-
.../manager/load/cache/RegionGroupCacheTest.java | 6 +-
.../manager/load/cache/RegionRouteCacheTest.java | 2 +-
.../iotdb/consensus/config/IoTConsensusConfig.java | 16 -
.../apache/iotdb/consensus/config/RatisConfig.java | 16 -
.../exception/RatisReadUnavailableException.java | 13 +-
.../iot/client/IoTConsensusClientPool.java | 2 -
.../consensus/iot/logdispatcher/LogDispatcher.java | 24 +-
.../iotdb/consensus/ratis/RatisConsensus.java | 18 +-
.../resources/conf/iotdb-datanode.properties | 5 -
.../src/assembly/resources/sbin/daemon-datanode.sh | 18 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 30 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 27 +-
.../db/consensus/DataRegionConsensusImpl.java | 2 -
.../db/consensus/SchemaRegionConsensusImpl.java | 1 -
.../agent/runtime/PipePeriodicalJobExecutor.java | 1 +
.../db/pipe/task/subtask/PipeDataNodeSubtask.java | 105 +++--
.../subtask/connector/PipeConnectorSubtask.java | 142 +++----
.../protocol/client/DataNodeClientPoolFactory.java | 2 -
.../execution/load/LoadTsFileManager.java | 144 ++++---
.../execution/operator/AggregationUtil.java | 6 +
.../plan/planner/LogicalPlanBuilder.java | 19 +-
.../plan/planner/distribution/SourceRewriter.java | 9 +-
.../plan/parameter/AggregationDescriptor.java | 14 +-
.../scheduler/FragmentInstanceDispatcherImpl.java | 24 +-
.../plan/scheduler/load/LoadTsFileScheduler.java | 8 +-
.../schemafile/pagemgr/BTreePageManager.java | 8 +-
.../pbtree/schemafile/pagemgr/PageIOChannel.java | 184 +++++++++
.../schemafile/pagemgr/PageIndexSortBuckets.java | 129 ++++++
.../pbtree/schemafile/pagemgr/PageManager.java | 452 ++-------------------
.../impl/pbtree/schemafile/pagemgr/PagePool.java | 152 +++++++
.../schemafile/pagemgr/SchemaPageContext.java | 109 +++++
.../iotdb/db/storageengine/StorageEngine.java | 19 +-
.../impl/ReadChunkCompactionPerformer.java | 7 +-
.../ReadChunkAlignedSeriesCompactionExecutor.java | 10 +-
.../execute/utils/reader/PointPriorityReader.java | 2 +-
.../storageengine/dataregion/wal/node/WALNode.java | 15 +-
.../org/apache/iotdb/db/utils/SchemaUtils.java | 27 +-
.../FastInnerCompactionPerformerTest.java | 108 +++++
...nkCompactionPerformerWithAlignedSeriesTest.java | 102 ++++-
.../resources/conf/iotdb-cluster.properties | 2 +-
.../resources/conf/iotdb-common.properties | 15 +-
.../iotdb/commons/client/ClientPoolFactory.java | 29 +-
.../client/property/ClientPoolProperty.java | 35 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 20 +-
.../iotdb/commons/conf/CommonDescriptor.java | 23 --
.../iotdb/commons/pipe/config/PipeConfig.java | 5 -
.../iotdb/commons/client/ClientManagerTest.java | 47 ++-
69 files changed, 1387 insertions(+), 1016 deletions(-)
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/pagemgr/PageIOChannel.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/pagemgr/PageIndexSortBuckets.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/pagemgr/PagePool.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/pagemgr/SchemaPageContext.java
(iotdb) 01/01: resolve conflicts
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch ty/HA
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5493b32b60ef8a0f7801b2e9b4df5eac8aa1ff9f
Merge: 28b276bf130 827735d1d77
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Tue Jan 23 18:31:53 2024 +0800
resolve conflicts
.../src/assembly/resources/tools/collect-info.bat | 146 ++++---
.../resources/conf/iotdb-confignode.properties | 5 -
.../assembly/resources/sbin/daemon-confignode.sh | 18 +-
.../heartbeat/DataNodeHeartbeatHandler.java | 2 +-
.../iotdb/confignode/conf/ConfigNodeConfig.java | 17 +-
.../confignode/conf/ConfigNodeDescriptor.java | 8 -
.../manager/consensus/ConsensusManager.java | 2 -
.../confignode/manager/load/cache/LoadCache.java | 2 +-
.../manager/load/cache/node/BaseNodeCache.java | 2 +-
.../load/cache/node/ConfigNodeHeartbeatCache.java | 2 +-
.../load/cache/node/DataNodeHeartbeatCache.java | 2 +-
.../load/cache/node/NodeHeartbeatSample.java | 6 +-
.../manager/load/cache/node/NodeStatistics.java | 2 +-
.../manager/load/cache/region/RegionCache.java | 5 +-
.../load/cache/region/RegionHeartbeatSample.java | 2 +-
.../manager/load/service/HeartbeatService.java | 4 +-
.../procedure/env/ConfigNodeProcedureEnv.java | 2 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 2 +-
.../router/priority/GreedyPriorityTest.java | 6 +-
.../priority/LeaderPriorityBalancerTest.java | 6 +-
.../manager/load/cache/LoadCacheTest.java | 2 +-
.../manager/load/cache/NodeCacheTest.java | 6 +-
.../manager/load/cache/RegionGroupCacheTest.java | 6 +-
.../manager/load/cache/RegionRouteCacheTest.java | 2 +-
.../iotdb/consensus/config/IoTConsensusConfig.java | 16 -
.../apache/iotdb/consensus/config/RatisConfig.java | 16 -
.../exception/RatisReadUnavailableException.java | 13 +-
.../iot/client/IoTConsensusClientPool.java | 2 -
.../consensus/iot/logdispatcher/LogDispatcher.java | 24 +-
.../iotdb/consensus/ratis/RatisConsensus.java | 18 +-
.../resources/conf/iotdb-datanode.properties | 5 -
.../src/assembly/resources/sbin/daemon-datanode.sh | 18 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 30 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 27 +-
.../db/consensus/DataRegionConsensusImpl.java | 2 -
.../db/consensus/SchemaRegionConsensusImpl.java | 1 -
.../agent/runtime/PipePeriodicalJobExecutor.java | 1 +
.../db/pipe/task/subtask/PipeDataNodeSubtask.java | 105 +++--
.../subtask/connector/PipeConnectorSubtask.java | 142 +++----
.../protocol/client/DataNodeClientPoolFactory.java | 2 -
.../execution/load/LoadTsFileManager.java | 144 ++++---
.../execution/operator/AggregationUtil.java | 6 +
.../plan/planner/LogicalPlanBuilder.java | 19 +-
.../plan/planner/distribution/SourceRewriter.java | 9 +-
.../plan/parameter/AggregationDescriptor.java | 14 +-
.../scheduler/FragmentInstanceDispatcherImpl.java | 24 +-
.../plan/scheduler/load/LoadTsFileScheduler.java | 8 +-
.../schemafile/pagemgr/BTreePageManager.java | 8 +-
.../pbtree/schemafile/pagemgr/PageIOChannel.java | 184 +++++++++
.../schemafile/pagemgr/PageIndexSortBuckets.java | 129 ++++++
.../pbtree/schemafile/pagemgr/PageManager.java | 452 ++-------------------
.../impl/pbtree/schemafile/pagemgr/PagePool.java | 152 +++++++
.../schemafile/pagemgr/SchemaPageContext.java | 109 +++++
.../iotdb/db/storageengine/StorageEngine.java | 19 +-
.../impl/ReadChunkCompactionPerformer.java | 7 +-
.../ReadChunkAlignedSeriesCompactionExecutor.java | 10 +-
.../execute/utils/reader/PointPriorityReader.java | 2 +-
.../storageengine/dataregion/wal/node/WALNode.java | 15 +-
.../org/apache/iotdb/db/utils/SchemaUtils.java | 27 +-
.../FastInnerCompactionPerformerTest.java | 108 +++++
...nkCompactionPerformerWithAlignedSeriesTest.java | 102 ++++-
.../resources/conf/iotdb-cluster.properties | 2 +-
.../resources/conf/iotdb-common.properties | 15 +-
.../iotdb/commons/client/ClientPoolFactory.java | 29 +-
.../client/property/ClientPoolProperty.java | 35 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 20 +-
.../iotdb/commons/conf/CommonDescriptor.java | 23 --
.../iotdb/commons/pipe/config/PipeConfig.java | 5 -
.../iotdb/commons/client/ClientManagerTest.java | 47 ++-
69 files changed, 1387 insertions(+), 1016 deletions(-)
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java
index 22babfca96f,b48e8201629..c542bd09c8c
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java
@@@ -283,9 -284,8 +284,9 @@@ public class FragmentInstanceDispatcher
return this.localhostIpAddr.equals(endPoint.getIp()) && localhostInternalPort == endPoint.port;
}
- /** return true if need retry, false if no need to retry */
- private void dispatchRemote(FragmentInstance instance, TEndPoint endPoint)
- throws FragmentInstanceDispatchException {
+ private void dispatchRemoteHelper(FragmentInstance instance, TEndPoint endPoint)
- throws FragmentInstanceDispatchException, TException, ClientManagerException {
++ throws FragmentInstanceDispatchException, TException, ClientManagerException,
++ RatisReadUnavailableException {
try (SyncDataNodeInternalServiceClient client =
syncInternalServiceClientManager.borrowClient(endPoint)) {
switch (instance.getType()) {
@@@ -343,35 -348,18 +349,35 @@@
TSStatusCode.EXECUTE_STATEMENT_ERROR,
String.format("unknown read type [%s]", instance.getType())));
}
+ }
+ }
+
+ private void dispatchRemote(FragmentInstance instance, TEndPoint endPoint)
+ throws FragmentInstanceDispatchException {
+
+ try {
+ dispatchRemoteHelper(instance, endPoint);
- } catch (ClientManagerException | TException e) {
+ } catch (ClientManagerException | TException | RatisReadUnavailableException e) {
logger.warn(
- "can't connect to node {}, error msg is {}, and we try to reconnect this node.",
- "can't execute request on node {}, error msg is {}.",
++ "can't execute request on node {}, error msg is {}, and we try to reconnect this node.",
endPoint,
ExceptionUtils.getRootCause(e).toString());
- TSStatus status = new TSStatus();
- status.setCode(TSStatusCode.DISPATCH_ERROR.getStatusCode());
- status.setMessage("can't connect to node " + endPoint);
- // If the DataNode cannot be connected, its endPoint will be put into black list
- // so that the following retry will avoid dispatching instance towards this DataNode.
- queryContext.addFailedEndPoint(endPoint);
- throw new FragmentInstanceDispatchException(status);
+ // we just retry once to clear stale connection for a restart node.
+ try {
+ dispatchRemoteHelper(instance, endPoint);
- } catch (ClientManagerException | TException e1) {
++ } catch (ClientManagerException | TException | RatisReadUnavailableException e1) {
+ logger.warn(
- "can't connect to node {} in second try, error msg is {}.",
++ "can't execute request on node {} in second try, error msg is {}.",
+ endPoint,
+ ExceptionUtils.getRootCause(e1).toString());
+ TSStatus status = new TSStatus();
+ status.setCode(TSStatusCode.DISPATCH_ERROR.getStatusCode());
+ status.setMessage("can't connect to node " + endPoint);
+ // If the DataNode cannot be connected, its endPoint will be put into black list
+ // so that the following retry will avoid dispatching instance towards this DataNode.
+ queryContext.addFailedEndPoint(endPoint);
+ throw new FragmentInstanceDispatchException(status);
+ }
}
}