You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ma...@apache.org on 2022/08/10 15:16:06 UTC
[iotdb] branch IOTDB-3771 updated (9f255fb0f7 -> c9bccacba0)
This is an automated email from the ASF dual-hosted git repository.
marklau99 pushed a change to branch IOTDB-3771
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 9f255fb0f7 fix ci
add c04767fb60 update administration user-guide (#6913)
add a8f757d6e4 disable the log in state tracker for previous debugging (#6917)
add a2a690b3d8 [IOTDB-2952] IOTDB Grafana Plugin: Build on Windows (#6907)
add b7c4062f28 [IOTDB-4056] Fix the problem that could not find or load main class in shell `remove-confignode.sh` (#6916)
add 87d200e54b Move clientPool factory from ConfigNode to node-commons (#6910)
add e514211827 [IOTDB-4065] support negative timestamp (before 1970 year.) (#6918)
add 51cf191fe1 Refine writeValueColumn() to TsBlockBuilder for common use (#6912)
add a04a1eb62b [IOTDB-3996] Forbidden CQ in REST query API (#6922)
add 0cac2c3237 [IOTDB-4046] Enhance PipeData serialize()/deserialize() codes (#6902)
add 67c469305f purge log instantly when take a snapshot (#6724)
add a8880b51d2 [IOTDB-4054] Add some annotations for confignode.thrift (#6919)
add 6421f4ea77 [IOTDB-4077] DataNode report region migrate result to FOLLOWER ConfigNode sometimes, the LEADER Confignode is still waiting (#6933)
add b63c4aba78 [IOTDB-4010] Use FilterAndProjectOperator for query that does not contain non-mappable UDF (#6929)
add ec873773e6 Correct typos in log (#6943)
add c30bb7c00d [IOTDB-3296] ext-pipe suport .mods file (#6102)
add 6188f62c81 [IOTDB-4067] Handle null pointer in getRegisteredNodeInfoList (#6930)
add f12e15eb03 [IOTDB-4045] Optimize DataNode query relevant interfaces (#6944)
add 2fee7a4b6a [IOTDB-3191][IOTDB-3192] Refactoring sync module to use AbstractSyncInfo to manage pipe metadata (#6817)
add 517dc13a63 [IOTDB-4090] Add getLatestSnapshotFiles interface in consensus (#6948)
add 085928af01 [IOTDB-4039] Distinguish the selector thread-name of ClientPools (#6927)
add 588bf40de3 [IOTDB-3821] add some new test cases for ConfigNodeIT (#6906)
add f1787b83c5 [IOTDB-3938] Optimize Standalone schema fetch performance (#6949)
add 9ec5a2dfab Merge remote-tracking branch 'upstream/master' into IOTDB-3771
add c9bccacba0 implement get file info in SnapshotLoader
No new revisions were added by this update.
Summary of changes:
.github/workflows/sync.yml | 20 +-
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 2 +-
.../assembly/resources/sbin/remove-confignode.sh | 40 +-
.../assembly/resources/sbin/start-confignode.sh | 36 +-
.../src/assembly/resources/sbin/stop-confignode.sh | 11 +-
.../AsyncConfigNodeHeartbeatClientPool.java | 4 +-
.../async/datanode/AsyncDataNodeClientPool.java | 4 +-
.../datanode/AsyncDataNodeHeartbeatClientPool.java | 4 +-
.../sync/datanode/SyncDataNodeClientPool.java | 4 +-
.../iotdb/confignode/conf/ConfigNodeConfig.java | 41 --
.../confignode/conf/ConfigNodeDescriptor.java | 17 -
.../confignode/manager/ClusterSchemaManager.java | 4 +-
.../iotdb/confignode/manager/ConfigManager.java | 12 +-
.../apache/iotdb/confignode/manager/IManager.java | 9 +
.../iotdb/confignode/manager/NodeManager.java | 54 +-
.../confignode/manager/PermissionManager.java | 2 +-
.../iotdb/confignode/manager/UDFManager.java | 4 +-
.../iotdb/confignode/manager/load/LoadManager.java | 16 +-
.../manager/load/balancer/RegionBalancer.java | 2 +-
.../manager/load/balancer/RouteBalancer.java | 2 +-
.../iotdb/confignode/persistence/NodeInfo.java | 16 +-
.../procedure/env/ConfigNodeProcedureEnv.java | 4 +-
.../procedure/env/DataNodeRemoveHandler.java | 6 +-
.../service/thrift/ConfigNodeRPCService.java | 15 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 3 +-
.../iotdb/confignode/persistence/NodeInfoTest.java | 4 +-
.../thrift/ConfigNodeRPCServiceProcessorTest.java | 510 ----------------
.../org/apache/iotdb/consensus/IStateMachine.java | 15 +
.../apache/iotdb/consensus/config/RatisConfig.java | 2 +-
.../client/AsyncMultiLeaderServiceClient.java | 5 +-
.../client/MultiLeaderConsensusClientPool.java | 5 +-
.../Administration-Management/Administration.md | 68 +--
.../Administration-Management/Administration.md | 68 +--
.../backend-compile.bat | 29 +-
grafana-plugin/go.mod | 2 +-
grafana-plugin/go.sum | 15 +-
grafana-plugin/pom.xml | 54 +-
integration-test/import-control.xml | 4 +-
.../org/apache/iotdb/db/it/IoTDBConfigNodeIT.java | 639 +++++++++++++++++++++
.../sync/IoTDBSyncReceiverCollectorIT.java | 513 -----------------
.../db/integration/sync/IoTDBSyncReceiverIT.java | 200 +------
.../db/integration/sync/IoTDBSyncSenderIT.java | 2 +
.../db/integration/sync/TransportClientMock.java | 9 -
.../db/integration/sync/TransportHandlerMock.java | 3 -
node-commons/pom.xml | 5 +
.../commons/client/AsyncBaseClientFactory.java | 24 +-
.../iotdb/commons/client/ClientPoolFactory.java | 31 +-
.../AsyncConfigNodeHeartbeatServiceClient.java | 5 +-
.../async/AsyncConfigNodeIServiceClient.java | 5 +-
.../async/AsyncDataNodeHeartbeatServiceClient.java | 5 +-
.../async/AsyncDataNodeInternalServiceClient.java | 5 +-
.../AsyncDataNodeMPPDataExchangeServiceClient.java | 5 +-
.../iotdb/commons/concurrent/ThreadName.java | 3 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 53 ++
.../iotdb/commons/conf/CommonDescriptor.java | 18 +
.../apache/iotdb/commons/sync}/SyncConstant.java | 22 +-
.../apache/iotdb/commons/sync}/SyncPathUtil.java | 24 +-
.../iotdb/commons/client/ClientManagerTest.java | 4 +-
.../schemaregion/rocksdb/RSchemaRegion.java | 8 +
.../apache/iotdb/db/client/ConfigNodeClient.java | 3 +-
.../iotdb/db/client/DataNodeClientPoolFactory.java | 28 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 13 -
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 2 -
.../statemachine/DataRegionStateMachine.java | 19 +
.../iotdb/db/engine/snapshot/SnapshotLoader.java | 124 +++-
.../db/engine/snapshot/SnapshotLogAnalyzer.java | 4 +
.../iotdb/db/engine/snapshot/SnapshotTaker.java | 3 +-
.../sync/PipeDataLoadBearableException.java | 25 -
.../sync/PipeDataLoadUnbearableException.java | 25 -
.../db/metadata/schemaregion/ISchemaRegion.java | 6 +
.../schemaregion/SchemaRegionMemoryImpl.java | 45 ++
.../schemaregion/SchemaRegionSchemaFileImpl.java | 8 +
.../mpp/common/schematree/ClusterSchemaTree.java | 15 +-
.../common/schematree/DeviceGroupSchemaTree.java | 98 ++++
.../db/mpp/common/schematree/DeviceSchemaInfo.java | 110 +++-
.../common/schematree/MeasurementSchemaInfo.java} | 33 +-
.../visitor/SchemaTreeDeviceVisitor.java | 13 +-
.../db/mpp/execution/exchange/SourceHandle.java | 2 +-
.../operator/process/FilterAndProjectOperator.java | 11 +-
.../schema/TimeSeriesSchemaScanOperator.java | 25 +-
.../mpp/plan/analyze/StandaloneSchemaFetcher.java | 231 ++------
.../plan/expression/multi/FunctionExpression.java | 3 +
.../iotdb/db/mpp/plan/parser/ASTVisitor.java | 3 +
.../db/mpp/plan/planner/OperatorTreeGenerator.java | 56 +-
.../scheduler/FixedRateFragInsStateTracker.java | 10 +-
.../handler/PhysicalPlanValidationHandler.java | 4 +-
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 55 +-
.../iotdb/db/qp/physical/sys/CreatePipePlan.java | 2 +-
.../db/qp/physical/sys/CreatePipeSinkPlan.java | 2 +-
.../apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java | 3 +
.../iotdb/db/sync/common/ISyncInfoFetcher.java | 69 +++
.../iotdb/db/sync/common/LocalSyncInfoFetcher.java | 186 ++++++
.../org/apache/iotdb/db/sync/common/SyncInfo.java | 292 ++++++++++
.../db/sync/common/persistence/SyncLogReader.java | 186 ++++++
.../db/sync/common/persistence/SyncLogWriter.java | 147 +++++
.../iotdb/db/sync/datasource/AbstractOpBlock.java | 16 +-
.../iotdb/db/sync/datasource/DeletionGroup.java | 242 ++++++++
.../iotdb/db/sync/datasource/ModsfileOpBlock.java | 53 --
.../iotdb/db/sync/datasource/PipeOpManager.java | 6 +-
.../iotdb/db/sync/datasource/TsFileOpBlock.java | 463 ++++++++++++---
.../iotdb/db/sync/externalpipe/ExtPipePlugin.java | 15 +-
.../db/sync/externalpipe/ExtPipePluginManager.java | 4 +-
.../iotdb/db/sync/pipedata/DeletionPipeData.java | 12 +-
.../apache/iotdb/db/sync/pipedata/PipeData.java | 24 +-
.../iotdb/db/sync/pipedata/SchemaPipeData.java | 12 +-
.../iotdb/db/sync/pipedata/TsFilePipeData.java | 33 +-
.../sync/pipedata/queue/BufferedPipeDataQueue.java | 6 +-
.../sync/pipedata/queue/PipeDataQueueFactory.java | 57 --
.../iotdb/db/sync/receiver/ReceiverService.java | 195 +------
.../db/sync/receiver/collector/Collector.java | 171 ------
.../db/sync/receiver/load/DeletionLoader.java | 5 +-
.../iotdb/db/sync/receiver/load/SchemaLoader.java | 8 +-
.../iotdb/db/sync/receiver/load/TsFileLoader.java | 3 +-
.../db/sync/receiver/manager/ReceiverManager.java | 229 --------
.../db/sync/receiver/recovery/ReceiverLog.java | 127 ----
.../receiver/recovery/ReceiverLogAnalyzer.java | 157 -----
.../iotdb/db/sync/sender/pipe/IoTDBPipeSink.java | 2 +-
.../org/apache/iotdb/db/sync/sender/pipe/Pipe.java | 4 -
.../manager => sender/pipe}/PipeInfo.java | 61 +-
.../iotdb/db/sync/sender/pipe/TsFilePipe.java | 14 +-
.../iotdb/db/sync/sender/pipe/TsFilePipeInfo.java | 63 ++
.../db/sync/sender/recovery/SenderLogAnalyzer.java | 165 ------
.../db/sync/sender/recovery/SenderLogger.java | 141 -----
.../db/sync/sender/recovery/TsFilePipeLogger.java | 4 +-
.../iotdb/db/sync/sender/service/MsgManager.java | 114 ----
.../db/sync/sender/service/SenderService.java | 377 ++++++------
.../db/sync/sender/service/TransportHandler.java | 56 +-
.../db/sync/transport/client/ClientWrapper.java | 4 +-
.../db/sync/transport/client/ITransportClient.java | 10 +-
...rtClient.java => IoTDBSInkTransportClient.java} | 148 +----
.../db/sync/transport/conf/TransportConstant.java | 36 --
.../transport/server/TransportServiceImpl.java | 50 +-
.../apache/iotdb/db/utils/sync/SyncPipeUtil.java | 96 ++++
.../java/org/apache/iotdb/db/qp/PlannerTest.java | 11 +
.../db/sync/datasource/DeletionGroupTest.java | 231 ++++++++
.../db/sync/datasource/PipeOpManagerTest.java | 226 +++++++-
.../db/sync/datasource/TsFileOpBlockTest.java | 372 +++++++++++-
.../sync/pipedata/BufferedPipeDataQueueTest.java | 4 +-
.../iotdb/db/sync/pipedata/PipeDataTest.java | 12 +-
...{ReceiverManagerTest.java => SyncInfoTest.java} | 71 +--
...ceiverLogAnalyzerTest.java => SyncLogTest.java} | 68 ++-
.../db/sync/transport/TransportServiceTest.java | 197 +++----
.../apache/iotdb/db/utils/EnvironmentUtils.java | 13 +-
.../datanode1conf/iotdb-datanode.properties | 1 +
.../datanode2conf/iotdb-datanode.properties | 1 +
.../datanode3conf/iotdb-datanode.properties | 1 +
.../src/main/thrift/confignode.thrift | 14 +-
thrift-sync/src/main/thrift/transport.thrift | 27 -
.../tsfile/read/common/block/TsBlockBuilder.java | 16 +
149 files changed, 4744 insertions(+), 4261 deletions(-)
copy server/src/assembly/resources/sbin/stop-server.bat => grafana-plugin/backend-compile.bat (60%)
mode change 100755 => 100644
delete mode 100644 integration/src/test/java/org/apache/iotdb/db/integration/sync/IoTDBSyncReceiverCollectorIT.java
rename confignode/src/main/java/org/apache/iotdb/confignode/client/ConfigNodeClientPoolFactory.java => node-commons/src/main/java/org/apache/iotdb/commons/client/ClientPoolFactory.java (84%)
rename {server/src/main/java/org/apache/iotdb/db/sync/conf => node-commons/src/main/java/org/apache/iotdb/commons/sync}/SyncConstant.java (82%)
rename {server/src/main/java/org/apache/iotdb/db/sync/conf => node-commons/src/main/java/org/apache/iotdb/commons/sync}/SyncPathUtil.java (87%)
delete mode 100644 server/src/main/java/org/apache/iotdb/db/exception/sync/PipeDataLoadBearableException.java
delete mode 100644 server/src/main/java/org/apache/iotdb/db/exception/sync/PipeDataLoadUnbearableException.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/DeviceGroupSchemaTree.java
copy server/src/main/java/org/apache/iotdb/db/{exception/metadata/DeleteFailedException.java => mpp/common/schematree/MeasurementSchemaInfo.java} (51%)
create mode 100644 server/src/main/java/org/apache/iotdb/db/sync/common/ISyncInfoFetcher.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/sync/common/LocalSyncInfoFetcher.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/sync/common/SyncInfo.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/sync/common/persistence/SyncLogReader.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/sync/common/persistence/SyncLogWriter.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/sync/datasource/DeletionGroup.java
delete mode 100644 server/src/main/java/org/apache/iotdb/db/sync/datasource/ModsfileOpBlock.java
delete mode 100644 server/src/main/java/org/apache/iotdb/db/sync/pipedata/queue/PipeDataQueueFactory.java
delete mode 100644 server/src/main/java/org/apache/iotdb/db/sync/receiver/collector/Collector.java
delete mode 100644 server/src/main/java/org/apache/iotdb/db/sync/receiver/manager/ReceiverManager.java
delete mode 100644 server/src/main/java/org/apache/iotdb/db/sync/receiver/recovery/ReceiverLog.java
delete mode 100644 server/src/main/java/org/apache/iotdb/db/sync/receiver/recovery/ReceiverLogAnalyzer.java
rename server/src/main/java/org/apache/iotdb/db/sync/{receiver/manager => sender/pipe}/PipeInfo.java (60%)
create mode 100644 server/src/main/java/org/apache/iotdb/db/sync/sender/pipe/TsFilePipeInfo.java
delete mode 100644 server/src/main/java/org/apache/iotdb/db/sync/sender/recovery/SenderLogAnalyzer.java
delete mode 100644 server/src/main/java/org/apache/iotdb/db/sync/sender/recovery/SenderLogger.java
delete mode 100644 server/src/main/java/org/apache/iotdb/db/sync/sender/service/MsgManager.java
rename server/src/main/java/org/apache/iotdb/db/sync/transport/client/{TransportClient.java => IoTDBSInkTransportClient.java} (73%)
delete mode 100644 server/src/main/java/org/apache/iotdb/db/sync/transport/conf/TransportConstant.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/utils/sync/SyncPipeUtil.java
create mode 100644 server/src/test/java/org/apache/iotdb/db/sync/datasource/DeletionGroupTest.java
rename server/src/test/java/org/apache/iotdb/db/sync/receiver/manager/{ReceiverManagerTest.java => SyncInfoTest.java} (53%)
rename server/src/test/java/org/apache/iotdb/db/sync/receiver/recovery/{ReceiverLogAnalyzerTest.java => SyncLogTest.java} (59%)