You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2022/08/19 03:24:34 UTC
[iotdb] 02/04: Merge branch 'IOTDB-4020' into ml_0815_data_lost
This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch ml_0815_data_lost
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit d2842e5e6e2631185caaacd3a463995c8852889a
Merge: b994998590 6e66ae5de0
Author: Jinrui.Zhang <xi...@gmail.com>
AuthorDate: Fri Aug 19 11:07:51 2022 +0800
Merge branch 'IOTDB-4020' into ml_0815_data_lost
.github/workflows/e2e.yml | 2 +-
.mvn/wrapper/MavenWrapperDownloader.java | 115 ----------
.mvn/wrapper/maven-wrapper.properties | 21 +-
LICENSE | 2 +-
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 21 +-
.../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 4 -
cli/pom.xml | 37 +++-
cli/src/assembly/cli.xml | 3 +
.../main/java/org/apache/iotdb/tool/ImportCsv.java | 95 ++++----
.../confignode/client/DataNodeRequestType.java | 3 +-
.../async/datanode/AsyncDataNodeClientPool.java | 81 +++++--
.../async/handlers/LoadConfigurationHandler.java | 82 +++++++
.../sync/datanode/SyncDataNodeClientPool.java | 13 +-
.../consensus/request/ConfigPhysicalPlan.java | 6 +-
.../consensus/request/ConfigPhysicalPlanType.java | 2 +-
.../request/write/CreateRegionGroupsPlan.java | 11 +-
...egionsPlan.java => DeleteRegionGroupsPlan.java} | 48 ++--
.../statemachine/PartitionRegionStateMachine.java | 6 +
.../iotdb/confignode/manager/ConfigManager.java | 25 +--
.../iotdb/confignode/manager/ConsensusManager.java | 28 ++-
.../apache/iotdb/confignode/manager/IManager.java | 3 +
.../iotdb/confignode/manager/NodeManager.java | 14 ++
.../iotdb/confignode/manager/PartitionManager.java | 67 ++++--
.../iotdb/confignode/manager/ProcedureManager.java | 57 +++--
.../iotdb/confignode/manager/load/LoadManager.java | 28 +--
.../manager/load/balancer/RegionBalancer.java | 2 +
.../persistence/executor/ConfigPlanExecutor.java | 63 +++---
.../persistence/partition/PartitionInfo.java | 50 ++++-
.../partition/StorageGroupPartitionTable.java | 9 +
.../procedure/env/ConfigNodeProcedureEnv.java | 50 +++++
.../impl/CreateRegionGroupsProcedure.java | 135 ++++++++++++
.../procedure/state/CreateRegionGroupsState.java | 13 +-
.../procedure/store/ProcedureFactory.java | 6 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 7 +-
.../request/ConfigPhysicalPlanSerDeTest.java | 41 ++--
.../confignode1conf/iotdb-confignode.properties | 1 +
.../org/apache/iotdb/consensus/IConsensus.java | 55 ++++-
.../org/apache/iotdb/consensus/IStateMachine.java | 4 +
.../iotdb/consensus/config/MultiLeaderConfig.java | 32 +--
.../multileader/MultiLeaderConsensus.java | 16 +-
.../multileader/MultiLeaderServerImpl.java | 44 +++-
.../multileader/logdispatcher/LogDispatcher.java | 6 +-
.../service/MultiLeaderRPCServiceProcessor.java | 8 +
.../ratis/ApplicationStateMachineProxy.java | 34 ++-
.../iotdb/consensus/ratis/RatisConsensus.java | 4 +-
.../consensus/standalone/StandAloneConsensus.java | 16 +-
.../consensus/standalone/StandAloneServerImpl.java | 5 +
.../multileader/MultiLeaderConsensusTest.java | 12 +-
.../iotdb/consensus/multileader/RecoveryTest.java | 6 +-
.../iotdb/consensus/ratis/RatisConsensusTest.java | 34 +--
.../iotdb/consensus/standalone/RecoveryTest.java | 4 +-
.../standalone/StandAloneConsensusTest.java | 22 +-
docs/Development/VoteRelease.md | 4 +-
.../Administration-Management/Administration.md | 22 +-
docs/UserGuide/QuickStart/WayToGetIoTDB.md | 2 +-
docs/UserGuide/Write-And-Delete-Data/CSV-Tool.md | 21 +-
docs/zh/Development/VoteRelease.md | 4 +-
.../Administration-Management/Administration.md | 24 +-
docs/zh/UserGuide/QuickStart/WayToGetIoTDB.md | 2 +-
.../zh/UserGuide/Write-And-Delete-Data/CSV-Tool.md | 18 +-
example/pom.xml | 2 +-
.../iotdb/db/it/groupby/IoTDBLeftORightCIT.java | 70 ++++++
.../iotdb/db/integration/IoTDBAuthorizationIT.java | 90 ++++++++
.../aligned/IoTDBDeleteTimeseriesIT.java | 64 ++++++
.../db/integration/sync/IoTDBSyncReceiverIT.java | 25 +--
mvnw.sh => mvnw | 92 ++++++--
mvnw.cmd | 75 +++++--
node-commons/pom.xml | 7 +-
.../apache/iotdb/commons/ServerCommandLine.java | 2 +-
.../apache/iotdb/commons/partition/Partition.java | 2 +
.../executor/SeriesPartitionExecutor.java | 37 +++-
.../apache/iotdb/commons/sync/SyncConstant.java | 6 -
.../apache/iotdb/commons/sync/SyncPathUtil.java | 6 +-
.../org/apache/iotdb/commons/utils/FileUtils.java | 17 ++
.../org/apache/iotdb/commons/utils/PathUtils.java | 2 +-
pom.xml | 1 -
.../schemaregion/rocksdb/RSchemaRegion.java | 3 +-
server/pom.xml | 11 +-
.../resources/conf/iotdb-datanode.properties | 10 +-
.../org/apache/iotdb/db/auth/AuthorityChecker.java | 17 +-
.../iotdb/db/auth/ClusterAuthorityFetcher.java | 11 +-
.../apache/iotdb/db/client/ConfigNodeClient.java | 16 ++
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 120 ++++++++--
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 24 +-
.../SystemStatus.java} | 26 +--
.../db/conf/directories/DirectoryManager.java | 86 ++++----
.../iotdb/db/conf/directories/FolderManager.java | 18 +-
.../directories/strategy/DirectoryStrategy.java | 4 +-
.../db/consensus/DataRegionConsensusImpl.java | 2 +-
.../statemachine/DataRegionStateMachine.java | 27 ++-
.../statemachine/SchemaRegionStateMachine.java | 6 +
.../org/apache/iotdb/db/engine/StorageEngine.java | 2 +-
.../apache/iotdb/db/engine/StorageEngineV2.java | 8 +-
.../db/engine/compaction/CompactionScheduler.java | 6 +-
.../constant/InnerSequenceCompactionSelector.java | 8 +-
.../InnerUnsequenceCompactionSelector.java | 8 +-
.../sizetiered/SizeTieredCompactionSelector.java | 22 +-
.../compaction/task/CompactionRecoverTask.java | 3 +-
.../iotdb/db/engine/memtable/AbstractMemTable.java | 3 +
.../iotdb/db/engine/snapshot/SnapshotFileSet.java | 44 ++++
.../iotdb/db/engine/snapshot/SnapshotLoader.java | 144 ++++++++++--
.../db/engine/snapshot/SnapshotLogAnalyzer.java | 79 +++++++
.../iotdb/db/engine/snapshot/SnapshotLogger.java | 58 +++++
.../iotdb/db/engine/snapshot/SnapshotTaker.java | 243 ++++++++++++---------
.../exception/DirectoryNotLegalException.java | 7 +-
.../iotdb/db/engine/storagegroup/DataRegion.java | 169 +++++++-------
.../db/engine/storagegroup/TsFileManager.java | 14 +-
.../db/engine/storagegroup/TsFileProcessor.java | 42 ++--
.../engine/storagegroup/TsFileProcessorInfo.java | 6 +-
.../dataregion/StorageGroupManager.java | 4 +-
.../iotdb/db/localconfignode/LocalConfigNode.java | 17 +-
.../db/metadata/idtable/IDTableHashmapImpl.java | 2 +-
.../iotdb/db/metadata/logfile/MLogWriter.java | 25 ++-
.../schemaregion/SchemaRegionMemoryImpl.java | 3 +-
.../schemaregion/SchemaRegionSchemaFileImpl.java | 3 +-
.../org/apache/iotdb/db/mpp/plan/Coordinator.java | 2 +-
.../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 12 +-
.../iotdb/db/mpp/plan/constant/StatementType.java | 2 +-
.../plan/execution/config/ConfigTaskVisitor.java | 8 +
.../config/executor/ClusterConfigTaskExecutor.java | 43 +++-
.../config/executor/IConfigTaskExecutor.java | 2 +
.../executor/StandaloneConfigTaskExecutor.java | 88 ++++----
.../config/sys/LoadConfigurationTask.java | 42 ++++
.../iotdb/db/mpp/plan/parser/ASTVisitor.java | 115 +++++++---
.../db/mpp/plan/planner/OperatorTreeGenerator.java | 6 +-
.../plan/planner/plan/node/PlanGraphPrinter.java | 36 ++-
.../db/mpp/plan/scheduler/StandaloneScheduler.java | 8 +
.../db/mpp/plan/statement/StatementVisitor.java | 6 +
.../metadata/ShowChildNodesStatement.java | 8 +
.../metadata/ShowChildPathsStatement.java | 8 +
.../metadata/ShowTimeSeriesStatement.java | 8 +
.../statement/sys/LoadConfigurationStatement.java} | 42 ++--
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 36 +--
.../org/apache/iotdb/db/qp/logical/Operator.java | 2 -
.../db/qp/logical/sys/ShowPipeServerOperator.java | 38 ----
.../db/qp/logical/sys/StartPipeServerOperator.java | 38 ----
.../db/qp/logical/sys/StopPipeServerOperator.java | 38 ----
.../apache/iotdb/db/qp/physical/PhysicalPlan.java | 8 -
.../apache/iotdb/db/qp/physical/sys/ShowPlan.java | 1 -
.../db/qp/physical/sys/StopPipeServerPlan.java | 56 -----
.../apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java | 121 +++++-----
.../apache/iotdb/db/qp/utils/DatetimeUtils.java | 24 +-
.../org/apache/iotdb/db/rescon/SystemInfo.java | 10 +-
.../java/org/apache/iotdb/db/service/DataNode.java | 3 +-
.../db/service/DataNodeServerCommandLine.java | 39 ++--
.../java/org/apache/iotdb/db/service/IoTDB.java | 12 +-
.../apache/iotdb/db/service/IoTDBShutdownHook.java | 2 +-
.../java/org/apache/iotdb/db/service/NewIoTDB.java | 1 +
.../iotdb/db/service/RegionMigrateService.java | 4 +-
.../db/service/metrics/predefined/FileMetrics.java | 65 ++++--
.../service/thrift/impl/ClientRPCServiceImpl.java | 23 ++
.../impl/DataNodeInternalRPCServiceImpl.java | 24 +-
.../db/service/thrift/impl/TSServiceImpl.java | 21 ++
.../java/org/apache/iotdb/db/sync/SyncService.java | 127 +++++------
.../iotdb/db/sync/common/ISyncInfoFetcher.java | 8 -
.../iotdb/db/sync/common/LocalSyncInfoFetcher.java | 28 ---
.../org/apache/iotdb/db/sync/common/SyncInfo.java | 33 +--
.../db/sync/common/persistence/SyncLogReader.java | 13 --
.../db/sync/common/persistence/SyncLogWriter.java | 14 --
.../db/sync/sender/service/TransportHandler.java | 4 +-
.../db/sync/transport/client/ClientWrapper.java | 25 +--
...rtClient.java => IoTDBSinkTransportClient.java} | 52 ++---
...nsportServiceImpl.java => ReceiverManager.java} | 183 ++++++++++------
.../transport/server/TransportServerManager.java | 137 ------------
.../server/TransportServerThriftHandler.java | 71 ------
.../allocation/AbstractNodeAllocationStrategy.java | 8 +-
.../org/apache/iotdb/db/wal/buffer/WALBuffer.java | 15 +-
.../iotdb/db/wal/checkpoint/CheckpointManager.java | 9 +-
.../java/org/apache/iotdb/db/wal/node/WALNode.java | 2 +-
.../iotdb/db/wal/recover/WALNodeRecoverTask.java | 4 +-
.../iotdb/db/wal/recover/WALRecoverManager.java | 22 +-
.../SizeTieredCompactionSelectorTest.java | 66 ++++++
.../engine/modification/DeletionFileNodeTest.java | 2 +-
.../db/engine/snapshot/IoTDBSnapshotTest.java | 199 +++++++++++++++++
.../engine/storagegroup/FakedTsFileResource.java | 9 +-
.../iotdb/db/qp/physical/PhysicalPlanTest.java | 28 ---
.../db/qp/utils/DatetimeQueryDataSetUtilsTest.java | 6 +-
.../DataNodeInternalRPCServiceImplTest.java | 4 +-
.../db/sync/receiver/manager/SyncInfoTest.java | 1 -
.../db/sync/receiver/recovery/SyncLogTest.java | 4 -
...portServiceTest.java => SyncTransportTest.java} | 21 +-
.../apache/iotdb/db/utils/EnvironmentUtils.java | 20 +-
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 3 +
.../apache/iotdb/spark/db/EnvironmentUtils.java | 3 +-
.../src/main/thrift/confignode.thrift | 3 +
thrift-sync/README.md | 22 --
thrift-sync/pom.xml | 62 ------
thrift-sync/rpc-changelist.md | 181 ---------------
thrift-sync/src/main/thrift/transport.thrift | 63 ------
thrift/src/main/thrift/client.thrift | 33 +++
thrift/src/main/thrift/datanode.thrift | 2 +
.../iotdb/tsfile/utils/TsFileGeneratorUtils.java | 100 +++++----
192 files changed, 3541 insertions(+), 2397 deletions(-)
diff --cc server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/StandaloneScheduler.java
index 89a69f1022,5a1f6ee329..989cfa0165
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/StandaloneScheduler.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/StandaloneScheduler.java
@@@ -120,9 -121,16 +121,16 @@@ public class StandaloneScheduler implem
stateMachine.initialFragInstanceState(
instance.getId(), FragmentInstanceState.RUNNING));
this.stateTracker.start();
- LOGGER.info("{} state tracker starts", getLogHeader());
+ LOGGER.debug("{} state tracker starts", getLogHeader());
break;
case WRITE:
+ // reject non-query operations when system is read-only
+ if (IoTDBDescriptor.getInstance().getConfig().isReadOnly()) {
+ TSStatus failedStatus = new TSStatus(TSStatusCode.READ_ONLY_SYSTEM_ERROR.getStatusCode());
+ failedStatus.setMessage("Fail to do non-query operations because system is read-only.");
+ stateMachine.transitionToFailed(failedStatus);
+ return;
+ }
try {
for (FragmentInstance fragmentInstance : instances) {
PlanNode planNode = fragmentInstance.getFragment().getRoot();
diff --cc server/src/main/java/org/apache/iotdb/db/service/IoTDBShutdownHook.java
index f9ca0277c3,895273ccc5..3d2d457b5a
--- a/server/src/main/java/org/apache/iotdb/db/service/IoTDBShutdownHook.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/IoTDBShutdownHook.java
@@@ -50,12 -51,12 +50,12 @@@ public class IoTDBShutdownHook extends
}
WALManager.getInstance().deleteOutdatedWALFiles();
- if (IoTDB.isClusterMode()) {
+ if (IoTDBDescriptor.getInstance().getConfig().isClusterMode()) {
// This setting ensures that compaction work is not discarded
// even if there are frequent restarts
- DataRegionConsensusImpl.getInstance()
- .getAllConsensusGroupIds()
- .forEach(id -> DataRegionConsensusImpl.getInstance().triggerSnapshot(id));
+ // DataRegionConsensusImpl.getInstance()
+ // .getAllConsensusGroupIds()
+ // .forEach(id -> DataRegionConsensusImpl.getInstance().triggerSnapshot(id));
}
if (logger.isInfoEnabled()) {