You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xu...@apache.org on 2020/12/09 05:40:26 UTC
[iotdb] branch virtual_partition_2_merge created (now 4578a56)
This is an automated email from the ASF dual-hosted git repository.
xuekaifeng pushed a change to branch virtual_partition_2_merge
in repository https://gitbox.apache.org/repos/asf/iotdb.git.
at 4578a56 Merge branch 'master' of https://github.com/apache/iotdb into virtual_partition_2_merge
This branch includes the following new commits:
new 4578a56 Merge branch 'master' of https://github.com/apache/iotdb into virtual_partition_2_merge
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: Merge branch 'master' of
https://github.com/apache/iotdb into virtual_partition_2_merge
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuekaifeng pushed a commit to branch virtual_partition_2_merge
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 4578a56818c35e7cb39a316a30f29d115f0ca596
Merge: aa6d629 bbbd88b
Author: 151250176 <15...@smail.nju.edu.cn>
AuthorDate: Wed Dec 9 13:39:05 2020 +0800
Merge branch 'master' of https://github.com/apache/iotdb into virtual_partition_2_merge
# Conflicts:
# server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
# server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
.gitignore | 6 +
.travis.yml | 13 +-
.../java/org/apache/iotdb/cli/AbstractScript.java | 5 +
client-cpp/README.md | 14 +-
client-cpp/client-cpp-example/pom.xml | 1 -
.../client-cpp-example/src/SessionExample.cpp | 2 +-
client-cpp/pom.xml | 17 +-
.../src/main/{IOTDBSession.cpp => Session.cpp} | 72 +-
client-cpp/src/main/{IOTDBSession.h => Session.h} | 54 +-
client-cpp/src/test/CMakeLists.txt | 1 +
client-cpp/src/test/cpp/sessionIT.cpp | 2 +-
client-cpp/src/test/main.cpp | 2 +-
client-py/readme.md | 11 +-
cluster/pom.xml | 183 ++
cluster/src/assembly/cluster.xml | 27 +
.../resources/conf/iotdb-cluster.properties | 151 ++
cluster/src/assembly/resources/sbin/add-node.bat | 108 +
cluster/src/assembly/resources/sbin/add-node.sh | 76 +
cluster/src/assembly/resources/sbin/nodetool.bat | 58 +
cluster/src/assembly/resources/sbin/nodetool.sh | 48 +
cluster/src/assembly/resources/sbin/start-node.bat | 129 ++
cluster/src/assembly/resources/sbin/start-node.sh | 100 +
cluster/src/assembly/resources/sbin/stop-node.bat | 27 +
cluster/src/assembly/resources/sbin/stop-node.sh | 39 +
.../java/org/apache/iotdb/cluster/ClientMain.java | 461 +++++
.../iotdb/cluster/ClusterFileFlushPolicy.java | 67 +
.../java/org/apache/iotdb/cluster/ClusterMain.java | 319 +++
.../apache/iotdb/cluster/RemoteTsFileResource.java | 174 ++
.../iotdb/cluster/client/DataClientProvider.java | 79 +
.../cluster/client/async/AsyncClientFactory.java | 62 +
.../cluster/client/async/AsyncClientPool.java | 234 +++
.../cluster/client/async/AsyncDataClient.java | 148 ++
.../client/async/AsyncDataHeartbeatClient.java | 69 +
.../cluster/client/async/AsyncMetaClient.java | 125 ++
.../client/async/AsyncMetaHeartbeatClient.java | 70 +
.../cluster/client/sync/SyncClientAdaptor.java | 456 ++++
.../cluster/client/sync/SyncClientFactory.java | 31 +-
.../iotdb/cluster/client/sync/SyncClientPool.java | 137 ++
.../iotdb/cluster/client/sync/SyncDataClient.java | 105 +
.../client/sync/SyncDataHeartbeatClient.java | 71 +
.../iotdb/cluster/client/sync/SyncMetaClient.java | 80 +
.../client/sync/SyncMetaHeartbeatClient.java | 71 +
.../apache/iotdb/cluster/config/ClusterConfig.java | 444 ++++
.../iotdb/cluster/config/ClusterConstant.java | 28 +-
.../iotdb/cluster/config/ClusterDescriptor.java | 371 ++++
.../iotdb/cluster/config/ConsistencyLevel.java | 69 +
.../iotdb/cluster/exception/AddSelfException.java | 19 +-
.../exception/BadSeedUrlFormatException.java | 17 +-
.../exception/CheckConsistencyException.java | 22 +-
.../exception/ConfigInconsistentException.java | 17 +-
.../cluster/exception/EmptyIntervalException.java | 17 +-
.../cluster/exception/EntryCompactedException.java | 18 +-
.../exception/EntryUnavailableException.java | 17 +-
.../GetEntriesWrongParametersException.java | 16 +-
.../cluster/exception/LeaderUnknownException.java | 20 +-
.../cluster/exception/LogExecutionException.java | 16 +-
.../cluster/exception/MemberReadOnlyException.java | 19 +-
.../cluster/exception/NoHeaderNodeException.java | 20 +-
.../cluster/exception/NotInSameGroupException.java | 20 +-
.../cluster/exception/NotManagedSlotException.java | 19 +-
.../PartitionTableUnavailableException.java | 21 +-
.../iotdb/cluster/exception/PullFileException.java | 15 +-
.../cluster/exception/QueryTimeOutException.java | 16 +-
.../cluster/exception/ReaderNotFoundException.java | 16 +-
.../cluster/exception/RequestTimeOutException.java | 18 +-
.../exception/SnapshotInstallationException.java | 22 +-
.../exception/StartUpCheckFailureException.java | 20 +-
.../exception/TruncateCommittedEntryException.java | 18 +-
.../cluster/exception/UnknownLogTypeException.java | 19 +-
.../exception/UnsupportedPlanException.java | 18 +-
.../iotdb/cluster/log/CommitLogCallback.java | 29 +-
.../apache/iotdb/cluster/log/CommitLogTask.java | 68 +
.../org/apache/iotdb/cluster/log/HardState.java | 133 ++
.../java/org/apache/iotdb/cluster/log/Log.java | 130 ++
.../org/apache/iotdb/cluster/log/LogApplier.java | 22 +-
.../apache/iotdb/cluster/log/LogDispatcher.java | 424 ++++
.../org/apache/iotdb/cluster/log/LogParser.java | 100 +
.../org/apache/iotdb/cluster/log/Snapshot.java | 71 +
.../iotdb/cluster/log/StableEntryManager.java | 55 +
.../cluster/log/applier/AsyncDataLogApplier.java | 270 +++
.../iotdb/cluster/log/applier/BaseApplier.java | 150 ++
.../iotdb/cluster/log/applier/DataLogApplier.java | 114 +
.../iotdb/cluster/log/applier/MetaLogApplier.java | 70 +
.../iotdb/cluster/log/catchup/CatchUpTask.java | 359 ++++
.../iotdb/cluster/log/catchup/LogCatchUpTask.java | 343 ++++
.../cluster/log/catchup/SnapshotCatchUpTask.java | 153 ++
.../iotdb/cluster/log/logtypes/AddNodeLog.java | 93 +
.../iotdb/cluster/log/logtypes/CloseFileLog.java | 119 ++
.../cluster/log/logtypes/EmptyContentLog.java | 63 +
.../iotdb/cluster/log/logtypes/LargeTestLog.java | 78 +
.../cluster/log/logtypes/PhysicalPlanLog.java | 112 +
.../iotdb/cluster/log/logtypes/RemoveNodeLog.java | 87 +
.../cluster/log/manage/CommittedEntryManager.java | 236 +++
.../manage/FilePartitionedSnapshotLogManager.java | 205 ++
.../log/manage/MetaSingleSnapshotLogManager.java | 84 +
.../log/manage/PartitionedSnapshotLogManager.java | 100 +
.../iotdb/cluster/log/manage/RaftLogManager.java | 949 +++++++++
.../log/manage/UnCommittedEntryManager.java | 259 +++
.../log/manage/serializable/LogManagerMeta.java | 136 ++
.../serializable/SyncLogDequeSerializer.java | 1329 ++++++++++++
.../iotdb/cluster/log/snapshot/FileSnapshot.java | 621 ++++++
.../cluster/log/snapshot/MetaSimpleSnapshot.java | 300 +++
.../cluster/log/snapshot/PartitionedSnapshot.java | 212 ++
.../cluster/log/snapshot/PullSnapshotTask.java | 231 +++
.../log/snapshot/PullSnapshotTaskDescriptor.java | 122 ++
.../cluster/log/snapshot/SnapshotFactory.java | 17 +-
.../cluster/log/snapshot/SnapshotInstaller.java | 22 +-
.../log/snapshot/TimeseriesSchemaSnapshot.java | 19 +-
.../apache/iotdb/cluster/metadata/CMManager.java | 1428 +++++++++++++
.../apache/iotdb/cluster/metadata/MetaPuller.java | 207 ++
.../cluster/partition/NodeAdditionResult.java | 25 +-
.../iotdb/cluster/partition/NodeRemovalResult.java | 37 +-
.../iotdb/cluster/partition/PartitionGroup.java | 65 +
.../iotdb/cluster/partition/PartitionTable.java | 130 ++
.../iotdb/cluster/partition/slot/SlotManager.java | 315 +++
.../partition/slot/SlotNodeAdditionResult.java | 24 +-
.../partition/slot/SlotNodeRemovalResult.java | 27 +-
.../cluster/partition/slot/SlotPartitionTable.java | 508 +++++
.../iotdb/cluster/partition/slot/SlotStrategy.java | 36 +
.../cluster/query/ClusterConcatPathOptimizer.java | 26 +-
.../cluster/query/ClusterDataQueryExecutor.java | 103 +
.../cluster/query/ClusterPhysicalGenerator.java | 104 +
.../iotdb/cluster/query/ClusterPlanExecutor.java | 543 +++++
.../iotdb/cluster/query/ClusterPlanRouter.java | 365 ++++
.../apache/iotdb/cluster/query/ClusterPlanner.java | 57 +
.../iotdb/cluster/query/ClusterQueryRouter.java | 88 +
.../iotdb/cluster/query/LocalQueryExecutor.java | 690 +++++++
.../iotdb/cluster/query/RemoteQueryContext.java | 72 +
.../query/aggregate/ClusterAggregateExecutor.java | 96 +
.../cluster/query/aggregate/ClusterAggregator.java | 224 ++
.../query/dataset/ClusterAlignByDeviceDataSet.java | 60 +
.../cluster/query/fill/ClusterFillExecutor.java | 61 +
.../cluster/query/fill/ClusterLinearFill.java | 70 +
.../cluster/query/fill/ClusterPreviousFill.java | 222 ++
.../cluster/query/fill/PreviousFillArguments.java | 67 +
.../iotdb/cluster/query/filter/SlotSgFilter.java | 34 +-
.../cluster/query/filter/SlotTsFileFilter.java | 57 +
.../groupby/ClusterGroupByNoVFilterDataSet.java | 59 +
.../groupby/ClusterGroupByVFilterDataSet.java | 74 +
.../query/groupby/MergeGroupByExecutor.java | 134 ++
.../query/groupby/RemoteGroupByExecutor.java | 144 ++
.../query/last/ClusterLastQueryExecutor.java | 249 +++
.../cluster/query/manage/ClusterQueryManager.java | 130 ++
.../iotdb/cluster/query/manage/NodeStatus.java | 87 +
.../cluster/query/manage/QueryCoordinator.java | 169 ++
.../cluster/query/reader/ClusterReaderFactory.java | 571 +++++
.../cluster/query/reader/ClusterTimeGenerator.java | 75 +
.../iotdb/cluster/query/reader/DataSourceInfo.java | 249 +++
.../iotdb/cluster/query/reader/EmptyReader.java | 170 ++
.../cluster/query/reader/ManagedMergeReader.java | 93 +
.../cluster/query/reader/MergedReaderByTime.java | 33 +-
.../reader/RemoteSeriesReaderByTimestamp.java | 105 +
.../query/reader/RemoteSimpleSeriesReader.java | 159 ++
.../apache/iotdb/cluster/server/ClientServer.java | 322 +++
.../iotdb/cluster/server/DataClusterServer.java | 860 ++++++++
.../iotdb/cluster/server/HardLinkCleaner.java | 109 +
.../iotdb/cluster/server/MetaClusterServer.java | 346 ++++
.../apache/iotdb/cluster/server/NodeCharacter.java | 19 +-
.../apache/iotdb/cluster/server/NodeReport.java | 183 ++
.../java/org/apache/iotdb/cluster/server/Peer.java | 70 +
.../cluster/server/PullSnapshotHintService.java | 141 ++
.../apache/iotdb/cluster/server/RaftServer.java | 241 +++
.../org/apache/iotdb/cluster/server/Response.java | 59 +
.../iotdb/cluster/server/StoppedMemberManager.java | 153 ++
.../org/apache/iotdb/cluster/server/Timer.java | 261 +++
.../handlers/caller/AppendGroupEntryHandler.java | 147 ++
.../handlers/caller/AppendNodeEntryHandler.java | 171 ++
.../server/handlers/caller/ElectionHandler.java | 127 ++
.../server/handlers/caller/GenericHandler.java | 91 +
.../caller/GetChildNodeNextLevelPathHandler.java | 57 +
.../handlers/caller/GetNodesListHandler.java | 58 +
.../caller/GetTimeseriesSchemaHandler.java | 57 +
.../server/handlers/caller/HeartbeatHandler.java | 132 ++
.../server/handlers/caller/JoinClusterHandler.java | 63 +
.../server/handlers/caller/LogCatchUpHandler.java | 104 +
.../handlers/caller/LogCatchUpInBatchHandler.java | 104 +
.../server/handlers/caller/NodeStatusHandler.java | 54 +
.../handlers/caller/PreviousFillHandler.java | 78 +
.../caller/PullMeasurementSchemaHandler.java | 69 +
.../handlers/caller/PullSnapshotHandler.java | 78 +
.../caller/PullTimeseriesSchemaHandler.java | 69 +
.../handlers/caller/SnapshotCatchUpHandler.java | 62 +
.../server/handlers/caller/package-info.java | 15 +-
.../handlers/forwarder/ForwardPlanHandler.java | 66 +
.../server/heartbeat/DataHeartbeatServer.java | 82 +
.../server/heartbeat/DataHeartbeatThread.java | 74 +
.../cluster/server/heartbeat/HeartbeatServer.java | 212 ++
.../cluster/server/heartbeat/HeartbeatThread.java | 383 ++++
.../server/heartbeat/MetaHeartbeatServer.java | 83 +
.../server/heartbeat/MetaHeartbeatThread.java | 72 +
.../cluster/server/member/DataGroupMember.java | 828 ++++++++
.../cluster/server/member/MetaGroupMember.java | 2172 ++++++++++++++++++++
.../iotdb/cluster/server/member/RaftMember.java | 1869 +++++++++++++++++
.../iotdb/cluster/server/member/package-info.java | 17 +-
.../cluster/server/service/BaseAsyncService.java | 159 ++
.../cluster/server/service/BaseSyncService.java | 175 ++
.../cluster/server/service/DataAsyncService.java | 349 ++++
.../cluster/server/service/DataSyncService.java | 350 ++++
.../cluster/server/service/MetaAsyncService.java | 202 ++
.../cluster/server/service/MetaSyncService.java | 194 ++
.../iotdb/cluster/server/service/package-info.java | 17 +-
.../apache/iotdb/cluster/utils/ClientUtils.java | 70 +
.../iotdb/cluster/utils/ClusterConsistent.java | 23 +-
.../apache/iotdb/cluster/utils/ClusterNode.java | 71 +
.../iotdb/cluster/utils/ClusterQueryUtils.java | 68 +
.../apache/iotdb/cluster/utils/ClusterUtils.java | 328 +++
.../org/apache/iotdb/cluster/utils/IOUtils.java | 87 +
.../apache/iotdb/cluster/utils/PartitionUtils.java | 401 ++++
.../apache/iotdb/cluster/utils/PlanSerializer.java | 51 +
.../apache/iotdb/cluster/utils/StatusUtils.java | 219 ++
.../cluster/utils/nodetool/ClusterMonitor.java | 179 ++
.../utils/nodetool/ClusterMonitorMBean.java | 84 +
.../iotdb/cluster/utils/nodetool/NodeTool.java | 47 +
.../iotdb/cluster/utils/nodetool/Printer.java | 24 +-
.../cluster/utils/nodetool/function/Host.java | 70 +
.../cluster/utils/nodetool/function/LogView.java | 82 +
.../utils/nodetool/function/NodeToolCmd.java | 102 +
.../cluster/utils/nodetool/function/Partition.java | 85 +
.../cluster/utils/nodetool/function/Ring.java | 42 +
.../cluster/utils/nodetool/function/Status.java | 44 +
.../cluster/client/DataClientProviderTest.java | 73 +
.../cluster/client/async/AsyncClientPoolTest.java | 188 ++
.../cluster/client/async/AsyncDataClientTest.java | 61 +
.../client/async/AsyncDataHeartbeatClientTest.java | 41 +
.../cluster/client/async/AsyncMetaClientTest.java | 61 +
.../client/async/AsyncMetaHeartbeatClientTest.java | 41 +
.../cluster/client/sync/SyncClientAdaptorTest.java | 366 ++++
.../cluster/client/sync/SyncClientPoolTest.java | 153 ++
.../cluster/client/sync/SyncDataClientTest.java | 80 +
.../client/sync/SyncDataHeartbeatClientTest.java | 63 +
.../cluster/client/sync/SyncMetaClientTest.java | 62 +
.../client/sync/SyncMetaHeartbeatClientTest.java | 63 +
.../iotdb/cluster/common/EnvironmentUtils.java | 218 ++
.../org/apache/iotdb/cluster/common/IoTDBTest.java | 172 ++
.../iotdb/cluster/common/TestAsyncClient.java | 35 +-
.../cluster/common/TestAsyncClientFactory.java | 50 +
.../iotdb/cluster/common/TestAsyncDataClient.java | 190 ++
.../iotdb/cluster/common/TestAsyncMetaClient.java | 37 +-
.../iotdb/cluster/common/TestDataGroupMember.java | 46 +
.../apache/iotdb/cluster/common/TestException.java | 16 +-
.../org/apache/iotdb/cluster/common/TestLog.java | 67 +
.../iotdb/cluster/common/TestLogApplier.java | 65 +
.../iotdb/cluster/common/TestLogManager.java | 29 +-
.../cluster/common/TestManagedSeriesReader.java | 91 +
.../iotdb/cluster/common/TestMetaGroupMember.java | 29 +-
.../cluster/common/TestPartitionedLogManager.java | 48 +
.../apache/iotdb/cluster/common/TestSnapshot.java | 110 +
.../iotdb/cluster/common/TestSyncClient.java | 29 +-
.../cluster/common/TestSyncClientFactory.java | 71 +
.../org/apache/iotdb/cluster/common/TestUtils.java | 388 ++++
.../cluster/integration/BaseSingleNodeTest.java | 80 +
.../iotdb/cluster/integration/SingleNodeTest.java | 93 +
.../iotdb/cluster/log/CommitLogCallbackTest.java | 51 +
.../iotdb/cluster/log/CommitLogTaskTest.java | 72 +
.../apache/iotdb/cluster/log/HardStateTest.java | 36 +-
.../iotdb/cluster/log/LogDispatcherTest.java | 254 +++
.../apache/iotdb/cluster/log/LogParserTest.java | 99 +
.../log/applier/AsyncDataLogApplierTest.java | 166 ++
.../cluster/log/applier/DataLogApplierTest.java | 316 +++
.../cluster/log/applier/MetaLogApplierTest.java | 124 ++
.../iotdb/cluster/log/catchup/CatchUpTaskTest.java | 393 ++++
.../cluster/log/catchup/LogCatchUpTaskTest.java | 289 +++
.../log/catchup/SnapshotCatchUpTaskTest.java | 270 +++
.../cluster/log/logtypes/SerializeLogTest.java | 128 ++
.../log/manage/CommittedEntryManagerTest.java | 429 ++++
.../FilePartitionedSnapshotLogManagerTest.java | 101 +
.../manage/MetaSingleSnapshotLogManagerTest.java | 91 +
.../cluster/log/manage/RaftLogManagerTest.java | 1412 +++++++++++++
.../log/manage/UnCommittedEntryManagerTest.java | 442 ++++
.../serializable/SyncLogDequeSerializerTest.java | 595 ++++++
.../cluster/log/snapshot/DataSnapshotTest.java | 163 ++
.../cluster/log/snapshot/FileSnapshotTest.java | 300 +++
.../log/snapshot/MetaSimpleSnapshotTest.java | 187 ++
.../log/snapshot/PartitionedSnapshotTest.java | 116 ++
.../snapshot/PullSnapshotTaskDescriptorTest.java | 62 +
.../cluster/log/snapshot/PullSnapshotTaskTest.java | 299 +++
.../iotdb/cluster/log/snapshot/SimpleSnapshot.java | 132 ++
.../iotdb/cluster/partition/MManagerWhiteBox.java | 53 +
.../iotdb/cluster/partition/SlotManagerTest.java | 149 ++
.../cluster/partition/SlotPartitionTableTest.java | 533 +++++
.../apache/iotdb/cluster/query/BaseQueryTest.java | 115 ++
.../query/ClusterAggregateExecutorTest.java | 129 ++
.../query/ClusterDataQueryExecutorTest.java | 66 +
.../cluster/query/ClusterFillExecutorTest.java | 115 ++
.../query/ClusterPhysicalGeneratorTest.java | 63 +
.../cluster/query/ClusterPlanExecutorTest.java | 83 +
.../iotdb/cluster/query/ClusterPlannerTest.java | 54 +
.../cluster/query/ClusterQueryRouterTest.java | 259 +++
.../ClusterGroupByNoVFilterDataSetTest.java | 83 +
.../groupby/ClusterGroupByVFilterDataSetTest.java | 94 +
.../query/groupby/MergeGroupByExecutorTest.java | 98 +
.../query/groupby/RemoteGroupByExecutorTest.java | 153 ++
.../query/manage/ClusterQueryManagerTest.java | 192 ++
.../cluster/query/manage/QueryCoordinatorTest.java | 117 ++
.../query/reader/ClusterTimeGeneratorTest.java | 70 +
.../cluster/query/reader/DatasourceInfoTest.java | 77 +
.../reader/RemoteSeriesReaderByTimestampTest.java | 202 ++
.../query/reader/RemoteSimpleSeriesReaderTest.java | 206 ++
.../caller/AppendGroupEntryHandlerTest.java | 159 ++
.../caller/AppendNodeEntryHandlerTest.java | 172 ++
.../handlers/caller/ElectionHandlerTest.java | 133 ++
.../server/handlers/caller/GenericHandlerTest.java | 57 +
.../handlers/caller/HeartbeatHandlerTest.java | 99 +
.../handlers/caller/JoinClusterHandlerTest.java | 66 +
.../handlers/caller/LogCatchUpHandlerTest.java | 125 ++
.../caller/PullMeasurementSchemaHandlerTest.java | 89 +
.../handlers/caller/PullSnapshotHandlerTest.java | 81 +
.../caller/SnapshotCatchUpHandlerTest.java | 57 +
.../handlers/forwarder/ForwardPlanHandlerTest.java | 57 +
.../server/heartbeat/DataHeartbeatThreadTest.java | 158 ++
.../server/heartbeat/HeartbeatThreadTest.java | 246 +++
.../server/heartbeat/MetaHeartbeatThreadTest.java | 214 ++
.../cluster/server/member/DataGroupMemberTest.java | 1047 ++++++++++
.../iotdb/cluster/server/member/MemberTest.java | 418 ++++
.../cluster/server/member/MetaGroupMemberTest.java | 1211 +++++++++++
.../iotdb/cluster/utils/SerializeUtilTest.java | 179 ++
{server => cluster}/src/test/resources/logback.xml | 24 +-
compile-tools/README.md | 6 +-
docs/SystemDesign/Client/RPC.md | 1 -
docs/UserGuide/Operation Manual/Administration.md | 26 +-
.../DDL Data Definition Language.md | 177 +-
.../DML Data Manipulation Language.md | 665 ++++--
docs/UserGuide/Server/Cluster Setup.md | 50 +-
docs/UserGuide/Server/Config Manual.md | 29 +
docs/UserGuide/System Tools/NodeTool.md | 77 +-
docs/zh/SystemDesign/Client/RPC.md | 1 -
.../DDL Data Definition Language.md | 129 +-
.../DML Data Manipulation Language.md | 685 ++++--
docs/zh/UserGuide/Server/Cluster Setup.md | 51 +-
docs/zh/UserGuide/Server/Config Manual.md | 38 +-
docs/zh/UserGuide/System Tools/NodeTool.md | 84 +-
.../org/apache/iotdb/jdbc/IoTDBConnection.java | 16 +-
.../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 54 +-
pom.xml | 1 +
.../resources/conf/iotdb-engine.properties | 9 +
server/src/assembly/resources/conf/logback.xml | 6 +-
.../java/org/apache/iotdb/db/auth/entity/Role.java | 8 +
.../java/org/apache/iotdb/db/auth/entity/User.java | 12 +
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 17 +-
.../org/apache/iotdb/db/conf/IoTDBConstant.java | 3 +
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 8 +
.../org/apache/iotdb/db/engine/StorageEngine.java | 51 +-
.../no/NoCompactionTsFileManagement.java | 2 +-
.../engine/compaction/utils/CompactionUtils.java | 5 +-
.../iotdb/db/engine/flush/MemTableFlushTask.java | 4 +-
.../db/engine/memtable/PrimitiveMemTable.java | 4 +
.../iotdb/db/engine/merge/task/MergeFileTask.java | 32 +-
.../engine/storagegroup/StorageGroupProcessor.java | 231 +--
.../db/engine/storagegroup/TsFileProcessor.java | 165 +-
.../db/engine/storagegroup/TsFileResource.java | 174 +-
...onException.java => BatchProcessException.java} | 13 +-
.../db/exception/TsFileProcessorException.java | 2 +-
...eryProcessException.java => PathException.java} | 18 +-
.../db/exception/query/QueryProcessException.java | 4 +
.../org/apache/iotdb/db/metadata/MManager.java | 217 +-
.../java/org/apache/iotdb/db/metadata/MTree.java | 10 +-
.../org/apache/iotdb/db/metadata/MetaUtils.java | 10 +-
.../org/apache/iotdb/db/metadata/PartialPath.java | 40 +-
.../org/apache/iotdb/db/monitor/StatMonitor.java | 3 +-
.../main/java/org/apache/iotdb/db/qp/Planner.java | 2 -
.../apache/iotdb/db/qp/executor/IPlanExecutor.java | 4 +-
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 264 +--
.../apache/iotdb/db/qp/physical/PhysicalPlan.java | 31 +-
.../iotdb/db/qp/physical/crud/InsertPlan.java | 22 +-
.../iotdb/db/qp/physical/crud/InsertRowPlan.java | 119 +-
.../db/qp/physical/crud/InsertTabletPlan.java | 65 +-
.../db/qp/physical/crud/RawDataQueryPlan.java | 3 +
.../qp/physical/sys/CreateMultiTimeSeriesPlan.java | 117 +-
.../apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java | 2 +-
.../iotdb/db/qp/strategy/PhysicalGenerator.java | 19 +-
.../qp/strategy/optimizer/ConcatPathOptimizer.java | 4 +-
.../iotdb/db/query/control/TracingManager.java | 2 +-
.../db/query/dataset/ShowTimeseriesDataSet.java | 13 +-
.../dataset/groupby/GroupByEngineDataSet.java | 7 +-
.../query/dataset/groupby/GroupByFillDataSet.java | 2 +-
.../groupby/GroupByWithValueFilterDataSet.java | 5 +
.../db/query/executor/AggregationExecutor.java | 1 +
.../iotdb/db/query/executor/LastQueryExecutor.java | 34 +-
.../db/query/executor/RawDataQueryExecutor.java | 2 +-
.../iotdb/db/query/reader/chunk/MemPageReader.java | 7 +-
.../reader/series/BaseManagedSeriesReader.java} | 29 +-
.../reader/series/SeriesReaderByTimestamp.java | 4 +
.../org/apache/iotdb/db/service/TSServiceImpl.java | 193 +-
.../db/service/thrift/ThriftServiceThread.java | 31 +-
.../iotdb/db/sync/sender/transfer/SyncClient.java | 14 +-
.../iotdb/db/tools/TsFileResourcePrinter.java | 13 +-
.../org/apache/iotdb/db/utils/CommonUtils.java | 60 +-
.../org/apache/iotdb/db/utils/FileLoaderUtils.java | 2 +
.../org/apache/iotdb/db/utils/SchemaUtils.java | 20 +-
.../org/apache/iotdb/db/utils/SerializeUtils.java | 280 ++-
.../org/apache/iotdb/db/writelog/io/LogWriter.java | 9 +-
.../db/writelog/node/ExclusiveWriteLogNode.java | 154 +-
.../writelog/recover/TsFileRecoverPerformer.java | 24 +-
.../iotdb/db/engine/merge/MergeOverLapTest.java | 9 +-
.../apache/iotdb/db/engine/merge/MergeTest.java | 9 +-
.../storagegroup/StorageGroupProcessorTest.java | 13 +-
.../engine/storagegroup/TsFileProcessorTest.java | 8 +-
.../iotdb/db/integration/IoTDBCompleteIT.java | 4 +
.../iotdb/db/integration/IoTDBQuotedPathIT.java | 4 +-
.../db/integration/IoTDBRpcCompressionIT.java | 153 ++
.../iotdb/db/integration/IoTDBSimpleQueryIT.java | 6 +-
.../iotdb/db/metadata/MManagerBasicTest.java | 2 +-
.../org/apache/iotdb/db/metadata/MTreeTest.java | 13 +-
.../apache/iotdb/db/metadata/MetaUtilsTest.java | 6 +-
.../reader/series/SeriesAggregateReaderTest.java | 3 +-
.../reader/series/SeriesReaderByTimestampTest.java | 3 +-
.../db/query/reader/series/SeriesReaderTest.java | 3 +-
.../query/reader/series/SeriesReaderTestUtil.java | 16 +-
.../db/sync/receiver/load/FileLoaderTest.java | 7 +-
.../apache/iotdb/db/writelog/WriteLogNodeTest.java | 11 +-
.../db/writelog/recover/SeqTsFileRecoverTest.java | 3 -
server/src/test/resources/logback.xml | 2 +-
.../org/apache/iotdb/rpc/AutoResizingBuffer.java | 80 +
.../iotdb/rpc/AutoScalingBufferReadTransport.java | 79 +
.../iotdb/rpc/AutoScalingBufferWriteTransport.java | 63 +
.../apache/iotdb/rpc/IoTDBConnectionException.java | 2 +-
...nectionException.java => NonOpenTransport.java} | 27 +-
...cutionException.java => RedirectException.java} | 33 +-
...atementExecutionException.java => RpcStat.java} | 33 +-
.../org/apache/iotdb/rpc/RpcTransportFactory.java | 56 +
.../main/java/org/apache/iotdb/rpc/RpcUtils.java | 66 +-
.../iotdb/rpc/StatementExecutionException.java | 2 +-
.../rpc/TCompressedElasticFramedTransport.java | 113 +
.../apache/iotdb/rpc/TElasticFramedTransport.java | 145 ++
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 3 +
.../iotdb/rpc/TSnappyElasticFramedTransport.java | 80 +
.../rpc/TimeoutChangeableTFastFramedTransport.java | 55 +
.../TimeoutChangeableTSnappyFramedTransport.java | 54 +
...eption.java => TimeoutChangeableTransport.java} | 19 +-
.../main/java/org/apache/iotdb/session/Config.java | 2 +
.../java/org/apache/iotdb/session/Session.java | 1343 ++++++------
.../apache/iotdb/session/SessionConnection.java | 624 ++++++
.../java/org/apache/iotdb/session/SessionUT.java | 4 +-
thrift/src/main/thrift/cluster.thrift | 39 +-
thrift/src/main/thrift/rpc.thrift | 6 +
tsfile/format-changelist.md | 5 +
.../iotdb/tsfile/file/footer/ChunkGroupFooter.java | 47 +-
.../iotdb/tsfile/read/TsFileSequenceReader.java | 12 +
.../tsfile/read/query/dataset/QueryDataSet.java | 14 +-
.../iotdb/tsfile/read/reader/page/PageReader.java | 7 +-
.../iotdb/tsfile/utils/ReadWriteIOUtils.java | 116 +-
.../apache/iotdb/tsfile/write/TsFileWriter.java | 2 +-
.../write/writer/RestorableTsFileIOWriter.java | 15 +
.../iotdb/tsfile/write/writer/TsFileIOWriter.java | 21 +-
.../org/apache/iotdb/tsfile/utils/RecordUtils.java | 2 +-
445 files changed, 57291 insertions(+), 3060 deletions(-)
diff --cc server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 366d8dd,b620ab9..e324323
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@@ -2104,19 -2100,12 +2110,28 @@@ public class IoTDBConfig
this.defaultIndexWindowRange = defaultIndexWindowRange;
}
+ public boolean isEnableVirtualPartition() {
+ return enableVirtualPartition;
+ }
+
+ public void setEnableVirtualPartition(boolean enableVirtualPartition) {
+ this.enableVirtualPartition = enableVirtualPartition;
+ }
+
+ public int getVirtualPartitionNum() {
+ return virtualPartitionNum;
+ }
+
+ public void setVirtualPartitionNum(int virtualPartitionNum) {
+ this.virtualPartitionNum = virtualPartitionNum;
+ }
++
+ public boolean isRpcAdvancedCompressionEnable() {
+ return rpcAdvancedCompressionEnable;
+ }
+
+ public void setRpcAdvancedCompressionEnable(boolean rpcAdvancedCompressionEnable) {
+ this.rpcAdvancedCompressionEnable = rpcAdvancedCompressionEnable;
+ RpcTransportFactory.setUseSnappy(this.rpcAdvancedCompressionEnable);
+ }
}
diff --cc server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
index 15e8efa,781836f..71290f5
--- a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
@@@ -53,8 -53,7 +53,8 @@@ import org.apache.iotdb.db.engine.stora
import org.apache.iotdb.db.engine.storagegroup.StorageGroupProcessor.TimePartitionFilter;
import org.apache.iotdb.db.engine.storagegroup.TsFileProcessor;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.engine.storagegroup.virtualSg.VirtualStorageGroupManager;
- import org.apache.iotdb.db.exception.BatchInsertionException;
+ import org.apache.iotdb.db.exception.BatchProcessException;
import org.apache.iotdb.db.exception.LoadFileException;
import org.apache.iotdb.db.exception.ShutdownException;
import org.apache.iotdb.db.exception.StorageEngineException;
@@@ -373,65 -324,56 +373,92 @@@ public class StorageEngine implements I
try {
StorageGroupMNode storageGroupMNode = IoTDB.metaManager.getStorageGroupNodeByPath(path);
storageGroupPath = storageGroupMNode.getPartialPath();
- StorageGroupProcessor processor = processorMap.get(storageGroupPath);
- if (processor == null) {
+ return getStorageGroupProcessorByPath(storageGroupPath, storageGroupMNode);
+ } catch (StorageGroupProcessorException | MetadataException e) {
+ throw new StorageEngineException(e);
+ }
+ }
+
+ /**
+ * This method is for insert and query or sth like them, this may get a virtual storage group
+ *
+ * @param path device path
+ * @return storage group processor
+ */
+ public StorageGroupProcessor getProcessor(PartialPath path) throws StorageEngineException {
+ try {
+ StorageGroupMNode storageGroupMNode = IoTDB.metaManager.getStorageGroupNodeByPath(path);
+ return getStorageGroupProcessorByPath(path, storageGroupMNode);
+ } catch (StorageGroupProcessorException | MetadataException e) {
+ throw new StorageEngineException(e);
+ }
+ }
+
+ // get storage group processor by partial path
+ private StorageGroupProcessor getStorageGroupProcessorByPath(PartialPath storageGroupPath,
+ StorageGroupMNode storageGroupMNode)
+ throws StorageGroupProcessorException, StorageEngineException {
+ VirtualStorageGroupManager virtualStorageGroupManager = processorMap.get(storageGroupMNode.getPartialPath());
+ if (virtualStorageGroupManager == null) {
+ // if finish recover
+ if (isAllSgReady.get()) {
+ waitAllSgReady(storageGroupPath);
- // if finish recover
- if (isAllSgReady.get()) {
- synchronized (storageGroupMNode) {
- processor = processorMap.get(storageGroupPath);
- if (processor == null) {
- logger.info("construct a processor instance, the storage group is {}, Thread is {}",
- storageGroupPath, Thread.currentThread().getId());
- processor = new StorageGroupProcessor(systemDir, storageGroupPath.getFullPath(),
- fileFlushPolicy);
- processor.setDataTTL(storageGroupMNode.getDataTTL());
- processor.setCustomFlushListeners(customFlushListeners);
- processor.setCustomCloseFileListeners(customCloseFileListeners);
- processorMap.put(storageGroupPath, processor);
- }
+ synchronized (storageGroupMNode) {
+ virtualStorageGroupManager = processorMap.get(storageGroupMNode.getPartialPath());
+ if (virtualStorageGroupManager == null) {
+ virtualStorageGroupManager = new VirtualStorageGroupManager();
+ processorMap.put(storageGroupMNode.getPartialPath(), virtualStorageGroupManager);
}
}
+ } else {
+ // not finished recover, refuse the request
+ throw new StorageEngineException(
+ "the sg " + storageGroupMNode.getPartialPath() + " may not ready now, please wait and retry later",
+ TSStatusCode.STORAGE_GROUP_NOT_READY.getStatusCode());
}
- return processor;
- } catch (StorageGroupProcessorException | MetadataException e) {
- throw new StorageEngineException(e);
}
+ return virtualStorageGroupManager.getProcessor(storageGroupPath, storageGroupMNode);
}
+ public StorageGroupProcessor buildNewStorageGroupProcessor(PartialPath storageGroupPath,
+ StorageGroupMNode storageGroupMNode, String storageGroupName)
+ throws StorageGroupProcessorException {
- StorageGroupProcessor processor;
- logger.info("construct a processor instance, the storage group is {}, Thread is {}",
- storageGroupPath, Thread.currentThread().getId());
- processor = new StorageGroupProcessor(systemDir + File.separator + storageGroupPath,
- storageGroupName,
- fileFlushPolicy, storageGroupMNode.getFullPath());
- processor.setDataTTL(storageGroupMNode.getDataTTL());
- processor.setCustomFlushListeners(customFlushListeners);
- processor.setCustomCloseFileListeners(customCloseFileListeners);
- return processor;
++ StorageGroupProcessor processor;
++ logger.info("construct a processor instance, the storage group is {}, Thread is {}",
++ storageGroupPath, Thread.currentThread().getId());
++ processor = new StorageGroupProcessor(systemDir + File.separator + storageGroupPath,
++ storageGroupName,
++ fileFlushPolicy, storageGroupMNode.getFullPath());
++ processor.setDataTTL(storageGroupMNode.getDataTTL());
++ processor.setCustomFlushListeners(customFlushListeners);
++ processor.setCustomCloseFileListeners(customCloseFileListeners);
++ return processor;
++ }
++
+ private void waitAllSgReady(PartialPath storageGroupPath) throws StorageEngineException {
+ if (isAllSgReady.get()) {
+ return;
+ }
+
+ long waitStart = System.currentTimeMillis();
+ long waited = 0L;
+ final long MAX_WAIT = 5000L;
+ while (!isAllSgReady.get() && waited < MAX_WAIT) {
+ try {
+ Thread.sleep(10);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new StorageEngineException(e);
+ }
+ waited = System.currentTimeMillis() - waitStart;
+ }
+
+ if (!isAllSgReady.get()) {
+ // not finished recover, refuse the request
+ throw new StorageEngineException(
+ "the sg " + storageGroupPath + " may not ready now, please wait and retry later",
+ TSStatusCode.STORAGE_GROUP_NOT_READY.getStatusCode());
+ }
}
/**
diff --cc server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index 5817c63,62c7607..4699396
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@@ -1231,7 -1198,7 +1201,7 @@@ public class StorageGroupProcessor
private void deleteAllSGFolders(List<String> folder) {
for (String tsfilePath : folder) {
-- File storageGroupFolder = fsFactory.getFile(tsfilePath, storageGroupName);
++ File storageGroupFolder = fsFactory.getFile(tsfilePath, logicalStorageGroupName + File.separator + storageGroupName);
if (storageGroupFolder.exists()) {
try {
org.apache.iotdb.db.utils.FileUtils.deleteDirectory(storageGroupFolder);
diff --cc server/src/test/java/org/apache/iotdb/db/integration/IoTDBCompleteIT.java
index f53b3f2,f53b3f2..72a7753
--- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBCompleteIT.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBCompleteIT.java
@@@ -18,7 -18,7 +18,9 @@@
*/
package org.apache.iotdb.db.integration;
++import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.constant.TestConstant;
++import org.apache.iotdb.db.engine.storagegroup.virtualSg.HashVirtualPartitioner;
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.jdbc.Config;
import org.junit.After;
@@@ -36,6 -36,6 +38,7 @@@ public class IoTDBCompleteIT
@Before
public void setUp() throws Exception {
++ HashVirtualPartitioner.getInstance().setStorageGroupNum(1);
EnvironmentUtils.closeStatMonitor();
EnvironmentUtils.envSetUp();
}
@@@ -43,6 -43,6 +46,7 @@@
@After
public void tearDown() throws Exception {
EnvironmentUtils.cleanEnv();
++ HashVirtualPartitioner.getInstance().setStorageGroupNum(IoTDBDescriptor.getInstance().getConfig().getVirtualPartitionNum());
}
@Test