You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by lt...@apache.org on 2021/01/19 11:32:39 UTC
[iotdb] 01/02: merge master
This is an automated email from the ASF dual-hosted git repository.
lta pushed a commit to branch cluster_scalability
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit e7f8e286c27e22c23775f08478a8feae4aff9506
Merge: f8a1432 48d24e7
Author: lta <li...@163.com>
AuthorDate: Tue Jan 19 19:23:32 2021 +0800
merge master
LICENSE-binary | 20 +-
.../antlr4/org/apache/iotdb/db/qp/sql/SqlBase.g4 | 13 +-
cluster/pom.xml | 6 +-
cluster/src/assembly/cluster.xml | 4 +-
cluster/src/assembly/resources/sbin/add-node.sh | 2 +-
cluster/src/assembly/resources/sbin/start-node.sh | 2 +-
.../iotdb/cluster/ClusterFileFlushPolicy.java | 2 +-
.../iotdb/cluster/coordinator/Coordinator.java | 165 ++++--
.../cluster/log/applier/AsyncDataLogApplier.java | 13 +-
.../iotdb/cluster/log/applier/BaseApplier.java | 2 +-
.../iotdb/cluster/log/applier/DataLogApplier.java | 13 +-
.../apache/iotdb/cluster/metadata/CMManager.java | 28 +-
.../apache/iotdb/cluster/metadata/MetaPuller.java | 5 +-
.../iotdb/cluster/query/ClusterPlanExecutor.java | 53 +-
.../iotdb/cluster/query/ClusterPlanRouter.java | 74 +++
.../cluster/query/filter/SlotTsFileFilter.java | 4 +-
.../utils/nodetool/function/NodeToolCmd.java | 22 +-
.../DDL Data Definition Language.md | 2 +
.../DML Data Manipulation Language.md | 37 +-
docs/UserGuide/Operation Manual/Kill Query.md | 60 ++
docs/UserGuide/Operation Manual/SQL Reference.md | 15 +
docs/UserGuide/System Tools/NodeTool.md | 9 +-
docs/zh/SystemDesign/StorageEngine/Compaction.md | 2 +-
.../DDL Data Definition Language.md | 2 +
.../DML Data Manipulation Language.md | 42 --
docs/zh/UserGuide/Operation Manual/Kill Query.md | 61 +++
.../zh/UserGuide/Operation Manual/SQL Reference.md | 21 +-
docs/zh/UserGuide/System Tools/NodeTool.md | 10 +-
.../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 12 +-
pom.xml | 12 +-
server/pom.xml | 5 +
.../resources/conf/iotdb-engine.properties | 5 +
server/src/assembly/resources/conf/iotdb-env.bat | 1 -
server/src/assembly/resources/conf/iotdb-env.sh | 1 -
server/src/assembly/resources/conf/logback.xml | 68 +--
server/src/assembly/resources/sbin/start-server.sh | 2 +-
.../assembly/resources/tools/start-WalChecker.sh | 2 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 15 +-
.../org/apache/iotdb/db/conf/IoTDBConfigCheck.java | 8 +
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 6 +-
.../org/apache/iotdb/db/engine/StorageEngine.java | 609 +++++++++++----------
.../engine/compaction/utils/CompactionUtils.java | 2 -
.../iotdb/db/engine/flush/MemTableFlushTask.java | 36 +-
.../iotdb/db/engine/memtable/AbstractMemTable.java | 44 +-
.../apache/iotdb/db/engine/memtable/IMemTable.java | 17 +-
.../db/engine/memtable/PrimitiveMemTable.java | 8 -
.../iotdb/db/engine/merge/task/MergeFileTask.java | 6 +-
.../db/engine/merge/task/MergeMultiChunkTask.java | 1 -
.../iotdb/db/engine/merge/task/MergeTask.java | 28 +-
.../iotdb/db/engine/modification/Deletion.java | 8 +-
.../iotdb/db/engine/modification/Modification.java | 18 +-
.../io/LocalTextModificationAccessor.java | 2 +-
.../db/engine/querycontext/ReadOnlyMemChunk.java | 9 +-
.../db/engine/storagegroup/StorageGroupInfo.java | 51 +-
.../engine/storagegroup/StorageGroupProcessor.java | 344 +++++++-----
.../db/engine/storagegroup/TsFileProcessor.java | 94 +++-
.../db/engine/storagegroup/TsFileResource.java | 15 +
.../virtualSg/HashVirtualPartitioner.java | 67 +++
.../storagegroup/virtualSg/VirtualPartitioner.java | 29 +-
.../virtualSg/VirtualStorageGroupManager.java | 434 +++++++++++++++
.../org/apache/iotdb/db/metadata/MManager.java | 89 +--
.../org/apache/iotdb/db/monitor/StatMonitor.java | 14 +-
.../apache/iotdb/db/qp/executor/IPlanExecutor.java | 8 +
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 24 +-
.../org/apache/iotdb/db/qp/logical/Operator.java | 2 +-
.../apache/iotdb/db/qp/physical/PhysicalPlan.java | 7 +-
.../db/qp/physical/crud/InsertMultiTabletPlan.java | 326 +++++++++++
.../db/qp/physical/crud/InsertTabletPlan.java | 6 +
.../iotdb/db/qp/strategy/LogicalGenerator.java | 4 +
.../iotdb/db/query/control/QueryTimeManager.java | 74 +--
.../db/query/dataset/NonAlignEngineDataSet.java | 22 +-
.../dataset/RawQueryDataSetWithoutValueFilter.java | 37 +-
.../db/query/executor/RawDataQueryExecutor.java | 11 +-
.../db/query/executor/fill/LastPointReader.java | 36 +-
.../chunk/metadata/DiskChunkMetadataLoader.java | 4 +
.../chunk/metadata/MemChunkMetadataLoader.java | 4 +
.../iotdb/db/query/reader/series/SeriesReader.java | 33 +-
.../reader/universal/DescPriorityMergeReader.java | 4 +-
.../reader/universal/PriorityMergeReader.java | 50 +-
.../iotdb/db/query/udf/core/input/InputLayer.java | 2 +-
.../org/apache/iotdb/db/service/TSServiceImpl.java | 87 ++-
.../apache/iotdb/db/tools/TsFileSketchTool.java | 18 +-
.../db/tools/upgrade/TsFileOnlineUpgradeTool.java | 34 +-
.../db/tools/virtualsg/DeviceMappingViewer.java | 61 +++
.../org/apache/iotdb/db/utils/FilePathUtils.java | 9 +
.../java/org/apache/iotdb/db/utils/MmapUtil.java | 25 +-
.../java/org/apache/iotdb/db/utils/QueryUtils.java | 5 +-
.../writelog/manager/MultiFileLogNodeManager.java | 27 +-
.../db/writelog/manager/WriteLogNodeManager.java | 7 +-
.../db/writelog/node/ExclusiveWriteLogNode.java | 18 +-
.../iotdb/db/writelog/node/WriteLogNode.java | 8 +-
.../iotdb/db/writelog/recover/LogReplayer.java | 26 +-
.../writelog/recover/TsFileRecoverPerformer.java | 29 +-
.../db/engine/cache/ChunkMetadataCacheTest.java | 2 +-
.../db/engine/memtable/PrimitiveMemTableTest.java | 4 +-
.../apache/iotdb/db/engine/merge/MergeLogTest.java | 1 +
.../iotdb/db/engine/merge/MergeOverLapTest.java | 1 -
.../iotdb/db/engine/merge/MergeTaskTest.java | 3 +-
.../apache/iotdb/db/engine/merge/MergeTest.java | 3 +
.../engine/modification/DeletionFileNodeTest.java | 27 +-
.../storagegroup/StorageGroupProcessorTest.java | 15 +-
.../iotdb/db/engine/storagegroup/TTLTest.java | 34 +-
.../engine/storagegroup/TsFileProcessorTest.java | 10 +-
.../virtualSg/HashVirtualPartitionerTest.java | 64 +++
.../iotdb/db/integration/IoTDBCompleteIT.java | 5 +
.../iotdb/db/integration/IoTDBDeletionIT.java | 40 ++
.../db/integration/IoTDBLoadExternalTsfileIT.java | 33 +-
.../iotdb/db/integration/IoTDBMultiDeviceIT.java | 322 +++++++++++
.../db/integration/IoTDBQueryTimeoutTest.java | 27 +-
.../iotdb/db/integration/IoTDBRestartIT.java | 16 +
.../db/integration/auth/IoTDBAuthorizationIT.java | 21 -
.../iotdb/db/monitor/IoTDBStatMonitorTest.java | 21 +-
.../db/qp/physical/InsertTabletMultiPlanTest.java | 99 ++++
.../iotdb/db/qp/physical/InsertTabletPlanTest.java | 105 ++++
.../iotdb/db/qp/physical/PhysicalPlanTest.java | 2 +-
.../query/reader/series/SeriesReaderTestUtil.java | 4 +-
.../db/sync/receiver/load/FileLoaderTest.java | 38 +-
.../recover/SyncReceiverLogAnalyzerTest.java | 2 +-
.../apache/iotdb/db/utils/EnvironmentUtils.java | 3 +
.../apache/iotdb/db/utils/VersionUtilsTest.java | 55 --
.../iotdb/db/writelog/IoTDBLogFileSizeTest.java | 25 +-
.../apache/iotdb/db/writelog/PerformanceTest.java | 48 +-
.../iotdb/db/writelog/WriteLogNodeManagerTest.java | 63 ++-
.../apache/iotdb/db/writelog/WriteLogNodeTest.java | 71 ++-
.../iotdb/db/writelog/recover/LogReplayerTest.java | 43 +-
.../recover/RecoverResourceFromReaderTest.java | 56 +-
.../db/writelog/recover/SeqTsFileRecoverTest.java | 55 +-
.../writelog/recover/UnseqTsFileRecoverTest.java | 36 +-
.../org/apache/iotdb/rpc/AutoResizingBuffer.java | 15 +-
.../iotdb/rpc/AutoScalingBufferReadTransport.java | 9 +
.../iotdb/rpc/AutoScalingBufferWriteTransport.java | 5 +-
.../main/java/org/apache/iotdb/rpc/RpcUtils.java | 7 +
.../rpc/TCompressedElasticFramedTransport.java | 61 +--
.../apache/iotdb/rpc/TElasticFramedTransport.java | 14 +-
.../rpc/TimeoutChangeableTFastFramedTransport.java | 2 +
.../TimeoutChangeableTSnappyFramedTransport.java | 2 +
.../main/java/org/apache/iotdb/session/Config.java | 1 -
.../java/org/apache/iotdb/session/Session.java | 10 +-
.../apache/iotdb/session/SessionConnection.java | 2 +-
.../org/apache/iotdb/session/SessionDataSet.java | 3 +-
site/src/main/.vuepress/config.js | 2 +
thrift/rpc-changelist.md | 24 +
.../iotdb/tsfile/file/metadata/TsFileMetadata.java | 31 --
.../iotdb/tsfile/read/TsFileSequenceReader.java | 14 -
.../iotdb/tsfile/read/reader/LocalTsFileInput.java | 20 -
.../apache/iotdb/tsfile/utils/VersionUtils.java | 49 --
.../apache/iotdb/tsfile/write/TsFileWriter.java | 5 -
.../write/writer/ForceAppendTsFileWriter.java | 1 -
.../write/writer/RestorableTsFileIOWriter.java | 6 +-
.../iotdb/tsfile/write/writer/TsFileIOWriter.java | 23 +-
.../tsfile/file/metadata/utils/TestHelper.java | 9 -
.../iotdb/tsfile/write/TsFileIOWriterTest.java | 6 -
.../iotdb/tsfile/write/TsFileWriterTest.java | 1 -
.../write/writer/RestorableTsFileIOWriterTest.java | 15 +-
154 files changed, 3789 insertions(+), 1661 deletions(-)
diff --cc cluster/src/main/java/org/apache/iotdb/cluster/coordinator/Coordinator.java
index dbfb9ff,de36ac4..c6bccfb
--- a/cluster/src/main/java/org/apache/iotdb/cluster/coordinator/Coordinator.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/coordinator/Coordinator.java
@@@ -365,13 -368,13 +370,19 @@@ public class Coordinator
if (entry.getValue().contains(thisNode)) {
// the query should be handled by a group the local node is in, handle it with in the group
long startTime = Timer.Statistic.META_GROUP_MEMBER_EXECUTE_NON_QUERY_IN_LOCAL_GROUP
- .getOperationStartTime();
+ .getOperationStartTime();
logger.debug("Execute {} in a local group of {}", entry.getKey(),
++<<<<<<< HEAD
+ entry.getValue().getHeader());
+ result = metaGroupMember.getLocalDataMember(new RaftNode(entry.getValue().getHeader(), entry.getValue().getId()))
+ .executeNonQueryPlan(entry.getKey());
++=======
+ entry.getValue().getHeader());
+ result = metaGroupMember.getLocalDataMember(entry.getValue().getHeader())
+ .executeNonQueryPlan(entry.getKey());
++>>>>>>> master
Timer.Statistic.META_GROUP_MEMBER_EXECUTE_NON_QUERY_IN_LOCAL_GROUP
- .calOperationCostTimeFromStart(startTime);
+ .calOperationCostTimeFromStart(startTime);
} else {
// forward the query to the group that should handle it
long startTime = Timer.Statistic.META_GROUP_MEMBER_EXECUTE_NON_QUERY_IN_REMOTE_GROUP
diff --cc cluster/src/main/java/org/apache/iotdb/cluster/log/applier/DataLogApplier.java
index ceed787,e590489..84fad46
--- a/cluster/src/main/java/org/apache/iotdb/cluster/log/applier/DataLogApplier.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/log/applier/DataLogApplier.java
@@@ -62,14 -62,12 +64,16 @@@ public class DataLogApplier extends Bas
logger.debug("DataMember [{}] start applying Log {}", dataGroupMember.getName(), log);
try {
- if (log instanceof PhysicalPlanLog) {
+ if (log instanceof AddNodeLog) {
+ metaGroupMember.getDataClusterServer().preAddNodeForDataGroup((AddNodeLog) log, dataGroupMember);
+ } else if (log instanceof RemoveNodeLog) {
+ metaGroupMember.getDataClusterServer().preRemoveNodeForDataGroup((RemoveNodeLog) log, dataGroupMember);
+ } else if (log instanceof PhysicalPlanLog) {
PhysicalPlanLog physicalPlanLog = (PhysicalPlanLog) log;
PhysicalPlan plan = physicalPlanLog.getPlan();
- if (plan instanceof InsertPlan) {
+ if (plan instanceof InsertMultiTabletPlan) {
+ applyInsert((InsertMultiTabletPlan) plan);
+ } else if (plan instanceof InsertPlan) {
applyInsert((InsertPlan) plan);
} else {
applyPhysicalPlan(plan, dataGroupMember);
diff --cc cluster/src/main/java/org/apache/iotdb/cluster/query/ClusterPlanRouter.java
index 487b88d,8c63386..65f66dc
--- a/cluster/src/main/java/org/apache/iotdb/cluster/query/ClusterPlanRouter.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/query/ClusterPlanRouter.java
@@@ -116,9 -109,11 +117,11 @@@ public class ClusterPlanRouter
}
public Map<PhysicalPlan, PartitionGroup> splitAndRoutePlan(PhysicalPlan plan)
- throws UnsupportedPlanException, MetadataException {
+ throws UnsupportedPlanException, MetadataException, UnknownLogTypeException {
if (plan instanceof InsertTabletPlan) {
return splitAndRoutePlan((InsertTabletPlan) plan);
+ } else if (plan instanceof InsertMultiTabletPlan) {
+ return splitAndRoutePlan((InsertMultiTabletPlan) plan);
} else if (plan instanceof CountPlan) {
return splitAndRoutePlan((CountPlan) plan);
} else if (plan instanceof CreateTimeSeriesPlan) {
diff --cc server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java
index dc553be,0df63f7..ab9335e
--- a/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java
@@@ -366,8 -366,8 +371,8 @@@ public abstract class PhysicalPlan
REVOKE_USER_PRIVILEGE, GRANT_ROLE_PRIVILEGE, GRANT_USER_PRIVILEGE, GRANT_USER_ROLE, MODIFY_PASSWORD, DELETE_USER,
DELETE_STORAGE_GROUP, SHOW_TIMESERIES, DELETE_TIMESERIES, LOAD_CONFIGURATION, CREATE_MULTI_TIMESERIES,
ALTER_TIMESERIES, FLUSH, CREATE_INDEX, DROP_INDEX,
- CHANGE_TAG_OFFSET, CHANGE_ALIAS, MNODE, MEASUREMENT_MNODE, STORAGE_GROUP_MNODE,
+ CHANGE_TAG_OFFSET, CHANGE_ALIAS, MNODE, MEASUREMENT_MNODE, STORAGE_GROUP_MNODE, CLUSTER_LOG,
- BATCH_INSERT_ONE_DEVICE
+ BATCH_INSERT_ONE_DEVICE, MULTI_BATCH_INSERT
}
public long getIndex() {