You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2020/02/10 10:03:08 UTC
[incubator-iotdb] branch cluster_node_deletion updated (ad5be9f ->
1278054)
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a change to branch cluster_node_deletion
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.
from ad5be9f add unit tests
add bedb29c support load properties from System.getProperties()
add c5cb678 add javadocs for MManager
add 2395075 Add design documents (#757)
add 615b856 [IOTDB-445] Unify the keyword of "timestamp" and "time" (#762)
add 776da3b fix sync schema pos bug (#767)
add db7e8eb [IOTDB-448]Add IN operation (#763)
add 17f3a42 [IOTDB-459]Fix calmem tool bug (#770)
add d2a9021 add canBeSplit in PhysicalPlan
add c1ed8de for support plan partition (not finished)
add e7180be route all kinds of plans
add edf4574 add contribute docs
add 1a70943 [IOTDB-461] Correct the link in System Design page (#771)
add 1c2f378 Merge remote-tracking branch 'origin/master'
add 282402c add 0-content reminder
add 5c65994 Hive-Connector Design Document (#774)
add f461ba2 [IoTDB-463] [doc] Fix documents in progress version (#775)
add 367d7ef Merge remote-tracking branch 'origin/master' into cluster_new
add 880bebf finish all PhysicalPlan partition and their UTs
new 1278054 Merge branch 'cluster_new' into cluster_node_deletion
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.travis.yml | 2 +-
.../apache/iotdb/cluster/RemoteTsFileResource.java | 1 +
.../iotdb/cluster/config/ClusterDescriptor.java | 73 ++--
.../manage/FilePartitionedSnapshotLogManager.java | 3 +-
.../log/manage/PartitionedSnapshotLogManager.java | 6 +-
.../iotdb/cluster/partition/PartitionTable.java | 321 ++++++++++++--
.../cluster/partition/SlotPartitionTable.java | 95 ++--
.../cluster/server/member/MetaGroupMember.java | 46 +-
.../apache/iotdb/cluster/utils/PartitionUtils.java | 66 ++-
.../iotdb/cluster/common/TestPartitionTable.java | 25 +-
.../iotdb/cluster/partition/MManagerWhiteBox.java | 57 +++
.../cluster/partition/SlotPartitionTableTest.java | 441 ++++++++++++++++++-
.../server/heartbeat/MetaHeartBeatThreadTest.java | 24 +-
docs/{ => Community}/Community-History&Vision.md | 0
docs/{ => Community}/Community-Powered By.md | 0
.../Community-Project Committers.md | 0
docs/{ => Development}/Development-Chinese.md | 21 +
docs/{ => Development}/Development-Contributing.md | 3 +
docs/{ => Development}/Development-Document.md | 3 +
docs/{ => Development}/Development-IDE.md | 0
docs/{ => Development}/Development-VoteRelease.md | 0
.../SystemDesign/0-Architecture/1-Architecture.md | 55 +++
.../1-TsFile/1-TsFile.md} | 11 +-
.../1-TsFile/2-Format.md} | 7 +-
.../SystemDesign/1-TsFile/3-Write.md | 65 +++
.../SystemDesign/1-TsFile/4-Read.md} | 7 +-
.../SystemDesign/2-QueryEngine/1-QueryEngine.md | 64 +++
.../SystemDesign/2-QueryEngine/2-Planner.md | 63 +++
.../SystemDesign/2-QueryEngine/3-PlanExecutor.md} | 6 +-
.../3-SchemaManager/1-SchemaManager.md} | 6 +-
.../4-StorageEngine/1-StorageEngine.md | 92 ++++
.../SystemDesign/4-StorageEngine/2-WAL.md} | 6 +-
.../SystemDesign/4-StorageEngine/3-FlushManager.md | 84 ++++
.../4-StorageEngine/4-MergeManager.md} | 6 +-
.../SystemDesign/5-DataQuery/1-DataQuery.md | 67 +++
.../6-Tools/1-Sync.md} | 6 +-
.../SystemDesign/7-Connector/2-Hive-TsFile.md | 114 +++++
.../1-DDL (Data Definition Language).md | 8 +-
.../5-Operation Manual/4-SQL Reference.md | 13 +-
.../2-Files.md => 8-Architecture/1-Files.md} | 0
.../2-Writing Data on HDFS.md} | 0
.../3-Shared Nothing Cluster.md} | 0
.../SystemDesign/0-Content.md} | 28 +-
.../SystemDesign/1-TsFile/1-TsFile.md} | 10 +-
.../1-TsFile/2-Format.md} | 6 +-
.../SystemDesign/1-TsFile/3-Write.md} | 4 +-
.../SystemDesign/1-TsFile/4-Read.md} | 4 +-
.../{Developers => SystemDesign}/release.adoc | 0
docs/Documentation/UserGuide/0-Content.md | 9 +-
.../1-DDL (Data Definition Language).md | 9 +-
.../5-Operation Manual/4-SQL Reference.md | 18 +-
.../2-Files.md => 8-Architecture/1-Files.md} | 0
.../2-Writing Data on HDFS.md} | 0
.../3-Shared Nothing Cluster.md} | 0
pom.xml | 4 +
.../org/apache/iotdb/db/qp/strategy/SqlBase.g4 | 14 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 479 +++++++++++----------
.../db/conf/adapter/IoTDBConfigDynamicAdapter.java | 4 +-
.../org/apache/iotdb/db/engine/StorageEngine.java | 22 +-
.../org/apache/iotdb/db/metadata/MManager.java | 93 +++-
.../apache/iotdb/db/qp/constant/SQLConstant.java | 2 +
.../db/qp/executor/IQueryProcessExecutor.java | 1 +
.../org/apache/iotdb/db/qp/logical/Operator.java | 2 +-
.../db/qp/logical/crud/BasicFunctionOperator.java | 63 +--
.../iotdb/db/qp/logical/crud/FilterOperator.java | 9 +-
.../iotdb/db/qp/logical/crud/FunctionOperator.java | 7 +
.../iotdb/db/qp/logical/crud/InOperator.java | 200 +++++++++
.../apache/iotdb/db/qp/physical/PhysicalPlan.java | 11 +
.../iotdb/db/qp/physical/crud/BatchInsertPlan.java | 12 +-
.../iotdb/db/qp/physical/crud/InsertPlan.java | 4 +
.../apache/iotdb/db/qp/physical/sys/CountPlan.java | 3 +
.../db/qp/physical/sys/CreateTimeSeriesPlan.java | 2 +
.../db/qp/physical/sys/ShowChildPathsPlan.java | 2 +
.../iotdb/db/qp/strategy/LogicalGenerator.java | 82 ++--
.../iotdb/db/qp/strategy/PhysicalGenerator.java | 2 +-
.../qp/strategy/optimizer/ConcatPathOptimizer.java | 7 +-
.../qp/strategy/optimizer/DnfFilterOptimizer.java | 2 +-
.../db/qp/strategy/optimizer/IFilterOptimizer.java | 3 +-
.../qp/strategy/optimizer/RemoveNotOptimizer.java | 15 +-
.../receiver/recover/ISyncReceiverLogAnalyzer.java | 1 -
.../iotdb/db/sync/sender/transfer/SyncClient.java | 7 +-
.../iotdb/db/integration/IoTDBQueryDemoIT.java | 134 +++++-
.../org/apache/iotdb/db/qp/QueryProcessorTest.java | 4 +
.../iotdb/db/qp/plan/LogicalPlanSmallTest.java | 12 +
.../apache/iotdb/db/qp/plan/PhysicalPlanTest.java | 38 ++
.../read/expression/impl/GlobalTimeExpression.java | 2 +-
.../expression/impl/SingleSeriesExpression.java | 2 +-
.../iotdb/tsfile/read/filter/GroupByFilter.java | 2 +-
.../iotdb/tsfile/read/filter/TimeFilter.java | 16 +
.../iotdb/tsfile/read/filter/ValueFilter.java | 16 +
.../tsfile/read/filter/basic/BinaryFilter.java | 2 +-
.../iotdb/tsfile/read/filter/basic/Filter.java | 2 +-
.../tsfile/read/filter/basic/UnaryFilter.java | 3 +-
.../tsfile/read/filter/factory/FilterFactory.java | 2 -
.../read/filter/factory/FilterSerializeId.java | 2 +-
.../tsfile/read/filter/operator/AndFilter.java | 4 +-
.../iotdb/tsfile/read/filter/operator/Eq.java | 6 +-
.../iotdb/tsfile/read/filter/operator/Gt.java | 13 +-
.../iotdb/tsfile/read/filter/operator/GtEq.java | 13 +-
.../filter/operator/{NotFilter.java => In.java} | 92 ++--
.../iotdb/tsfile/read/filter/operator/Lt.java | 13 +-
.../iotdb/tsfile/read/filter/operator/LtEq.java | 13 +-
.../iotdb/tsfile/read/filter/operator/NotEq.java | 13 +-
.../tsfile/read/filter/operator/NotFilter.java | 4 +-
.../tsfile/read/filter/operator/OrFilter.java | 4 +-
.../apache/iotdb/tsfile/utils/Murmur128Hash.java | 2 +-
106 files changed, 2759 insertions(+), 663 deletions(-)
create mode 100644 cluster/src/test/java/org/apache/iotdb/cluster/partition/MManagerWhiteBox.java
copy docs/{ => Community}/Community-History&Vision.md (100%)
rename docs/{ => Community}/Community-Powered By.md (100%)
rename docs/{ => Community}/Community-Project Committers.md (100%)
rename docs/{ => Development}/Development-Chinese.md (83%)
rename docs/{ => Development}/Development-Contributing.md (98%)
rename docs/{ => Development}/Development-Document.md (90%)
rename docs/{ => Development}/Development-IDE.md (100%)
rename docs/{ => Development}/Development-VoteRelease.md (100%)
create mode 100644 docs/Documentation-CHN/SystemDesign/0-Architecture/1-Architecture.md
copy docs/Documentation-CHN/{UserGuide/8-System Design (Developer)/4-Shared Nothing Cluster.md => SystemDesign/1-TsFile/1-TsFile.md} (74%)
rename docs/Documentation-CHN/{UserGuide/8-System Design (Developer)/1-Hierarchy.md => SystemDesign/1-TsFile/2-Format.md} (99%)
create mode 100644 docs/Documentation-CHN/SystemDesign/1-TsFile/3-Write.md
copy docs/{Community-History&Vision.md => Documentation-CHN/SystemDesign/1-TsFile/4-Read.md} (93%)
create mode 100644 docs/Documentation-CHN/SystemDesign/2-QueryEngine/1-QueryEngine.md
create mode 100644 docs/Documentation-CHN/SystemDesign/2-QueryEngine/2-Planner.md
copy docs/{Community-History&Vision.md => Documentation-CHN/SystemDesign/2-QueryEngine/3-PlanExecutor.md} (90%)
copy docs/Documentation-CHN/{UserGuide/8-System Design (Developer)/4-Shared Nothing Cluster.md => SystemDesign/3-SchemaManager/1-SchemaManager.md} (70%)
create mode 100644 docs/Documentation-CHN/SystemDesign/4-StorageEngine/1-StorageEngine.md
copy docs/{Community-History&Vision.md => Documentation-CHN/SystemDesign/4-StorageEngine/2-WAL.md} (92%)
create mode 100644 docs/Documentation-CHN/SystemDesign/4-StorageEngine/3-FlushManager.md
copy docs/{Community-History&Vision.md => Documentation-CHN/SystemDesign/4-StorageEngine/4-MergeManager.md} (91%)
create mode 100644 docs/Documentation-CHN/SystemDesign/5-DataQuery/1-DataQuery.md
copy docs/Documentation-CHN/{UserGuide/8-System Design (Developer)/4-Shared Nothing Cluster.md => SystemDesign/6-Tools/1-Sync.md} (87%)
create mode 100644 docs/Documentation-CHN/SystemDesign/7-Connector/2-Hive-TsFile.md
rename docs/Documentation-CHN/UserGuide/{8-System Design (Developer)/2-Files.md => 8-Architecture/1-Files.md} (100%)
rename docs/Documentation-CHN/UserGuide/{8-System Design (Developer)/3-Writing Data on HDFS.md => 8-Architecture/2-Writing Data on HDFS.md} (100%)
copy docs/Documentation-CHN/UserGuide/{8-System Design (Developer)/4-Shared Nothing Cluster.md => 8-Architecture/3-Shared Nothing Cluster.md} (100%)
copy docs/{Documentation-CHN/UserGuide/8-System Design (Developer)/4-Shared Nothing Cluster.md => Documentation/SystemDesign/0-Content.md} (66%)
rename docs/{Documentation-CHN/UserGuide/8-System Design (Developer)/4-Shared Nothing Cluster.md => Documentation/SystemDesign/1-TsFile/1-TsFile.md} (79%)
rename docs/Documentation/{UserGuide/8-System Design (Developer)/1-Hierarchy.md => SystemDesign/1-TsFile/2-Format.md} (99%)
copy docs/{Community-History&Vision.md => Documentation/SystemDesign/1-TsFile/3-Write.md} (96%)
rename docs/{Community-History&Vision.md => Documentation/SystemDesign/1-TsFile/4-Read.md} (96%)
rename docs/Documentation/{Developers => SystemDesign}/release.adoc (100%)
rename docs/Documentation/UserGuide/{8-System Design (Developer)/2-Files.md => 8-Architecture/1-Files.md} (100%)
rename docs/Documentation/UserGuide/{8-System Design (Developer)/3-Writing Data on HDFS.md => 8-Architecture/2-Writing Data on HDFS.md} (100%)
rename docs/Documentation/UserGuide/{8-System Design (Developer)/4-Shared Nothing Cluster.md => 8-Architecture/3-Shared Nothing Cluster.md} (100%)
create mode 100644 server/src/main/java/org/apache/iotdb/db/qp/logical/crud/InOperator.java
copy tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/{NotFilter.java => In.java} (54%)
mode change 100755 => 100644
[incubator-iotdb] 01/01: Merge branch 'cluster_new' into
cluster_node_deletion
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch cluster_node_deletion
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 12780540a3b2e3c96a90cc0794665382771aea51
Merge: ad5be9f 880bebf
Author: jt2594838 <jt...@163.com>
AuthorDate: Mon Feb 10 18:02:51 2020 +0800
Merge branch 'cluster_new' into cluster_node_deletion
# Conflicts:
# cluster/src/main/java/org/apache/iotdb/cluster/partition/PartitionTable.java
# cluster/src/main/java/org/apache/iotdb/cluster/partition/SlotPartitionTable.java
.travis.yml | 2 +-
.../apache/iotdb/cluster/RemoteTsFileResource.java | 1 +
.../iotdb/cluster/config/ClusterDescriptor.java | 73 ++--
.../manage/FilePartitionedSnapshotLogManager.java | 3 +-
.../log/manage/PartitionedSnapshotLogManager.java | 6 +-
.../iotdb/cluster/partition/PartitionTable.java | 321 ++++++++++++--
.../cluster/partition/SlotPartitionTable.java | 95 ++--
.../cluster/server/member/MetaGroupMember.java | 46 +-
.../apache/iotdb/cluster/utils/PartitionUtils.java | 66 ++-
.../iotdb/cluster/common/TestPartitionTable.java | 25 +-
.../iotdb/cluster/partition/MManagerWhiteBox.java | 57 +++
.../cluster/partition/SlotPartitionTableTest.java | 441 ++++++++++++++++++-
.../server/heartbeat/MetaHeartBeatThreadTest.java | 24 +-
docs/{ => Community}/Community-History&Vision.md | 0
docs/{ => Community}/Community-Powered By.md | 0
.../Community-Project Committers.md | 0
docs/{ => Development}/Development-Chinese.md | 21 +
docs/{ => Development}/Development-Contributing.md | 3 +
docs/{ => Development}/Development-Document.md | 3 +
docs/{ => Development}/Development-IDE.md | 0
docs/{ => Development}/Development-VoteRelease.md | 0
.../SystemDesign/0-Architecture/1-Architecture.md | 55 +++
.../1-TsFile/1-TsFile.md} | 11 +-
.../1-TsFile/2-Format.md} | 7 +-
.../SystemDesign/1-TsFile/3-Write.md | 65 +++
.../SystemDesign/1-TsFile/4-Read.md} | 7 +-
.../SystemDesign/2-QueryEngine/1-QueryEngine.md | 64 +++
.../SystemDesign/2-QueryEngine/2-Planner.md | 63 +++
.../SystemDesign/2-QueryEngine/3-PlanExecutor.md} | 6 +-
.../3-SchemaManager/1-SchemaManager.md} | 6 +-
.../4-StorageEngine/1-StorageEngine.md | 92 ++++
.../SystemDesign/4-StorageEngine/2-WAL.md} | 6 +-
.../SystemDesign/4-StorageEngine/3-FlushManager.md | 84 ++++
.../4-StorageEngine/4-MergeManager.md} | 6 +-
.../SystemDesign/5-DataQuery/1-DataQuery.md | 67 +++
.../SystemDesign/6-Tools/1-Sync.md} | 4 +-
.../SystemDesign/7-Connector/2-Hive-TsFile.md | 114 +++++
.../1-DDL (Data Definition Language).md | 8 +-
.../5-Operation Manual/4-SQL Reference.md | 13 +-
.../2-Files.md => 8-Architecture/1-Files.md} | 0
.../2-Writing Data on HDFS.md} | 0
.../3-Shared Nothing Cluster.md} | 0
.../0-Content.md} | 28 +-
.../SystemDesign/1-TsFile/1-TsFile.md} | 10 +-
.../1-TsFile/2-Format.md} | 6 +-
.../SystemDesign/1-TsFile/3-Write.md} | 4 +-
.../SystemDesign/1-TsFile/4-Read.md} | 4 +-
.../{Developers => SystemDesign}/release.adoc | 0
docs/Documentation/UserGuide/0-Content.md | 9 +-
.../1-DDL (Data Definition Language).md | 9 +-
.../5-Operation Manual/4-SQL Reference.md | 18 +-
.../2-Files.md => 8-Architecture/1-Files.md} | 0
.../2-Writing Data on HDFS.md} | 0
.../3-Shared Nothing Cluster.md} | 0
pom.xml | 4 +
.../org/apache/iotdb/db/qp/strategy/SqlBase.g4 | 14 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 479 +++++++++++----------
.../db/conf/adapter/IoTDBConfigDynamicAdapter.java | 4 +-
.../org/apache/iotdb/db/engine/StorageEngine.java | 22 +-
.../org/apache/iotdb/db/metadata/MManager.java | 93 +++-
.../apache/iotdb/db/qp/constant/SQLConstant.java | 2 +
.../db/qp/executor/IQueryProcessExecutor.java | 1 +
.../org/apache/iotdb/db/qp/logical/Operator.java | 2 +-
.../db/qp/logical/crud/BasicFunctionOperator.java | 63 +--
.../iotdb/db/qp/logical/crud/FilterOperator.java | 9 +-
.../iotdb/db/qp/logical/crud/FunctionOperator.java | 7 +
.../iotdb/db/qp/logical/crud/InOperator.java | 200 +++++++++
.../apache/iotdb/db/qp/physical/PhysicalPlan.java | 11 +
.../iotdb/db/qp/physical/crud/BatchInsertPlan.java | 12 +-
.../iotdb/db/qp/physical/crud/InsertPlan.java | 4 +
.../apache/iotdb/db/qp/physical/sys/CountPlan.java | 3 +
.../db/qp/physical/sys/CreateTimeSeriesPlan.java | 2 +
.../db/qp/physical/sys/ShowChildPathsPlan.java | 2 +
.../iotdb/db/qp/strategy/LogicalGenerator.java | 82 ++--
.../iotdb/db/qp/strategy/PhysicalGenerator.java | 2 +-
.../qp/strategy/optimizer/ConcatPathOptimizer.java | 7 +-
.../qp/strategy/optimizer/DnfFilterOptimizer.java | 2 +-
.../db/qp/strategy/optimizer/IFilterOptimizer.java | 3 +-
.../qp/strategy/optimizer/RemoveNotOptimizer.java | 15 +-
.../receiver/recover/ISyncReceiverLogAnalyzer.java | 1 -
.../iotdb/db/sync/sender/transfer/SyncClient.java | 7 +-
.../iotdb/db/integration/IoTDBQueryDemoIT.java | 134 +++++-
.../org/apache/iotdb/db/qp/QueryProcessorTest.java | 4 +
.../iotdb/db/qp/plan/LogicalPlanSmallTest.java | 12 +
.../apache/iotdb/db/qp/plan/PhysicalPlanTest.java | 38 ++
.../read/expression/impl/GlobalTimeExpression.java | 2 +-
.../expression/impl/SingleSeriesExpression.java | 2 +-
.../iotdb/tsfile/read/filter/GroupByFilter.java | 2 +-
.../iotdb/tsfile/read/filter/TimeFilter.java | 16 +
.../iotdb/tsfile/read/filter/ValueFilter.java | 16 +
.../tsfile/read/filter/basic/BinaryFilter.java | 2 +-
.../iotdb/tsfile/read/filter/basic/Filter.java | 2 +-
.../tsfile/read/filter/basic/UnaryFilter.java | 3 +-
.../tsfile/read/filter/factory/FilterFactory.java | 2 -
.../read/filter/factory/FilterSerializeId.java | 2 +-
.../tsfile/read/filter/operator/AndFilter.java | 4 +-
.../iotdb/tsfile/read/filter/operator/Eq.java | 6 +-
.../iotdb/tsfile/read/filter/operator/Gt.java | 13 +-
.../iotdb/tsfile/read/filter/operator/GtEq.java | 13 +-
.../filter/operator/{NotFilter.java => In.java} | 92 ++--
.../iotdb/tsfile/read/filter/operator/Lt.java | 13 +-
.../iotdb/tsfile/read/filter/operator/LtEq.java | 13 +-
.../iotdb/tsfile/read/filter/operator/NotEq.java | 13 +-
.../tsfile/read/filter/operator/NotFilter.java | 4 +-
.../tsfile/read/filter/operator/OrFilter.java | 4 +-
.../apache/iotdb/tsfile/utils/Murmur128Hash.java | 2 +-
106 files changed, 2759 insertions(+), 661 deletions(-)
diff --cc cluster/src/main/java/org/apache/iotdb/cluster/log/manage/FilePartitionedSnapshotLogManager.java
index 6eed911,bad9847..280cdae
--- a/cluster/src/main/java/org/apache/iotdb/cluster/log/manage/FilePartitionedSnapshotLogManager.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/log/manage/FilePartitionedSnapshotLogManager.java
@@@ -37,8 -37,9 +37,9 @@@ import org.slf4j.LoggerFactory
/**
* Different from PartitionedSnapshotLogManager, FilePartitionedSnapshotLogManager does not store
- * the committed in memory, it considers the logs are contained in the TsFiles so it will record
+ * the committed in memory after snapshots, it considers the logs are contained in the TsFiles so it will record
* every TsFiles in the slot instead.
+ * TODO IOTDB-439 but the log is stored in memory already by appendLog method.
*/
public class FilePartitionedSnapshotLogManager extends PartitionedSnapshotLogManager<FileSnapshot> {
diff --cc cluster/src/main/java/org/apache/iotdb/cluster/partition/PartitionTable.java
index eefe9de,446fc01..bb2fb26
--- a/cluster/src/main/java/org/apache/iotdb/cluster/partition/PartitionTable.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/partition/PartitionTable.java
@@@ -30,8 -32,8 +32,8 @@@ import org.apache.iotdb.cluster.log.Log
import org.apache.iotdb.cluster.log.logtypes.PhysicalPlanLog;
import org.apache.iotdb.cluster.rpc.thrift.Node;
import org.apache.iotdb.cluster.utils.PartitionUtils;
-import org.apache.iotdb.db.engine.StorageEngine;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
- import org.apache.iotdb.db.engine.StorageEngine;
+ import org.apache.iotdb.db.exception.metadata.IllegalPathException;
import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.exception.metadata.StorageGroupNotSetException;
import org.apache.iotdb.db.metadata.MManager;
@@@ -50,6 -54,7 +54,7 @@@ import org.slf4j.LoggerFactory
public interface PartitionTable {
// static final is not necessary, it is redundant for an interface
Logger logger = LoggerFactory.getLogger(SlotPartitionTable.class);
- long PARTITION_INTERVAL = StorageEngine.getTimePartitionInterval();
++ long PARTITION_INTERVAL = IoTDBDescriptor.getInstance().getConfig().getPartitionInterval();
/**
* Given the storageGroupName and the timestamp, return the list of nodes on which the storage
@@@ -165,10 -247,183 +253,183 @@@
throw new UnsupportedPlanException(plan);
}
+ default Map<PhysicalPlan, PartitionGroup> splitAndRoutePlan(BatchInsertPlan plan)
- throws UnsupportedPlanException, StorageGroupNotSetException , IllegalPathException {
++ throws StorageGroupNotSetException {
+ String storageGroup = getMManager().getStorageGroupNameByPath(plan.getDeviceId());
+ Map<PhysicalPlan, PartitionGroup> result = new HashMap<>();
+ MultiKeyMap<Long, PartitionGroup> timeRangeMapRaftGroup = new MultiKeyMap<>();
+ long[] times = plan.getTimes();
+ if(times.length == 0) {
+ return Collections.emptyMap();
+ }
+ long startTime = (times[0] / PARTITION_INTERVAL ) * PARTITION_INTERVAL;//included
+ long endTime = startTime + PARTITION_INTERVAL;//excluded
+ int startLoc = 0; //included
+
+ //Map<PartitionGroup>
+ Map<PartitionGroup, List<Integer>> splitMap = new HashMap<>();
+ //for each List in split, they are range1.start, range.end, range2.start, range2.end, ...
+ for (int i = 1; i < times.length; i++) {// times are sorted in session API.
+ if (times[i] >= endTime) {
+ // a new range.
+ PartitionGroup group = route(storageGroup, startTime);
+ List<Integer> ranges = splitMap.computeIfAbsent(group, x -> new ArrayList<>());
+ ranges.add(startLoc);//include
+ ranges.add(i);//excluded
+ //next init
+ startLoc = i;
+ startTime = endTime;
+ endTime = (times[i] / PARTITION_INTERVAL + 1) * PARTITION_INTERVAL;
+ }
+ }
+ //the final range
+ PartitionGroup group = route(storageGroup, startTime);
+ List<Integer> ranges = splitMap.computeIfAbsent(group, x -> new ArrayList<>());
+ ranges.add(startLoc);//includec
+ ranges.add(times.length);//excluded
+
+ List<Integer> locs;
+ for(Map.Entry<PartitionGroup, List<Integer>> entry : splitMap.entrySet()) {
+ //generate a new times and values
+ locs = entry.getValue();
+ int count = 0;
+ for (int i = 0; i < locs.size(); i += 2) {
+ int start = locs.get(i);
+ int end = locs.get(i + 1);
+ count += end - start;
+ }
+ long[] subTimes = new long[count];
+ int destLoc = 0;
+ Object[] values = new Object[plan.getMeasurements().length];
+ for (int i = 0; i < values.length; i ++) {
+ switch (plan.getDataTypes()[i]) {
+ case TEXT:
+ values[i] = new Binary[count];
+ break;
+ case FLOAT:
+ values[i] = new float[count];
+ break;
+ case INT32:
+ values[i] = new int[count];
+ break;
+ case INT64:
+ values[i] = new long[count];
+ break;
+ case DOUBLE:
+ values[i] = new double[count];
+ break;
+ case BOOLEAN:
+ values[i] = new boolean[count];
+ break;
+ }
+ }
+ for (int i = 0; i < locs.size(); i += 2) {
+ int start = locs.get(i);
+ int end = locs.get(i + 1);
+ System.arraycopy(plan.getTimes(), start, subTimes, destLoc, end - start);
+ for (int k = 0; k < values.length; k ++) {
+ System.arraycopy(plan.getColumns()[k], start, values[k], destLoc, end - start);
+ }
+ destLoc += end -start;
+ }
+ BatchInsertPlan newBatch = PartitionUtils.copy(plan, subTimes, values);
+ result.put(newBatch, entry.getKey());
+ }
+ return result;
+ }
+
+ default Map<PhysicalPlan, PartitionGroup> splitAndRoutePlan(UpdatePlan plan)
+ throws UnsupportedPlanException {
+ logger.error("UpdatePlan is not implemented");
+ throw new UnsupportedPlanException(plan);
+ }
+
+ //TODO this case can be optimized, see the related UT for better understanding.
+ default Map<PhysicalPlan, PartitionGroup> splitAndRoutePlan(CountPlan plan)
+ throws StorageGroupNotSetException, IllegalPathException {
+ //CountPlan is quite special because it has the behavior of wildcard at the tail of the path
+ // even though there is no wildcard
+ Map<String, String> sgPathMap = getMManager().determineStorageGroup(plan.getPath().getFullPath()+".*");
+ if (sgPathMap.isEmpty()) {
+ throw new StorageGroupNotSetException(plan.getPath().getFullPath());
+ }
+ Map<PhysicalPlan, PartitionGroup> result = new HashMap<>();
+ if (plan.getShowContentType().equals(ShowContentType.COUNT_TIMESERIES)) {
+ //support wildcard
+ for (Map.Entry<String, String> entry : sgPathMap.entrySet()) {
+ CountPlan plan1 = new CountPlan(ShowContentType.COUNT_TIMESERIES, new Path(entry.getValue()), plan.getLevel());
+ result.put(plan1, route(entry.getKey(), 0));
+ }
+ } else {
+ //do not support wildcard
+ if (sgPathMap.size() == 1) {
+ // the path of the original plan has only one SG, or there is only one SG in the system.
+ for (Map.Entry<String, String> entry : sgPathMap.entrySet()) {
+ //actually, there is only one entry
+ result.put(plan, route(entry.getKey(), 0));
+ }
+ } else {
+ // the path of the original plan contains more than one SG, and we added a wildcard at the tail.
+ // we have to remove it.
+ for (Map.Entry<String, String> entry : sgPathMap.entrySet()) {
+ CountPlan plan1 = new CountPlan(ShowContentType.COUNT_TIMESERIES,
+ new Path(entry.getValue().substring(0, entry.getValue().lastIndexOf(".*"))), plan.getLevel());
+ result.put(plan1, route(entry.getKey(), 0));
+ }
+ }
+ }
+ return result;
+ }
+
+ default Map<PhysicalPlan, PartitionGroup> splitAndRoutePlan(DataAuthPlan plan)
+ throws UnsupportedPlanException,StorageGroupNotSetException, IllegalPathException {
+ //TODO
+ //why this plan has not Path field?
+ return null;
+ }
+
+ //TODO this case can be optimized, see the related UT for better understanding.
+ default Map<PhysicalPlan, PartitionGroup> splitAndRoutePlan(ShowDevicesPlan plan)
+ throws IllegalPathException {
+ //show devices is quite special because it has the behavior of wildcard at the tail of the path
+ // even though there is no wildcard
+ Map<String, String> sgPathMap = getMManager().determineStorageGroup(plan.getPath().getFullPath()+".*");
+ Map<PhysicalPlan, PartitionGroup> result =new HashMap<>();
+ for (Map.Entry<String, String> entry : sgPathMap.entrySet()) {
+ result.put(new ShowDevicesPlan(plan.getShowContentType(), new Path(entry.getValue())), route(entry.getKey(), 0));
+ }
+ return result;
+ }
+
+ //TODO this case can be optimized, see the related UT for better understanding.
+ default Map<PhysicalPlan, PartitionGroup> splitAndRoutePlan(ShowTimeSeriesPlan plan)
+ throws StorageGroupNotSetException, IllegalPathException {
+ //show timeseries is quite special because it has the behavior of wildcard at the tail of the path
+ // even though there is no wildcard
+ Map<String, String> sgPathMap = getMManager().determineStorageGroup(plan.getPath().getFullPath()+".*");
+ if (sgPathMap.isEmpty()) {
+ throw new StorageGroupNotSetException(plan.getPath().getFullPath());
+ }
+ Map<PhysicalPlan, PartitionGroup> result = new HashMap<>();
+ for (Map.Entry<String, String> entry : sgPathMap.entrySet()) {
+ ShowTimeSeriesPlan newShow = new ShowTimeSeriesPlan(ShowContentType.TIMESERIES, new Path(entry.getValue()));
+ result.put(newShow, route(entry.getKey(), 0));
+ }
+ return result;
+ }
+
+
+
+ /**
+ * @param path can be an incomplete path (but should contain a storage group name)
+ * e.g., if "root.sg" is a storage group, then path can not be "root".
+ * @param timestamp
+ * @return
+ * @throws StorageGroupNotSetException
+ */
default PartitionGroup partitionByPathTime(String path, long timestamp)
throws StorageGroupNotSetException {
- String storageGroup = MManager.getInstance().getStorageGroupNameByPath(path);
+ String storageGroup = getMManager().getStorageGroupNameByPath(path);
return this.route(storageGroup, timestamp);
}
diff --cc cluster/src/main/java/org/apache/iotdb/cluster/partition/SlotPartitionTable.java
index 03c9593,2490f1f..798f5d9
--- a/cluster/src/main/java/org/apache/iotdb/cluster/partition/SlotPartitionTable.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/partition/SlotPartitionTable.java
@@@ -89,18 -103,19 +103,18 @@@ public class SlotPartitionTable impleme
private void assignPartitions() {
// evenly assign the slots to each node
int nodeNum = nodeRing.size();
- int slotsPerNode = slotNum / nodeNum;
- for (Node node : nodeRing) {
- nodeSlotMap.put(node, new ArrayList<>());
- }
-
- for (int i = 0; i < slotNum; i++) {
- int nodeIdx = i / slotsPerNode;
- if (nodeIdx >= nodeNum) {
- // the last node may receive a little more if total slots cannot de divided by node number
- nodeIdx--;
+ int slotsPerNode = totalSlotNumbers / nodeNum;
+ for (Node node : nodeRing) {
- List<Integer> nodeSlots = new ArrayList<>();
- nodeSlotMap.put(node, nodeSlots);
++ nodeSlotMap.put(node, new ArrayList<>());
}
- nodeSlotMap.get(nodeRing.get(nodeIdx)).add(i);
+
+ for (int i = 0; i < totalSlotNumbers; i++) {
+ int nodeIdx = i / slotsPerNode;
+ if (nodeIdx >= nodeNum) {
+ // the last node may receive a little more if total slots cannot de divided by node number
+ nodeIdx--;
+ }
+ nodeSlotMap.get(nodeRing.get(nodeIdx)).add(i);
}
// build the index to find a node by slot