You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by lt...@apache.org on 2020/12/23 08:12:47 UTC
[iotdb] 01/05: merge master
This is an automated email from the ASF dual-hosted git repository.
lta pushed a commit to branch cluster_multi_raft
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 1aea5a7d6f782f6d472ccf1dcfda5d24b37481c7
Merge: 7ee34da 923cf57
Author: lta <li...@163.com>
AuthorDate: Mon Dec 21 16:20:06 2020 +0800
merge master
.github/workflows/sonar_and_coverall.yml | 2 +-
.travis.yml | 2 +-
Jenkinsfile | 29 +-
LICENSE-binary | 2 -
.../antlr4/org/apache/iotdb/db/qp/sql/SqlBase.g4 | 12 +-
.../java/org/apache/iotdb/cli/AbstractCli.java | 110 +-
cli/src/main/java/org/apache/iotdb/cli/Cli.java | 5 +-
cli/src/main/java/org/apache/iotdb/cli/WinCli.java | 5 +-
.../main/java/org/apache/iotdb/tool/ExportCsv.java | 4 +-
.../java/org/apache/iotdb/cli/AbstractCliIT.java | 35 +-
cluster/src/assembly/resources/sbin/add-node.bat | 6 +-
cluster/src/assembly/resources/sbin/add-node.sh | 6 +-
.../cluster/partition/slot/SlotPartitionTable.java | 36 +-
compile-tools/boost/pom.xml | 4 +
docs/SystemDesign/SchemaManager/SchemaManager.md | 20 +-
docs/SystemDesign/TsFile/Format.md | 4 +-
.../DML Data Manipulation Language.md | 100 +-
docs/UserGuide/Operation Manual/SQL Reference.md | 7 +-
docs/UserGuide/Server/Cluster Setup.md | 12 +-
docs/UserGuide/Server/ServerFileList.md | 1 -
.../UserGuide/System Tools/MLogParser Tool.md | 21 +-
docs/UserGuide/System Tools/Sync Tool.md | 2 +-
.../zh/SystemDesign/SchemaManager/SchemaManager.md | 20 +-
docs/zh/SystemDesign/StorageEngine/Compaction.md | 10 +-
docs/zh/SystemDesign/TsFile/Format.md | 4 +-
docs/zh/UserGuide/Concept/SDT.md | 22 +-
.../DML Data Manipulation Language.md | 103 +-
.../zh/UserGuide/Operation Manual/SQL Reference.md | 7 +-
docs/zh/UserGuide/Server/Cluster Setup.md | 16 +-
docs/zh/UserGuide/Server/ServerFileList.md | 1 -
.../zh/UserGuide/System Tools/MLogParser Tool.md | 21 +-
docs/zh/UserGuide/System Tools/Sync Tool.md | 2 +-
hive-connector/pom.xml | 6 +-
pom.xml | 12 +-
server/file-changelists/mlog-changelist.md | 4 +
server/pom.xml | 5 -
.../resources/conf/iotdb-engine.properties | 11 +-
.../assembly/resources/tools/mlog/mLogParser.bat | 62 +-
.../assembly/resources/tools/mlog/mLogParser.sh | 40 +-
.../org/apache/iotdb/db/auth/AuthException.java | 5 -
.../iotdb/db/auth/authorizer/BasicAuthorizer.java | 2 +-
.../db/auth/authorizer/LocalFileAuthorizer.java | 2 +-
.../iotdb/db/auth/authorizer/OpenIdAuthorizer.java | 10 +-
.../apache/iotdb/db/auth/entity/PathPrivilege.java | 4 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 16 +-
.../org/apache/iotdb/db/conf/IoTDBConfigCheck.java | 35 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 6 +
.../db/engine/cache/TimeSeriesMetadataCache.java | 10 +-
.../level/LevelCompactionTsFileManagement.java | 152 +--
.../no/NoCompactionTsFileManagement.java | 10 +-
.../compaction/utils/CompactionLogAnalyzer.java | 19 +-
.../engine/compaction/utils/CompactionLogger.java | 16 +-
.../engine/compaction/utils/CompactionUtils.java | 84 +-
.../iotdb/db/engine/flush/MemTableFlushTask.java | 1 +
.../db/engine/storagegroup/TsFileResource.java | 14 +-
.../{MLogWriter.java => MLogTxtWriter.java} | 72 +-
.../org/apache/iotdb/db/metadata/MManager.java | 236 ++--
.../java/org/apache/iotdb/db/metadata/MTree.java | 127 ++-
.../apache/iotdb/db/metadata/MetadataConstant.java | 10 +-
.../iotdb/db/metadata/logfile/MLogReader.java | 74 ++
.../iotdb/db/metadata/logfile/MLogTxtReader.java | 79 ++
.../iotdb/db/metadata/logfile/MLogWriter.java | 454 ++++++++
.../org/apache/iotdb/db/metadata/mnode/MNode.java | 25 +-
.../iotdb/db/metadata/mnode/MeasurementMNode.java | 37 +-
.../iotdb/db/metadata/mnode/StorageGroupMNode.java | 22 +-
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 2 +-
.../org/apache/iotdb/db/qp/logical/Operator.java | 5 +-
.../iotdb/db/qp/logical/crud/QueryOperator.java | 18 +
.../apache/iotdb/db/qp/physical/PhysicalPlan.java | 28 +-
.../iotdb/db/qp/physical/crud/GroupByTimePlan.java | 19 +
...tStorageGroupPlan.java => ChangeAliasPlan.java} | 74 +-
...rageGroupPlan.java => ChangeTagOffsetPlan.java} | 76 +-
.../qp/physical/sys/CreateMultiTimeSeriesPlan.java | 4 +-
.../db/qp/physical/sys/CreateTimeSeriesPlan.java | 66 +-
.../db/qp/physical/sys/DeleteTimeSeriesPlan.java | 4 +
.../{SetStorageGroupPlan.java => MNodePlan.java} | 88 +-
.../db/qp/physical/sys/MeasurementMNodePlan.java | 143 +++
.../db/qp/physical/sys/SetStorageGroupPlan.java | 18 +-
.../iotdb/db/qp/physical/sys/SetTTLPlan.java | 1 -
...geGroupPlan.java => StorageGroupMNodePlan.java} | 79 +-
.../apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java | 18 +-
.../iotdb/db/qp/strategy/PhysicalGenerator.java | 2 +
.../iotdb/db/query/control/TracingManager.java | 2 +-
.../db/query/dataset/ShowTimeseriesDataSet.java | 7 +-
.../dataset/groupby/GroupByEngineDataSet.java | 70 +-
.../iotdb/db/query/executor/LastQueryExecutor.java | 18 +-
.../org/apache/iotdb/db/service/TSServiceImpl.java | 5 +-
.../db/sync/receiver/transfer/SyncServiceImpl.java | 37 +-
.../org/apache/iotdb/db/tools/mlog/MLogParser.java | 194 ++++
.../db/tools/upgrade/TsFileOnlineUpgradeTool.java | 1 +
.../java/org/apache/iotdb/db/utils/QueryUtils.java | 1 -
.../org/apache/iotdb/db/utils/SerializeUtils.java | 12 +-
.../iotdb/db/utils/datastructure/TVList.java | 6 +-
.../org/apache/iotdb/db/writelog/io/LogWriter.java | 56 +-
.../db/writelog/node/ExclusiveWriteLogNode.java | 13 +-
.../apache/iotdb/db/auth/AuthorityCheckerTest.java | 147 +++
.../{ => authorizer}/LocalFileAuthorizerTest.java | 295 +++--
.../db/auth/authorizer/OpenIdAuthorizerTest.java | 107 +-
.../iotdb/db/auth/entity/PathPrivilegeTest.java | 42 +
.../org/apache/iotdb/db/auth/entity/RoleTest.java} | 41 +-
.../org/apache/iotdb/db/auth/entity/UserTest.java | 44 +
.../auth/{ => role}/LocalFileRoleAccessorTest.java | 13 +-
.../auth/{ => role}/LocalFileRoleManagerTest.java | 45 +-
.../auth/{ => user}/LocalFileUserAccessorTest.java | 13 +-
.../auth/{ => user}/LocalFileUserManagerTest.java | 69 +-
.../engine/compaction/LevelCompactionLogTest.java | 82 ++
.../compaction/LevelCompactionMergeTest.java | 142 +++
.../compaction/LevelCompactionRecoverTest.java | 507 +++++++++
.../compaction/LevelCompactionSelectorTest.java | 73 ++
.../LevelCompactionTest.java} | 68 +-
.../LevelCompactionTsFileManagementTest.java | 120 ++
.../NoCompactionTsFileManagementTest.java | 127 +++
.../apache/iotdb/db/engine/merge/MergeTest.java | 2 +-
.../iotdb/db/integration/IOTDBGroupByIT.java | 101 ++
.../apache/iotdb/db/integration/IOTDBInsertIT.java | 4 +-
.../iotdb/db/integration/IoTDBAddSubDeviceIT.java | 15 +-
.../iotdb/db/integration/IoTDBAlignByDeviceIT.java | 31 +-
.../org/apache/iotdb/db/integration/IoTDBAsIT.java | 10 +
.../db/integration/IoTDBAutoCreateSchemaIT.java | 19 +-
.../iotdb/db/integration/IoTDBCompactionIT.java | 154 ---
.../db/integration/IoTDBCreateSnapshotIT.java | 74 +-
.../iotdb/db/integration/IoTDBDeletionIT.java | 18 +
.../apache/iotdb/db/integration/IoTDBLastIT.java | 13 +-
.../db/integration/IoTDBLevelCompactionIT.java | 1180 ++++++++++++++++++++
.../iotdb/db/integration/IoTDBRestartIT.java | 2 -
.../iotdb/db/integration/IoTDBResultSetIT.java | 4 +-
.../IoTDBAggregationByLevelIT.java | 9 +-
.../IoTDBAggregationDeleteIT.java | 16 +-
.../{ => aggregation}/IoTDBAggregationIT.java | 2 +-
.../IoTDBAggregationLargeDataIT.java | 247 +---
.../IoTDBAggregationSmallDataIT.java | 179 +--
.../{ => auth}/IoTDBAuthorizationIT.java | 2 +-
.../iotdb/db/mqtt/BrokerAuthenticatorTest.java | 16 +
.../apache/iotdb/db/qp/plan/PhysicalPlanTest.java | 9 +-
.../db/query/aggregation/AggregateResultTest.java | 19 +-
.../query/aggregation/DescAggregateResultTest.java | 126 +++
.../iotdb/db/query/control/TracingManagerTest.java | 111 ++
.../iotdb/db/query/dataset/ListDataSetTest.java | 126 +++
.../db/query/dataset/ShowTimeSeriesResultTest.java | 57 +
.../iotdb/db/query/dataset/SingleDataSetTest.java | 133 +++
.../dataset/groupby/GroupByEngineDataSetTest.java | 476 ++++++++
.../{ => groupby}/GroupByFillDataSetTest.java | 2 +-
.../{ => groupby}/GroupByLevelDataSetTest.java | 2 +-
.../{ => groupby}/GroupByTimeDataSetTest.java | 2 +-
.../query/executor/GroupByEngineDataSetTest.java | 177 ---
.../org/apache/iotdb/db/tools/MLogParserTest.java | 150 +++
.../org/apache/iotdb/db/tools/WalCheckerTest.java | 5 +-
.../iotdb/db/utils/CopyOnReadLinkedListTest.java | 67 ++
.../iotdb/db/utils/EncodingInferenceUtilsTest.java | 45 +
.../org/apache/iotdb/db/utils/MemUtilsTest.java | 114 ++
.../org/apache/iotdb/db/utils/SchemaUtilsTest.java | 64 ++
.../apache/iotdb/db/utils/SerializeUtilsTest.java | 312 ++++++
.../db/utils/datastructure/BinaryTVListTest.java | 58 +
.../db/utils/datastructure/BooleanTVListTest.java | 62 +
.../db/utils/datastructure/DoubleTVListTest.java | 72 ++
.../db/utils/datastructure/FloatTVListTest.java | 72 ++
.../db/utils/datastructure/IntTVListTest.java | 71 ++
.../db/utils/datastructure/LongTVListTest.java | 18 +
.../iotdb/db/writelog/io/LogWriterReaderTest.java | 4 +-
.../db/writelog/io/MultiFileLogReaderTest.java | 4 +-
.../java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java | 43 +-
.../main/java/org/apache/iotdb/rpc/RpcUtils.java | 90 ++
.../iotdb/session/IoTDBSessionIteratorIT.java | 178 ++-
.../apache/iotdb/session/IoTDBSessionSimpleIT.java | 10 +-
site/README-zh.md | 2 +-
site/README.md | 2 +-
.../theme/global-components/Contributor.vue | 14 +
spark-iotdb-connector/pom.xml | 6 -
tsfile/format-changelist.md | 2 +-
.../org/apache/iotdb/tsfile/file/MetaMarker.java | 4 +
.../iotdb/tsfile/file/footer/ChunkGroupFooter.java | 31 +-
.../file/metadata/MetadataIndexConstructor.java | 9 +-
.../tsfile/file/metadata/MetadataIndexNode.java | 7 +-
.../file/metadata/enums/CompressionType.java | 118 +-
.../iotdb/tsfile/read/TsFileSequenceReader.java | 20 +-
.../iotdb/tsfile/write/writer/TsFileIOWriter.java | 10 +-
.../tsfile/read/TsFileSequenceReaderTest.java | 2 +-
.../iotdb/tsfile/write/TsFileIOWriterTest.java | 8 +
zeppelin-interpreter/IoTDB-Zeppelin-Demo.zpln | 377 +++++++
zeppelin-interpreter/README.md | 186 +++
zeppelin-interpreter/pom.xml | 126 +++
.../apache/zeppelin/iotdb/IoTDBInterpreter.java | 213 ++++
.../src/main/resources/interpreter-setting.json | 70 ++
.../zeppelin/iotdb/IoTDBInterpreterTest.java | 273 +++++
184 files changed, 9383 insertions(+), 2169 deletions(-)
diff --cc cluster/src/main/java/org/apache/iotdb/cluster/partition/slot/SlotPartitionTable.java
index 355ac95,e9295f3..d06de9d
--- a/cluster/src/main/java/org/apache/iotdb/cluster/partition/slot/SlotPartitionTable.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/partition/slot/SlotPartitionTable.java
@@@ -329,23 -310,22 +329,23 @@@ public class SlotPartitionTable impleme
try {
dataOutputStream.writeInt(totalSlotNumbers);
dataOutputStream.writeInt(nodeSlotMap.size());
- for (Entry<Node, List<Integer>> entry : nodeSlotMap.entrySet()) {
- SerializeUtils.serialize(entry.getKey(), dataOutputStream);
- SerializeUtils.serializeIntList(entry.getValue(), dataOutputStream);
+ for (Entry<RaftNode, List<Integer>> entry : nodeSlotMap.entrySet()) {
+ SerializeUtils.serialize(entry.getKey().getNode(), dataOutputStream);
+ dataOutputStream.writeInt(entry.getKey().getRaftId());
- SerializeUtils.serialize(entry.getValue(), dataOutputStream);
++// SerializeUtils.serialize(entry.getValue(), dataOutputStream);
}
- dataOutputStream.writeInt(previousNodeMap.size());
- for (Entry<Node, Map<Integer, Node>> nodeMapEntry : previousNodeMap.entrySet()) {
- dataOutputStream.writeInt(nodeMapEntry.getKey().getNodeIdentifier());
-
- Map<Integer, Node> prevHolders = nodeMapEntry.getValue();
- dataOutputStream.writeInt(prevHolders.size());
- for (Entry<Integer, Node> integerNodeEntry : prevHolders.entrySet()) {
- dataOutputStream.writeInt(integerNodeEntry.getKey());
- dataOutputStream.writeInt(integerNodeEntry.getValue().getNodeIdentifier());
- }
- }
+// dataOutputStream.writeInt(previousNodeMap.size());
+// for (Entry<Node, Map<Integer, Node>> nodeMapEntry : previousNodeMap.entrySet()) {
+// dataOutputStream.writeInt(nodeMapEntry.getKey().getNodeIdentifier());
+//
+// Map<Integer, Node> prevHolders = nodeMapEntry.getValue();
+// dataOutputStream.writeInt(prevHolders.size());
+// for (Entry<Integer, Node> integerNodeEntry : prevHolders.entrySet()) {
+// dataOutputStream.writeInt(integerNodeEntry.getKey());
+// dataOutputStream.writeInt(integerNodeEntry.getValue().getNodeIdentifier());
+// }
+// }
dataOutputStream.writeLong(lastLogIndex);
} catch (IOException ignored) {
@@@ -357,46 -337,40 +357,46 @@@
@Override
public void deserialize(ByteBuffer buffer) {
-- logger.info("Initializing the partition table from buffer");
-- totalSlotNumbers = buffer.getInt();
-- int size = buffer.getInt();
-- Map<Integer, Node> idNodeMap = new HashMap<>();
-- for (int i = 0; i < size; i++) {
-- Node node = new Node();
-- List<Integer> slots = new ArrayList<>();
-- SerializeUtils.deserialize(node, buffer);
- int id = buffer.getInt();
- SerializeUtils.deserialize(slots, buffer);
- RaftNode raftNode = new RaftNode(node, id);
- nodeSlotMap.put(raftNode, slots);
- SerializeUtils.deserializeIntList(slots, buffer);
- nodeSlotMap.put(node, slots);
-- idNodeMap.put(node.getNodeIdentifier(), node);
-- for (Integer slot : slots) {
- slotNodes[slot] = raftNode;
- slotNodes[slot] = node;
-- }
-- }
++// logger.info("Initializing the partition table from buffer");
++// totalSlotNumbers = buffer.getInt();
++// int size = buffer.getInt();
++// Map<Integer, Node> idNodeMap = new HashMap<>();
++// for (int i = 0; i < size; i++) {
++// Node node = new Node();
++// List<Integer> slots = new ArrayList<>();
++// SerializeUtils.deserialize(node, buffer);
++// int id = buffer.getInt();
++// SerializeUtils.deserialize(slots, buffer);
++// RaftNode raftNode = new RaftNode(node, id);
++// nodeSlotMap.put(raftNode, slots);
++// idNodeMap.put(node.getNodeIdentifier(), node);
++// for (Integer slot : slots) {
++// slotNodes[slot] = raftNode;
++// }
++// }
+
+// int prevNodeMapSize = buffer.getInt();
+// previousNodeMap = new HashMap<>();
+// for (int i = 0; i < prevNodeMapSize; i++) {
+// int nodeId = buffer.getInt();
+// Node node = idNodeMap.get(nodeId);
+//
+// Map<Integer, Node> prevHolders = new HashMap<>();
+// int holderNum = buffer.getInt();
+// for (int i1 = 0; i1 < holderNum; i1++) {
+// int slot = buffer.getInt();
+// Node holder = idNodeMap.get(buffer.getInt());
+// prevHolders.put(slot, holder);
+// }
+// previousNodeMap.put(node, prevHolders);
+// }
+ lastLogIndex = buffer.getLong();
- int prevNodeMapSize = buffer.getInt();
- previousNodeMap = new HashMap<>();
- for (int i = 0; i < prevNodeMapSize; i++) {
- int nodeId = buffer.getInt();
- Node node = idNodeMap.get(nodeId);
-
- Map<Integer, Node> prevHolders = new HashMap<>();
- int holderNum = buffer.getInt();
- for (int i1 = 0; i1 < holderNum; i1++) {
- int slot = buffer.getInt();
- Node holder = idNodeMap.get(buffer.getInt());
- prevHolders.put(slot, holder);
+ for (RaftNode raftNode : nodeSlotMap.keySet()) {
+ if (!nodeRing.contains(raftNode.getNode())) {
+ nodeRing.add(raftNode.getNode());
}
- previousNodeMap.put(node, prevHolders);
}
- lastLogIndex = buffer.getLong();
-
- nodeRing.addAll(nodeSlotMap.keySet());
nodeRing.sort(Comparator.comparingInt(Node::getNodeIdentifier));
logger.info("All known nodes: {}", nodeRing);