You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ta...@apache.org on 2022/06/11 03:04:20 UTC
[iotdb] branch master updated: [IOTDB-3448] Migrate the logic of deleteRegion onto the consensus module (#6216)
This is an automated email from the ASF dual-hosted git repository.
tanxinyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 033ecd148f [IOTDB-3448] Migrate the logic of deleteRegion onto the consensus module (#6216)
033ecd148f is described below
commit 033ecd148f1f1b9353ca28a9fc4c198982731d09
Author: Potato <TX...@gmail.com>
AuthorDate: Sat Jun 11 11:04:15 2022 +0800
[IOTDB-3448] Migrate the logic of deleteRegion onto the consensus module (#6216)
* finish
* fix statuscode
---
.../iotdb/consensus/ratis/RatisConsensus.java | 2 +-
.../statemachine/visitor/DataExecutionVisitor.java | 10 --
.../apache/iotdb/db/engine/StorageEngineV2.java | 5 +-
.../metadata/visitor/SchemaExecutionVisitor.java | 14 ---
.../plan/planner/plan/node/DeleteRegionNode.java | 113 ---------------------
.../mpp/plan/planner/plan/node/PlanNodeType.java | 3 -
.../db/mpp/plan/planner/plan/node/PlanVisitor.java | 4 -
.../service/thrift/impl/InternalServiceImpl.java | 39 ++++---
.../plan/node/write/DeleteRegionNodeSerdeTest.java | 60 -----------
9 files changed, 29 insertions(+), 221 deletions(-)
diff --git a/consensus/src/main/java/org/apache/iotdb/consensus/ratis/RatisConsensus.java b/consensus/src/main/java/org/apache/iotdb/consensus/ratis/RatisConsensus.java
index ff1a207403..e84e3fdb46 100644
--- a/consensus/src/main/java/org/apache/iotdb/consensus/ratis/RatisConsensus.java
+++ b/consensus/src/main/java/org/apache/iotdb/consensus/ratis/RatisConsensus.java
@@ -302,7 +302,7 @@ class RatisConsensus implements IConsensus {
client
.getRaftClient()
.getGroupManagementApi(myself.getId())
- .remove(raftGroupId, false, false);
+ .remove(raftGroupId, true, false);
if (!reply.isSuccess()) {
return failed(new RatisRequestFailedException(reply.getException()));
}
diff --git a/server/src/main/java/org/apache/iotdb/db/consensus/statemachine/visitor/DataExecutionVisitor.java b/server/src/main/java/org/apache/iotdb/db/consensus/statemachine/visitor/DataExecutionVisitor.java
index b963ec138d..bbda1bc260 100644
--- a/server/src/main/java/org/apache/iotdb/db/consensus/statemachine/visitor/DataExecutionVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/consensus/statemachine/visitor/DataExecutionVisitor.java
@@ -19,15 +19,12 @@
package org.apache.iotdb.db.consensus.statemachine.visitor;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
-import org.apache.iotdb.commons.consensus.DataRegionId;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.utils.StatusUtils;
-import org.apache.iotdb.db.engine.StorageEngineV2;
import org.apache.iotdb.db.engine.storagegroup.DataRegion;
import org.apache.iotdb.db.exception.BatchProcessException;
import org.apache.iotdb.db.exception.TriggerExecutionException;
import org.apache.iotdb.db.exception.WriteProcessException;
-import org.apache.iotdb.db.mpp.plan.planner.plan.node.DeleteRegionNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanVisitor;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.DeleteDataNode;
@@ -110,13 +107,6 @@ public class DataExecutionVisitor extends PlanVisitor<TSStatus, DataRegion> {
}
}
- @Override
- public TSStatus visitDeleteRegion(DeleteRegionNode node, DataRegion dataRegion) {
- dataRegion.syncDeleteDataFiles();
- StorageEngineV2.getInstance().deleteDataRegion((DataRegionId) node.getConsensusGroupId());
- return StatusUtils.OK;
- }
-
@Override
public TSStatus visitDeleteData(DeleteDataNode node, DataRegion dataRegion) {
try {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngineV2.java b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngineV2.java
index b33b6f0ef9..ccf1360309 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngineV2.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngineV2.java
@@ -575,7 +575,10 @@ public class StorageEngineV2 implements IService {
}
public void deleteDataRegion(DataRegionId regionId) {
- dataRegionMap.remove(regionId);
+ DataRegion region = dataRegionMap.remove(regionId);
+ if (region != null) {
+ region.syncDeleteDataFiles();
+ }
}
public DataRegion getDataRegion(DataRegionId regionId) {
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/visitor/SchemaExecutionVisitor.java b/server/src/main/java/org/apache/iotdb/db/metadata/visitor/SchemaExecutionVisitor.java
index f4bc5c4bae..3e5cc1068b 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/visitor/SchemaExecutionVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/visitor/SchemaExecutionVisitor.java
@@ -21,12 +21,9 @@ package org.apache.iotdb.db.metadata.visitor;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.conf.IoTDBConstant;
-import org.apache.iotdb.commons.consensus.SchemaRegionId;
import org.apache.iotdb.commons.exception.MetadataException;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.db.metadata.schemaregion.ISchemaRegion;
-import org.apache.iotdb.db.metadata.schemaregion.SchemaEngine;
-import org.apache.iotdb.db.mpp.plan.planner.plan.node.DeleteRegionNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanVisitor;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.metedata.write.AlterTimeSeriesNode;
@@ -201,16 +198,5 @@ public class SchemaExecutionVisitor extends PlanVisitor<TSStatus, ISchemaRegion>
}
}
- @Override
- public TSStatus visitDeleteRegion(DeleteRegionNode node, ISchemaRegion schemaRegion) {
- try {
- SchemaEngine.getInstance().deleteSchemaRegion((SchemaRegionId) node.getConsensusGroupId());
- } catch (MetadataException e) {
- logger.error("{}: MetaData error: ", IoTDBConstant.GLOBAL_DB_NAME, e);
- return RpcUtils.getStatus(TSStatusCode.METADATA_ERROR, e.getMessage());
- }
- return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS, "Execute successfully");
- }
-
private static class TransformerContext {}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/DeleteRegionNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/DeleteRegionNode.java
deleted file mode 100644
index ff1bb66f7e..0000000000
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/DeleteRegionNode.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.mpp.plan.planner.plan.node;
-
-import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
-import org.apache.iotdb.commons.consensus.ConsensusGroupId;
-import org.apache.iotdb.consensus.common.request.IConsensusRequest;
-import org.apache.iotdb.db.mpp.plan.analyze.Analysis;
-import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
-
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.List;
-
-public class DeleteRegionNode extends WritePlanNode implements IConsensusRequest {
-
- protected ConsensusGroupId consensusGroupId;
-
- public DeleteRegionNode(PlanNodeId id) {
- super(id);
- }
-
- public DeleteRegionNode(PlanNodeId id, ConsensusGroupId consensusGroupId) {
- super(id);
- this.consensusGroupId = consensusGroupId;
- }
-
- public ConsensusGroupId getConsensusGroupId() {
- return consensusGroupId;
- }
-
- public void setConsensusGroupId(ConsensusGroupId consensusGroupId) {
- this.consensusGroupId = consensusGroupId;
- }
-
- @Override
- public List<PlanNode> getChildren() {
- return null;
- }
-
- @Override
- public void addChild(PlanNode child) {}
-
- @Override
- public PlanNode clone() {
- return null;
- }
-
- @Override
- public int allowedChildCount() {
- return 0;
- }
-
- @Override
- public List<String> getOutputColumnNames() {
- return null;
- }
-
- @Override
- public TRegionReplicaSet getRegionReplicaSet() {
- return null;
- }
-
- @Override
- public List<WritePlanNode> splitByPartition(Analysis analysis) {
- return null;
- }
-
- @Override
- public <R, C> R accept(PlanVisitor<R, C> visitor, C schemaRegion) {
- return visitor.visitDeleteRegion(this, schemaRegion);
- }
-
- public static DeleteRegionNode deserialize(ByteBuffer byteBuffer) {
- int type = ReadWriteIOUtils.readInt(byteBuffer);
- int id = ReadWriteIOUtils.readInt(byteBuffer);
- ConsensusGroupId consensusGroupId = ConsensusGroupId.Factory.create(type, id);
- String planNodeId = ReadWriteIOUtils.readString(byteBuffer);
- return new DeleteRegionNode(new PlanNodeId(planNodeId), consensusGroupId);
- }
-
- @Override
- protected void serializeAttributes(ByteBuffer byteBuffer) {
- PlanNodeType.DELETE_REGION.serialize(byteBuffer);
- ReadWriteIOUtils.write(consensusGroupId.getType().getValue(), byteBuffer);
- ReadWriteIOUtils.write(consensusGroupId.getId(), byteBuffer);
- }
-
- @Override
- protected void serializeAttributes(DataOutputStream stream) throws IOException {
- PlanNodeType.DELETE_REGION.serialize(stream);
- ReadWriteIOUtils.write(consensusGroupId.getType().getValue(), stream);
- ReadWriteIOUtils.write(consensusGroupId.getId(), stream);
- }
-}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/PlanNodeType.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/PlanNodeType.java
index dfd2c5896a..d95329a3e3 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/PlanNodeType.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/PlanNodeType.java
@@ -114,7 +114,6 @@ public enum PlanNodeType {
DEVICE_MERGE((short) 35),
SCHEMA_FETCH_MERGE((short) 36),
TRANSFORM((short) 37),
- DELETE_REGION((short) 38),
CREATE_MULTI_TIME_SERIES((short) 39),
NODE_PATHS_SCAN((short) 40),
NODE_PATHS_CONVERT((short) 41),
@@ -233,8 +232,6 @@ public enum PlanNodeType {
return SchemaFetchMergeNode.deserialize(buffer);
case 37:
return TransformNode.deserialize(buffer);
- case 38:
- return DeleteRegionNode.deserialize(buffer);
case 39:
return CreateMultiTimeSeriesNode.deserialize(buffer);
case 40:
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/PlanVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/PlanVisitor.java
index 816dc19d8e..388f0e1a56 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/PlanVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/PlanVisitor.java
@@ -216,10 +216,6 @@ public abstract class PlanVisitor<R, C> {
return visitPlan(node, context);
}
- public R visitDeleteRegion(DeleteRegionNode node, C context) {
- return visitPlan(node, context);
- }
-
public R visitInsertRow(InsertRowNode node, C context) {
return visitPlan(node, context);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/InternalServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/InternalServiceImpl.java
index 6d62e18228..1d1b061a11 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/InternalServiceImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/InternalServiceImpl.java
@@ -26,6 +26,7 @@ import org.apache.iotdb.common.rpc.thrift.THeartbeatReq;
import org.apache.iotdb.common.rpc.thrift.THeartbeatResp;
import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.consensus.ConsensusGroupId;
import org.apache.iotdb.commons.consensus.DataRegionId;
import org.apache.iotdb.commons.consensus.SchemaRegionId;
@@ -38,6 +39,7 @@ import org.apache.iotdb.consensus.common.Peer;
import org.apache.iotdb.consensus.common.response.ConsensusGenericResponse;
import org.apache.iotdb.consensus.common.response.ConsensusReadResponse;
import org.apache.iotdb.consensus.common.response.ConsensusWriteResponse;
+import org.apache.iotdb.consensus.exception.PeerNotInConsensusGroupException;
import org.apache.iotdb.db.auth.AuthorizerManager;
import org.apache.iotdb.db.consensus.DataRegionConsensusImpl;
import org.apache.iotdb.db.consensus.SchemaRegionConsensusImpl;
@@ -47,16 +49,12 @@ import org.apache.iotdb.db.exception.sql.SemanticException;
import org.apache.iotdb.db.metadata.cache.DataNodeSchemaCache;
import org.apache.iotdb.db.metadata.schemaregion.SchemaEngine;
import org.apache.iotdb.db.mpp.common.FragmentInstanceId;
-import org.apache.iotdb.db.mpp.common.QueryId;
import org.apache.iotdb.db.mpp.execution.fragment.FragmentInstanceInfo;
import org.apache.iotdb.db.mpp.execution.fragment.FragmentInstanceManager;
-import org.apache.iotdb.db.mpp.plan.Coordinator;
import org.apache.iotdb.db.mpp.plan.analyze.ClusterPartitionFetcher;
import org.apache.iotdb.db.mpp.plan.analyze.SchemaValidator;
import org.apache.iotdb.db.mpp.plan.planner.plan.FragmentInstance;
-import org.apache.iotdb.db.mpp.plan.planner.plan.node.DeleteRegionNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode;
-import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeId;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeType;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertNode;
import org.apache.iotdb.db.service.metrics.MetricsService;
@@ -355,22 +353,33 @@ public class InternalServiceImpl implements InternalService.Iface {
@Override
public TSStatus deleteRegion(TConsensusGroupId tconsensusGroupId) throws TException {
- QueryId queryId = Coordinator.getInstance().createQueryId();
- PlanNodeId planNodeId = queryId.genPlanNodeId();
- DeleteRegionNode deleteRegionNode = new DeleteRegionNode(queryId.genPlanNodeId());
ConsensusGroupId consensusGroupId =
ConsensusGroupId.Factory.createFromTConsensusGroupId(tconsensusGroupId);
- deleteRegionNode.setConsensusGroupId(consensusGroupId);
- deleteRegionNode.setPlanNodeId(planNodeId);
if (consensusGroupId instanceof DataRegionId) {
- return DataRegionConsensusImpl.getInstance()
- .write(consensusGroupId, deleteRegionNode)
- .getStatus();
+ ConsensusGenericResponse response =
+ DataRegionConsensusImpl.getInstance().removeConsensusGroup(consensusGroupId);
+ if (!response.isSuccess()
+ && !(response.getException() instanceof PeerNotInConsensusGroupException)) {
+ return RpcUtils.getStatus(
+ TSStatusCode.DELETE_REGION_ERROR, response.getException().getMessage());
+ }
+ StorageEngineV2.getInstance().deleteDataRegion((DataRegionId) consensusGroupId);
} else {
- return SchemaRegionConsensusImpl.getInstance()
- .write(consensusGroupId, deleteRegionNode)
- .getStatus();
+ ConsensusGenericResponse response =
+ SchemaRegionConsensusImpl.getInstance().removeConsensusGroup(consensusGroupId);
+ if (!response.isSuccess()
+ && !(response.getException() instanceof PeerNotInConsensusGroupException)) {
+ return RpcUtils.getStatus(
+ TSStatusCode.DELETE_REGION_ERROR, response.getException().getMessage());
+ }
+ try {
+ SchemaEngine.getInstance().deleteSchemaRegion((SchemaRegionId) consensusGroupId);
+ } catch (MetadataException e) {
+ LOGGER.error("{}: MetaData error: ", IoTDBConstant.GLOBAL_DB_NAME, e);
+ return RpcUtils.getStatus(TSStatusCode.METADATA_ERROR, e.getMessage());
+ }
}
+ return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS, "Execute successfully");
}
@Override
diff --git a/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/write/DeleteRegionNodeSerdeTest.java b/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/write/DeleteRegionNodeSerdeTest.java
deleted file mode 100644
index 7c3024cca3..0000000000
--- a/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/write/DeleteRegionNodeSerdeTest.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.mpp.plan.plan.node.write;
-
-import org.apache.iotdb.commons.consensus.DataRegionId;
-import org.apache.iotdb.commons.consensus.SchemaRegionId;
-import org.apache.iotdb.commons.exception.IllegalPathException;
-import org.apache.iotdb.db.mpp.plan.planner.plan.node.DeleteRegionNode;
-import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeId;
-import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeType;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.nio.ByteBuffer;
-
-public class DeleteRegionNodeSerdeTest {
- @Test
- public void testDataSerializeAndDeserialize() throws IllegalPathException {
- DeleteRegionNode dataRegionNode = new DeleteRegionNode(new PlanNodeId("plan node 1"));
- DataRegionId dataRegionId = new DataRegionId(1);
- dataRegionNode.setConsensusGroupId(dataRegionId);
- ByteBuffer byteBuffer = ByteBuffer.allocate(10000);
- dataRegionNode.serialize(byteBuffer);
- byteBuffer.flip();
- Assert.assertEquals(PlanNodeType.DELETE_REGION.ordinal(), byteBuffer.getShort());
- DeleteRegionNode deserialize = DeleteRegionNode.deserialize(byteBuffer);
- Assert.assertEquals(deserialize.getConsensusGroupId(), dataRegionId);
- }
-
- @Test
- public void testSchemaSerializeAndDeserialize() throws IllegalPathException {
- DeleteRegionNode dataRegionNode = new DeleteRegionNode(new PlanNodeId("plan node 1"));
- SchemaRegionId schemaRegionId = new SchemaRegionId(1);
- dataRegionNode.setConsensusGroupId(schemaRegionId);
- ByteBuffer byteBuffer = ByteBuffer.allocate(10000);
- dataRegionNode.serialize(byteBuffer);
- byteBuffer.flip();
- Assert.assertEquals(PlanNodeType.DELETE_REGION.ordinal(), byteBuffer.getShort());
- DeleteRegionNode deserialize = DeleteRegionNode.deserialize(byteBuffer);
- Assert.assertEquals(deserialize.getConsensusGroupId(), schemaRegionId);
- }
-}