You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by lt...@apache.org on 2019/04/08 09:41:53 UTC
[incubator-iotdb] branch cluster updated: fix a bug of MManager to
get all file names of a path (#137)
This is an automated email from the ASF dual-hosted git repository.
lta pushed a commit to branch cluster
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/cluster by this push:
new 0e7ad31 fix a bug of MManager to get all file names of a path (#137)
0e7ad31 is described below
commit 0e7ad3131f838ea6b4e966a7b00614ab312b81f7
Author: Tianan Li <li...@163.com>
AuthorDate: Mon Apr 8 17:41:49 2019 +0800
fix a bug of MManager to get all file names of a path (#137)
---
.../cluster/entity/raft/DataStateMachine.java | 17 +++--
.../apache/iotdb/cluster/qp/ClusterQPExecutor.java | 7 --
.../cluster/qp/executor/NonQueryExecutor.java | 21 ++----
.../cluster/qp/executor/QueryMetadataExecutor.java | 16 ++---
.../org/apache/iotdb/cluster/utils/RaftUtils.java | 47 ------------
.../java/org/apache/iotdb/db/metadata/MGraph.java | 10 ---
.../org/apache/iotdb/db/metadata/MManager.java | 13 ----
.../java/org/apache/iotdb/db/metadata/MTree.java | 83 +++++++++-------------
.../iotdb/db/metadata/MManagerBasicTest.java | 28 ++++----
.../org/apache/iotdb/db/metadata/MTreeTest.java | 27 +++----
10 files changed, 85 insertions(+), 184 deletions(-)
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/entity/raft/DataStateMachine.java b/cluster/src/main/java/org/apache/iotdb/cluster/entity/raft/DataStateMachine.java
index 2ac5667..db86eb1 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/entity/raft/DataStateMachine.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/entity/raft/DataStateMachine.java
@@ -40,6 +40,7 @@ import org.apache.iotdb.cluster.rpc.closure.ResponseClosure;
import org.apache.iotdb.cluster.rpc.request.DataGroupNonQueryRequest;
import org.apache.iotdb.cluster.rpc.response.BasicResponse;
import org.apache.iotdb.cluster.utils.RaftUtils;
+import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.ProcessorException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.qp.executor.OverflowQPExecutor;
@@ -115,8 +116,8 @@ public class DataStateMachine extends StateMachineAdapter {
PhysicalPlan plan = PhysicalPlanLogTransfer.logToOperator(planByte);
/** If the request is to set path and sg of the path doesn't exist, it needs to run null-read in meta group to avoid out of data sync **/
- if (plan.getOperatorType() == OperatorType.CREATE_TIMESERIES && !MManager.getInstance()
- .checkStorageExistOfPath(((MetadataPlan) plan).getPath().getFullPath())) {
+ if (plan.getOperatorType() == OperatorType.CREATE_TIMESERIES && checkPathExistence(
+ ((MetadataPlan) plan).getPath().getFullPath())) {
SingleQPTask nullReadTask = new SingleQPTask(false, null);
handleNullReadToMetaGroup(nullReadTask, status, nullReadTask);
}
@@ -124,7 +125,7 @@ public class DataStateMachine extends StateMachineAdapter {
if (closure != null) {
response.addResult(true);
}
- } catch (ProcessorException | IOException e) {
+ } catch (ProcessorException | IOException | PathErrorException e) {
LOGGER.error("Execute physical plan error", e);
status = new Status(-1, e.getMessage());
if (closure != null) {
@@ -140,12 +141,20 @@ public class DataStateMachine extends StateMachineAdapter {
}
/**
+ * Check the existence of a specific path
+ */
+ private boolean checkPathExistence(String path) throws PathErrorException {
+ return !MManager.getInstance().getAllFileNamesByPath(path).isEmpty();
+ }
+
+ /**
* Handle null-read process in metadata group if the request is to set path.
*
* @param qpTask null-read task
* @param originStatus status to return result if this node is leader of the data group
*/
- private void handleNullReadToMetaGroup(QPTask qpTask, Status originStatus, SingleQPTask nullReadTask) {
+ private void handleNullReadToMetaGroup(QPTask qpTask, Status originStatus,
+ SingleQPTask nullReadTask) {
try {
LOGGER.info("Handle null-read in meta group for adding path request.");
final byte[] reqContext = new byte[4];
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/qp/ClusterQPExecutor.java b/cluster/src/main/java/org/apache/iotdb/cluster/qp/ClusterQPExecutor.java
index 3983a45..90438a4 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/qp/ClusterQPExecutor.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/qp/ClusterQPExecutor.java
@@ -119,13 +119,6 @@ public abstract class ClusterQPExecutor {
}
/**
- * Check if the storage group of given path exists in mTree or not.
- */
- public boolean checkStorageExistOfPath(String path) {
- return mManager.checkStorageExistOfPath(path);
- }
-
- /**
* Classify the input storage group list by which data group it belongs to.
*
* @return key is groupId, value is all SGs belong to this data group
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/qp/executor/NonQueryExecutor.java b/cluster/src/main/java/org/apache/iotdb/cluster/qp/executor/NonQueryExecutor.java
index 0a8ca81..af88816 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/qp/executor/NonQueryExecutor.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/qp/executor/NonQueryExecutor.java
@@ -218,22 +218,11 @@ public class NonQueryExecutor extends ClusterQPExecutor {
List<Path> paths = deletePlan.getPaths();
Set<String> sgSet = new HashSet<>();
for (Path path : paths) {
- if (mManager.checkStorageExistOfPath(path.getFullPath())) {
- sgSet.add(mManager.getFileNameByPath(path.getFullPath()));
- if (sgSet.size() > 1) {
- throw new ProcessorException(
- "Delete function in distributed iotdb only supports single storage group");
- }
- } else {
- List<String> storageGroups = mManager.getAllFileNamesByPath(path.getFullPath());
- if (!storageGroups.isEmpty()) {
- throw new ProcessorException(
- "Delete function in distributed iotdb only supports single storage group");
- } else {
- throw new ProcessorException(
- String.format("The path %s doesn't exist.", path.getFullPath()));
- }
-
+ List<String> storageGroupList = mManager.getAllFileNamesByPath(path.getFullPath());
+ sgSet.addAll(storageGroupList);
+ if (sgSet.size() > 1) {
+ throw new ProcessorException(
+ "Delete function in distributed iotdb only supports single storage group");
}
}
List<String> sgList = new ArrayList<>(sgSet);
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/qp/executor/QueryMetadataExecutor.java b/cluster/src/main/java/org/apache/iotdb/cluster/qp/executor/QueryMetadataExecutor.java
index 635948a..39112d9 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/qp/executor/QueryMetadataExecutor.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/qp/executor/QueryMetadataExecutor.java
@@ -70,21 +70,13 @@ public class QueryMetadataExecutor extends ClusterQPExecutor {
public List<List<String>> processTimeSeriesQuery(String path)
throws InterruptedException, PathErrorException, ProcessorException {
List<List<String>> res = new ArrayList<>();
- /** Check whether it's related to one storage group **/
- if (checkStorageExistOfPath(path)) {
- String storageGroup = mManager.getFileNameByPath(path);
- String groupId = getGroupIdBySG(storageGroup);
- List<String> paths = new ArrayList<>();
- paths.add(path);
- handleTimseriesQuery(groupId, paths, res);
+ List<String> storageGroupList = mManager.getAllFileNamesByPath(path);
+ if (storageGroupList.isEmpty()) {
+ throw new PathErrorException(String.format("The path %s doesn't exist", path));
} else {
- List<String> storageGroupList = getAllStroageGroupsByPath(path);
Map<String, Set<String>> groupIdSGMap = classifySGByGroupId(storageGroupList);
for (Entry<String, Set<String>> entry : groupIdSGMap.entrySet()) {
- List<String> paths = new ArrayList<>();
- for (String storageGroup : entry.getValue()) {
- paths.add(storageGroup);
- }
+ List<String> paths = new ArrayList<>(entry.getValue());
String groupId = entry.getKey();
handleTimseriesQuery(groupId, paths, res);
}
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/utils/RaftUtils.java b/cluster/src/main/java/org/apache/iotdb/cluster/utils/RaftUtils.java
index 6d2c162..000da40 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/utils/RaftUtils.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/utils/RaftUtils.java
@@ -63,30 +63,6 @@ public class RaftUtils {
private RaftUtils() {
}
-
- @Deprecated
- /**
- * @deprecated
- * Get leader node according to the group id.
- * <br/> This method will connect to one of nodes in the group to get the correct leader.
- *
- * @param groupId group id of raft group
- * @return PeerId of leader
- */
- public static PeerId getLeaderPeerID(String groupId, BoltCliClientService cliClientService)
- throws RaftConnectionException {
- Configuration conf = getConfiguration(groupId);
- RouteTable.getInstance().updateConfiguration(groupId, conf);
- try {
- if (!RouteTable.getInstance().refreshLeader(cliClientService, groupId, 1000).isOk()) {
- throw new RaftConnectionException("Refresh leader failed");
- }
- } catch (InterruptedException | TimeoutException e) {
- throw new RaftConnectionException("Refresh leader failed");
- }
- return RouteTable.getInstance().selectLeader(groupId);
- }
-
/**
* Get peer id to send request. If groupLeaderCache has the group id, then return leader id of the
* group.Otherwise, random get a peer of the group.
@@ -125,29 +101,6 @@ public class RaftUtils {
return ThreadLocalRandom.current().nextInt(bound);
}
- @Deprecated
- /**
- * Get raft group configuration by group id
- *
- * @param groupID raft group id
- * @return raft group configuration
- */
- public static Configuration getConfiguration(String groupID) {
- //TODO can we reuse Configuration instance?
- Configuration conf = new Configuration();
- RaftService service;
- if (groupID.equals(ClusterConfig.METADATA_GROUP_ID)) {
- service = (RaftService) server.getMetadataHolder().getService();
- conf.setPeers(service.getPeerIdList());
- } else {
- DataPartitionRaftHolder dataPartitionHolder = (DataPartitionRaftHolder) server
- .getDataPartitionHolderMap().get(groupID);
- service = (RaftService) dataPartitionHolder.getService();
- conf.setPeers(service.getPeerIdList());
- }
- return conf;
- }
-
public static PeerId getPeerIDFrom(PhysicalNode node) {
return new PeerId(node.ip, node.port);
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/metadata/MGraph.java b/iotdb/src/main/java/org/apache/iotdb/db/metadata/MGraph.java
index afe0cdc..c3a5f5a 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/metadata/MGraph.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/metadata/MGraph.java
@@ -170,16 +170,6 @@ public class MGraph implements Serializable {
}
/**
- * Check whether the storage group of the input path exists or not
- *
- * @param path Format: root.node.(node)*
- * @apiNote :for cluster
- */
- public boolean checkStorageExistOfPath(String path) {
- return mtree.checkStorageExistOfPath(path);
- }
-
- /**
* Get all paths for given seriesPath regular expression if given seriesPath belongs to MTree, or
* get all linked seriesPath for given seriesPath if given seriesPath belongs to PTree Notice:
* Regular expression in this method is formed by the amalgamation of seriesPath and the character
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/iotdb/src/main/java/org/apache/iotdb/db/metadata/MManager.java
index 2e6ed28..5a752f8 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/metadata/MManager.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/metadata/MManager.java
@@ -355,19 +355,6 @@ public class MManager {
}
/**
- * function for checking if the storage group of given path exists in mTree or not.
- * @apiNote :for cluster
- */
- public boolean checkStorageExistOfPath(String path) {
- lock.readLock().lock();
- try {
- return mgraph.checkStorageExistOfPath(path);
- } finally {
- lock.readLock().unlock();
- }
- }
-
- /**
* function for adding a pTree.
*/
public void addAPTree(String ptreeRootName) throws IOException, MetadataArgsErrorException {
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/metadata/MTree.java b/iotdb/src/main/java/org/apache/iotdb/db/metadata/MTree.java
index f53a2c6..22236da 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/metadata/MTree.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/metadata/MTree.java
@@ -248,38 +248,6 @@ public class MTree implements Serializable {
}
/**
- * Check whether the storage group of the path exists or not
- * @param path input path
- * @return If it's storage group exists, return true. Else return false
- * @apiNote :for cluster
- */
- public boolean checkStorageExistOfPath(String path) {
- String[] nodeNames = path.split(DOUB_SEPARATOR);
- MNode cur = root;
- if (nodeNames.length <= 1 || !nodeNames[0].equals(root.getName())) {
- return false;
- }
- int i = 1;
- while (i < nodeNames.length - 1) {
- MNode temp = cur.getChild(nodeNames[i]);
- if (temp == null) {
- return false;
- }
- if(temp.isStorageLevel()){
- return true;
- }
- cur = cur.getChild(nodeNames[i]);
- i++;
- }
- MNode temp = cur.getChild(nodeNames[i]);
- if(temp != null && temp.isStorageLevel()) {
- return true;
- } else {
- return false;
- }
- }
-
- /**
* Check whether set file seriesPath for this node or not. If not, throw an exception
*/
private void checkStorageGroup(MNode node) throws PathErrorException {
@@ -588,28 +556,45 @@ public class MTree implements Serializable {
* Get all the storage group seriesPaths for one seriesPath.
*
* @return List storage group seriesPath list
+ * @apiNote :for cluster
*/
- public List<String> getAllFileNamesByPath(String path) throws PathErrorException {
+ public List<String> getAllFileNamesByPath(String pathReg) throws PathErrorException {
+ ArrayList<String> fileNames = new ArrayList<>();
+ String[] nodes = pathReg.split(DOUB_SEPARATOR);
+ if (nodes.length == 0 || !nodes[0].equals(getRoot().getName())) {
+ throw new PathErrorException(String.format(SERIES_NOT_CORRECT, pathReg));
+ }
+ findFileName(getRoot(), nodes, 1, "", fileNames);
+ return fileNames;
+ }
- List<String> sgList = new ArrayList<>();
- String[] nodes = path.split(DOUB_SEPARATOR);
- MNode cur = getRoot();
- for (int i = 1; i < nodes.length; i++) {
- if (cur == null) {
- throw new PathErrorException(
- String.format(NOT_SERIES_PATH,
- path));
+ /**
+ * Recursively find all fileName according to a specific path
+ * @apiNote :for cluster
+ */
+ private void findFileName(MNode node, String[] nodes, int idx, String parent,
+ ArrayList<String> paths) {
+ if (node.isStorageLevel()) {
+ paths.add(node.getDataFileName());
+ return;
+ }
+ String nodeReg;
+ if (idx >= nodes.length) {
+ nodeReg = "*";
+ } else {
+ nodeReg = nodes[idx];
+ }
+
+ if (!("*").equals(nodeReg)) {
+ if (node.hasChild(nodeReg)) {
+ findFileName(node.getChild(nodeReg), nodes, idx + 1, parent + node.getName() + ".", paths);
}
- if (cur.isStorageLevel()) {
- sgList.add(cur.getDataFileName());
- return sgList;
+ } else {
+ for (MNode child : node.getChildren().values()) {
+ findFileName(child, nodes, idx + 1, parent + node.getName() + ".", paths);
}
- cur = cur.getChild(nodes[i]);
- }
- if (sgList.isEmpty()) {
- getAllStorageGroupsOfNode(cur, path, sgList);
}
- return sgList;
+ return;
}
private void getAllStorageGroupsOfNode(MNode node, String path, List<String> sgList) {
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java b/iotdb/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java
index 7a952ad..4fbf8bf 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java
@@ -19,6 +19,8 @@
package org.apache.iotdb.db.metadata;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.IOException;
@@ -320,23 +322,23 @@ public class MManagerBasicTest {
MManager manager = MManager.getInstance();
try {
- assertEquals(false, manager.checkStorageExistOfPath("root"));
- assertEquals(false, manager.checkStorageExistOfPath("root.vehicle"));
- assertEquals(false, manager.checkStorageExistOfPath("root.vehicle.device"));
- assertEquals(false, manager.checkStorageExistOfPath("root.vehicle.device.sensor"));
+ assertTrue(manager.getAllPathGroupByFileName("root").keySet().isEmpty());
+ assertTrue(manager.getAllFileNamesByPath("root.vehicle").isEmpty());
+ assertTrue(manager.getAllFileNamesByPath("root.vehicle.device").isEmpty());
+ assertTrue(manager.getAllFileNamesByPath("root.vehicle.device.sensor").isEmpty());
manager.setStorageLevelToMTree("root.vehicle");
- assertEquals(true, manager.checkStorageExistOfPath("root.vehicle"));
- assertEquals(true, manager.checkStorageExistOfPath("root.vehicle.device"));
- assertEquals(true, manager.checkStorageExistOfPath("root.vehicle.device.sensor"));
- assertEquals(false, manager.checkStorageExistOfPath("root.vehicle1"));
- assertEquals(false, manager.checkStorageExistOfPath("root.vehicle1.device"));
+ assertFalse(manager.getAllFileNamesByPath("root.vehicle").isEmpty());
+ assertFalse(manager.getAllFileNamesByPath("root.vehicle.device").isEmpty());
+ assertFalse(manager.getAllFileNamesByPath("root.vehicle.device.sensor").isEmpty());
+ assertTrue(manager.getAllFileNamesByPath("root.vehicle1").isEmpty());
+ assertTrue(manager.getAllFileNamesByPath("root.vehicle1.device").isEmpty());
manager.setStorageLevelToMTree("root.vehicle1.device");
- assertEquals(false, manager.checkStorageExistOfPath("root.vehicle1.device1"));
- assertEquals(false, manager.checkStorageExistOfPath("root.vehicle1.device2"));
- assertEquals(false, manager.checkStorageExistOfPath("root.vehicle1.device3"));
- assertEquals(true, manager.checkStorageExistOfPath("root.vehicle1.device"));
+ assertTrue(manager.getAllFileNamesByPath("root.vehicle1.device1").isEmpty());
+ assertTrue(manager.getAllFileNamesByPath("root.vehicle1.device2").isEmpty());
+ assertTrue(manager.getAllFileNamesByPath("root.vehicle1.device3").isEmpty());
+ assertFalse(manager.getAllFileNamesByPath("root.vehicle1.device").isEmpty());
} catch (PathErrorException | IOException e) {
e.printStackTrace();
fail(e.getMessage());
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/metadata/MTreeTest.java b/iotdb/src/test/java/org/apache/iotdb/db/metadata/MTreeTest.java
index d7bfb69..af1175a 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/metadata/MTreeTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/metadata/MTreeTest.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.metadata;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -291,23 +292,23 @@ public class MTreeTest {
// set storage group first
MTree root = new MTree("root");
try {
- assertEquals(false, root.checkStorageExistOfPath("root"));
- assertEquals(false, root.checkStorageExistOfPath("root.vehicle"));
- assertEquals(false, root.checkStorageExistOfPath("root.vehicle.device"));
- assertEquals(false, root.checkStorageExistOfPath("root.vehicle.device.sensor"));
+ assertTrue(root.getAllFileNamesByPath("root").isEmpty());
+ assertTrue(root.getAllFileNamesByPath("root.vehicle").isEmpty());
+ assertTrue(root.getAllFileNamesByPath("root.vehicle.device").isEmpty());
+ assertTrue(root.getAllFileNamesByPath("root.vehicle.device.sensor").isEmpty());
root.setStorageGroup("root.vehicle");
- assertEquals(true, root.checkStorageExistOfPath("root.vehicle"));
- assertEquals(true, root.checkStorageExistOfPath("root.vehicle.device"));
- assertEquals(true, root.checkStorageExistOfPath("root.vehicle.device.sensor"));
- assertEquals(false, root.checkStorageExistOfPath("root.vehicle1"));
- assertEquals(false, root.checkStorageExistOfPath("root.vehicle1.device"));
+ assertFalse(root.getAllFileNamesByPath("root.vehicle").isEmpty());
+ assertFalse(root.getAllFileNamesByPath("root.vehicle.device").isEmpty());
+ assertFalse(root.getAllFileNamesByPath("root.vehicle.device.sensor").isEmpty());
+ assertTrue(root.getAllFileNamesByPath("root.vehicle1").isEmpty());
+ assertTrue(root.getAllFileNamesByPath("root.vehicle1.device").isEmpty());
root.setStorageGroup("root.vehicle1.device");
- assertEquals(false, root.checkStorageExistOfPath("root.vehicle1.device1"));
- assertEquals(false, root.checkStorageExistOfPath("root.vehicle1.device2"));
- assertEquals(false, root.checkStorageExistOfPath("root.vehicle1.device3"));
- assertEquals(true, root.checkStorageExistOfPath("root.vehicle1.device"));
+ assertTrue(root.getAllFileNamesByPath("root.vehicle1.device1").isEmpty());
+ assertTrue(root.getAllFileNamesByPath("root.vehicle1.device2").isEmpty());
+ assertTrue(root.getAllFileNamesByPath("root.vehicle1.device3").isEmpty());
+ assertFalse(root.getAllFileNamesByPath("root.vehicle1.device").isEmpty());
} catch (PathErrorException e) {
e.printStackTrace();
fail(e.getMessage());