You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2020/06/04 12:13:36 UTC
[incubator-iotdb] branch master updated: [IOTDB-695] Accelerate the
count timeseries query (#1311)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new b1d422a [IOTDB-695] Accelerate the count timeseries query (#1311)
b1d422a is described below
commit b1d422ac880aff0975204f4f6f4c903d4be189f4
Author: Xiangwei Wei <34...@users.noreply.github.com>
AuthorDate: Thu Jun 4 20:13:24 2020 +0800
[IOTDB-695] Accelerate the count timeseries query (#1311)
* Accelerate count timeseires
---
.../org/apache/iotdb/db/metadata/MManager.java | 31 ++++-
.../java/org/apache/iotdb/db/metadata/MTree.java | 149 +++++++++++++++++----
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 23 +++-
.../org/apache/iotdb/db/metadata/MTreeTest.java | 27 ++++
4 files changed, 191 insertions(+), 39 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
index 6a02e7b..63f814a 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
@@ -663,8 +663,6 @@ public class MManager {
lock.readLock().lock();
try {
return mtree.getAllTimeseriesName(prefixPath);
- } catch (MetadataException e) {
- throw new MetadataException(e);
} finally {
lock.readLock().unlock();
}
@@ -677,8 +675,33 @@ public class MManager {
lock.readLock().lock();
try {
return mtree.getAllTimeseriesPath(prefixPath);
- } catch (MetadataException e) {
- throw new MetadataException(e);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ /**
+ * To calculate the count of timeseries for given prefix path.
+ */
+ public int getAllTimeseriesCount(String prefixPath) throws MetadataException {
+ lock.readLock().lock();
+ try {
+ return mtree.getAllTimeseriesCount(prefixPath);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ /**
+ * To calculate the count of nodes in the given level for given prefix path.
+ *
+ * @param prefixPath a prefix path or a full path, can not contain '*'
+ * @param level the level can not be smaller than the level of the prefixPath
+ */
+ public int getNodesCountInGivenLevel(String prefixPath, int level) throws MetadataException {
+ lock.readLock().lock();
+ try {
+ return mtree.getNodesCountInGivenLevel(prefixPath, level);
} finally {
lock.readLock().unlock();
}
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java b/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
index 86134b3..254b549 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
@@ -60,7 +60,9 @@ import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
-/** The hierarchical struct of the Metadata Tree is implemented in this class. */
+/**
+ * The hierarchical struct of the Metadata Tree is implemented in this class.
+ */
public class MTree implements Serializable {
private static final long serialVersionUID = -4200394435237291964L;
@@ -79,12 +81,12 @@ public class MTree implements Serializable {
* Create a timeseries with a full path from root to leaf node Before creating a timeseries, the
* storage group should be set first, throw exception otherwise
*
- * @param path timeseries path
- * @param dataType data type
- * @param encoding encoding
+ * @param path timeseries path
+ * @param dataType data type
+ * @param encoding encoding
* @param compressor compressor
- * @param props props
- * @param alias alias of measurement
+ * @param props props
+ * @param alias alias of measurement
*/
LeafMNode createTimeseries(
String path,
@@ -215,7 +217,9 @@ public class MTree implements Serializable {
}
}
- /** Delete a storage group */
+ /**
+ * Delete a storage group
+ */
List<LeafMNode> deleteStorageGroup(String path) throws MetadataException {
MNode cur = getNodeByPath(path);
if (!(cur instanceof StorageGroupMNode)) {
@@ -351,7 +355,9 @@ public class MTree implements Serializable {
return cur;
}
- /** Get storage group node, if the give path is not a storage group, throw exception */
+ /**
+ * Get storage group node, if the give path is not a storage group, throw exception
+ */
StorageGroupMNode getStorageGroupNode(String path) throws MetadataException {
MNode node = getNodeByPath(path);
if (node instanceof StorageGroupMNode) {
@@ -361,7 +367,9 @@ public class MTree implements Serializable {
}
}
- /** Get device node, if the give path is not a device, throw exception */
+ /**
+ * Get device node, if the give path is not a device, throw exception
+ */
MNode getDeviceNode(String path) throws MetadataException {
return getNodeByPath(path);
}
@@ -451,7 +459,9 @@ public class MTree implements Serializable {
return res;
}
- /** Get all storage group MNodes */
+ /**
+ * Get all storage group MNodes
+ */
List<StorageGroupMNode> getAllStorageGroupNodes() {
List<StorageGroupMNode> ret = new ArrayList<>();
Deque<MNode> nodeStack = new ArrayDeque<>();
@@ -488,7 +498,9 @@ public class MTree implements Serializable {
throw new StorageGroupNotSetException(path);
}
- /** Check whether the given path contains a storage group */
+ /**
+ * Check whether the given path contains a storage group
+ */
boolean checkStorageGroupByPath(String path) {
String[] nodes = MetaUtils.getNodeNames(path);
MNode cur = root;
@@ -530,7 +542,7 @@ public class MTree implements Serializable {
List<Path> paths = new ArrayList<>();
for (String[] p : res) {
Path path = new Path(p[0]);
- if (prePath.getMeasurement().equals(p[1])){
+ if (prePath.getMeasurement().equals(p[1])) {
path.setAlias(p[1]);
}
paths.add(path);
@@ -539,6 +551,83 @@ public class MTree implements Serializable {
}
/**
+ * Get the count of timeseries under the given prefix path.
+ *
+ * @param prefixPath a prefix path or a full path, may contain '*'.
+ */
+ int getAllTimeseriesCount(String prefixPath) throws MetadataException {
+ String[] nodes = MetaUtils.getNodeNames(prefixPath);
+ if (nodes.length == 0 || !nodes[0].equals(root.getName())) {
+ throw new IllegalPathException(prefixPath);
+ }
+ return getCount(root, nodes, 1);
+ }
+
+ /**
+ * Get the count of nodes in the given level under the given prefix path.
+ */
+ int getNodesCountInGivenLevel(String prefixPath, int level) throws MetadataException {
+ String[] nodes = MetaUtils.getNodeNames(prefixPath);
+ if (nodes.length == 0 || !nodes[0].equals(root.getName())) {
+ throw new IllegalPathException(prefixPath);
+ }
+ MNode node = root;
+ for (int i = 1; i < nodes.length; i++) {
+ if (node.getChild(nodes[i]) != null) {
+ node = node.getChild(nodes[i]);
+ } else {
+ throw new MetadataException(nodes[i - 1] + " does not have the child node " + nodes[i]);
+ }
+ }
+ return getCountInGivenLevel(node, level - (nodes.length - 1));
+ }
+
+ /**
+ * Traverse the MTree to get the count of timeseries.
+ */
+ private int getCount(MNode node, String[] nodes, int idx) throws MetadataException {
+ String nodeReg = MetaUtils.getNodeRegByIdx(idx, nodes);
+ if (!(PATH_WILDCARD).equals(nodeReg)) {
+ if (node.hasChild(nodeReg)) {
+ if (node.getChild(nodeReg) instanceof LeafMNode) {
+ return 1;
+ } else {
+ return getCount(node.getChild(nodeReg), nodes, idx + 1);
+ }
+ } else {
+ throw new MetadataException(node.getName() + " does not have the child node " + nodeReg);
+ }
+ } else {
+ int cnt = 0;
+ for (MNode child : node.getChildren().values()) {
+ if (child instanceof LeafMNode) {
+ cnt++;
+ } else {
+ cnt += getCount(child, nodes, idx + 1);
+ }
+ }
+ return cnt;
+ }
+ }
+
+ /**
+ * Traverse the MTree to get the count of timeseries in the given level.
+ * @param targetLevel Record the distance to the target level, 0 means the target level.
+ */
+ private int getCountInGivenLevel(MNode node, int targetLevel) {
+ if (targetLevel == 0) {
+ return 1;
+ }
+ int cnt = 0;
+ if (node instanceof InternalMNode) {
+ for (MNode child : node.getChildren().values()) {
+ cnt += getCountInGivenLevel(child, targetLevel - 1);
+ }
+ }
+ return cnt;
+ }
+
+ /**
* Get all time series schema under the given path
*
* <p>result: [name, alias, storage group, dataType, encoding, compression, offset]
@@ -645,11 +734,11 @@ public class MTree implements Serializable {
/**
* Traverse the MTree to match all child node path in next level
*
- * @param node the current traversing node
- * @param nodes split the prefix path with '.'
- * @param idx the current index of array nodes
+ * @param node the current traversing node
+ * @param nodes split the prefix path with '.'
+ * @param idx the current index of array nodes
* @param parent store the node string having traversed
- * @param res store all matched device names
+ * @param res store all matched device names
* @param length expected length of path
*/
private void findChildNodePathInNextLevel(
@@ -705,11 +794,11 @@ public class MTree implements Serializable {
/**
* Traverse the MTree to match all devices with prefix path.
*
- * @param node the current traversing node
- * @param nodes split the prefix path with '.'
- * @param idx the current index of array nodes
+ * @param node the current traversing node
+ * @param nodes split the prefix path with '.'
+ * @param idx the current index of array nodes
* @param parent store the node string having traversed
- * @param res store all matched device names
+ * @param res store all matched device names
*/
private void findDevices(MNode node, String[] nodes, int idx, String parent, Set<String> res) {
String nodeReg = MetaUtils.getNodeRegByIdx(idx, nodes);
@@ -718,12 +807,8 @@ public class MTree implements Serializable {
if (node.getChild(nodeReg) instanceof LeafMNode) {
res.add(parent + node.getName());
} else {
- findDevices(
- node.getChild(nodeReg),
- nodes,
- idx + 1,
- parent + node.getName() + PATH_SEPARATOR,
- res);
+ findDevices(node.getChild(nodeReg), nodes, idx + 1,
+ parent + node.getName() + PATH_SEPARATOR, res);
}
}
} else {
@@ -739,7 +824,9 @@ public class MTree implements Serializable {
}
}
- /** Get all paths from root to the given level */
+ /**
+ * Get all paths from root to the given level.
+ */
List<String> getNodesList(String path, int nodeLevel) throws MetadataException {
String[] nodes = MetaUtils.getNodeNames(path);
if (!nodes[0].equals(root.getName())) {
@@ -758,6 +845,10 @@ public class MTree implements Serializable {
return res;
}
+ /**
+ * Get all paths under the given level.
+ * @param targetLevel Record the distance to the target level, 0 means the target level.
+ */
private void findNodes(MNode node, String path, List<String> res, int targetLevel) {
if (node == null) {
return;
@@ -804,7 +895,9 @@ public class MTree implements Serializable {
return jsonObject;
}
- /** combine multiple metadata in string format */
+ /**
+ * combine multiple metadata in string format
+ */
static String combineMetadataInStrings(String[] metadataStrs) {
JSONObject[] jsonObjects = new JSONObject[metadataStrs.length];
for (int i = 0; i < jsonObjects.length; i++) {
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
index 14f78a9..08ef238 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
@@ -51,8 +51,8 @@ import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import org.apache.iotdb.db.auth.AuthException;
-import org.apache.iotdb.db.auth.authorizer.IAuthorizer;
import org.apache.iotdb.db.auth.authorizer.BasicAuthorizer;
+import org.apache.iotdb.db.auth.authorizer.IAuthorizer;
import org.apache.iotdb.db.auth.entity.PathPrivilege;
import org.apache.iotdb.db.auth.entity.Role;
import org.apache.iotdb.db.auth.entity.User;
@@ -361,8 +361,7 @@ public class PlanExecutor implements IPlanExecutor {
}
private QueryDataSet processCountNodes(CountPlan countPlan) throws MetadataException {
- List<String> nodes = getNodesList(countPlan.getPath().toString(), countPlan.getLevel());
- int num = nodes.size();
+ int num = getNodesNumInGivenLevel(countPlan.getPath().toString(), countPlan.getLevel());
SingleDataSet singleDataSet =
new SingleDataSet(
Collections.singletonList(new Path(COLUMN_COUNT)),
@@ -376,6 +375,7 @@ public class PlanExecutor implements IPlanExecutor {
}
private QueryDataSet processCountNodeTimeSeries(CountPlan countPlan) throws MetadataException {
+ // get the nodes that need to group by first
List<String> nodes = getNodesList(countPlan.getPath().toString(), countPlan.getLevel());
ListDataSet listDataSet =
new ListDataSet(
@@ -386,7 +386,8 @@ public class PlanExecutor implements IPlanExecutor {
Field field = new Field(TSDataType.TEXT);
field.setBinaryV(new Binary(columnPath));
Field field1 = new Field(TSDataType.TEXT);
- field1.setBinaryV(new Binary(Integer.toString(getPaths(columnPath).size())));
+ // get the count of every group
+ field1.setBinaryV(new Binary(Integer.toString(getPathsNum(columnPath))));
record.addField(field);
record.addField(field1);
listDataSet.putRecord(record);
@@ -394,7 +395,15 @@ public class PlanExecutor implements IPlanExecutor {
return listDataSet;
}
- protected List<String> getPaths(String path) throws MetadataException {
+ protected int getPathsNum(String path) throws MetadataException {
+ return MManager.getInstance().getAllTimeseriesCount(path);
+ }
+
+ protected int getNodesNumInGivenLevel(String path, int level) throws MetadataException {
+ return MManager.getInstance().getNodesCountInGivenLevel(path, level);
+ }
+
+ protected List<String> getPathsName(String path) throws MetadataException {
return MManager.getInstance().getAllTimeseriesName(path);
}
@@ -403,7 +412,7 @@ public class PlanExecutor implements IPlanExecutor {
}
private QueryDataSet processCountTimeSeries(CountPlan countPlan) throws MetadataException {
- int num = getPaths(countPlan.getPath().toString()).size();
+ int num = getPathsNum(countPlan.getPath().toString());
SingleDataSet singleDataSet =
new SingleDataSet(
Collections.singletonList(new Path(COLUMN_CHILD_PATHS)),
@@ -675,7 +684,7 @@ public class PlanExecutor implements IPlanExecutor {
try {
Set<String> existingPaths = new HashSet<>();
for (Path p : deletePlan.getPaths()) {
- existingPaths.addAll(getPaths(p.getFullPath()));
+ existingPaths.addAll(getPathsName(p.getFullPath()));
}
if (existingPaths.isEmpty()) {
throw new QueryProcessException("TimeSeries does not exist and its data cannot be deleted");
diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/MTreeTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/MTreeTest.java
index 1939d80..b84b568 100644
--- a/server/src/test/java/org/apache/iotdb/db/metadata/MTreeTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/metadata/MTreeTest.java
@@ -421,4 +421,31 @@ public class MTreeTest {
fail(e.getMessage());
}
}
+
+ @Test
+ public void testGetAllTimeseriesCount() {
+ // set storage group first
+ MTree root = new MTree();
+ try {
+ root.setStorageGroup("root.laptop");
+ root.createTimeseries("root.laptop.d1.s1", TSDataType.INT32, TSEncoding.PLAIN,
+ CompressionType.GZIP, null, null);
+ root.createTimeseries("root.laptop.d1.s2", TSDataType.INT32, TSEncoding.PLAIN,
+ CompressionType.GZIP, null, null);
+ root.createTimeseries("root.laptop.d2.s1", TSDataType.INT32, TSEncoding.PLAIN,
+ CompressionType.GZIP, null, null);
+ root.createTimeseries("root.laptop.d2.s2", TSDataType.INT32, TSEncoding.PLAIN,
+ CompressionType.GZIP, null, null);
+
+ assertEquals(4, root.getAllTimeseriesCount("root.laptop"));
+
+ assertEquals(2, root.getNodesCountInGivenLevel("root.laptop", 2));
+ assertEquals(4, root.getNodesCountInGivenLevel("root.laptop", 3));
+ assertEquals(2, root.getNodesCountInGivenLevel("root.laptop.d1", 3));
+ assertEquals(0, root.getNodesCountInGivenLevel("root.laptop.d1", 4));
+ } catch (MetadataException e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
}