You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2020/02/26 09:28:44 UTC
[incubator-iotdb] branch master updated: [IOTDB-444] Enhanced
wildcard query on device path (#837)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 9fef008 [IOTDB-444] Enhanced wildcard query on device path (#837)
9fef008 is described below
commit 9fef008f6cb8df196f721f34ac6b1e8c2c8de75f
Author: Zesong Sun <sz...@mails.tsinghua.edu.cn>
AuthorDate: Wed Feb 26 17:28:36 2020 +0800
[IOTDB-444] Enhanced wildcard query on device path (#837)
* [IOTDB-444] Enhanced wildcard query on device path
---
.../5-Operation Manual/4-SQL Reference.md | 6 +-
.../5-Operation Manual/4-SQL Reference.md | 6 +-
.../org/apache/iotdb/db/qp/strategy/SqlBase.g4 | 6 +-
.../org/apache/iotdb/db/metadata/MManager.java | 2 +-
.../java/org/apache/iotdb/db/metadata/MTree.java | 64 ++++++++++++++++++----
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 3 +-
.../iotdb/db/qp/strategy/PhysicalGenerator.java | 7 +--
.../iotdb/db/integration/IoTDBAlignByDeviceIT.java | 4 +-
.../iotdb/db/metadata/MManagerAdvancedTest.java | 14 +++--
.../iotdb/db/metadata/MManagerBasicTest.java | 57 ++++++++++++++++++-
10 files changed, 134 insertions(+), 35 deletions(-)
diff --git a/docs/Documentation-CHN/UserGuide/5-Operation Manual/4-SQL Reference.md b/docs/Documentation-CHN/UserGuide/5-Operation Manual/4-SQL Reference.md
index 1ec1d2d..d9d62ce 100644
--- a/docs/Documentation-CHN/UserGuide/5-Operation Manual/4-SQL Reference.md
+++ b/docs/Documentation-CHN/UserGuide/5-Operation Manual/4-SQL Reference.md
@@ -176,8 +176,9 @@ Note: This statement can be used in IoTDB Client and JDBC.
SHOW CHILD PATHS <Path>
Eg: IoTDB > SHOW CHILD PATHS root
Eg: IoTDB > SHOW CHILD PATHS root.ln
-Eg: IoTDB > SHOW CHILD PATHS root.ln.wf01
-Note: The path can only be prefix path.
+Eg: IoTDB > SHOW CHILD PATHS root.*.wf01
+Eg: IoTDB > SHOW CHILD PATHS root.ln.wf*
+Note: The path can be prefix path or star path, the nodes can be in a "prefix + star" format.
Note: This statement can be used in IoTDB Client and JDBC.
```
### 数据管理语句
@@ -236,6 +237,7 @@ SensorExpr : (<Timeseries> | <Path>) PrecedenceEqualOperator <PointValue>
Eg: IoTDB > SELECT status, temperature FROM root.ln.wf01.wt01 WHERE temperature < 24 and time > 2017-11-1 0:13:00
Eg. IoTDB > SELECT * FROM root
Eg. IoTDB > SELECT * FROM root where time > now() - 5m
+Eg. IoTDB > SELECT * FROM root.ln.*.wf*
Eg. IoTDB > SELECT COUNT(temperature) FROM root.ln.wf01.wt01 WHERE root.ln.wf01.wt01.temperature < 25
Eg. IoTDB > SELECT MIN_TIME(temperature) FROM root.ln.wf01.wt01 WHERE root.ln.wf01.wt01.temperature < 25
Eg. IoTDB > SELECT MAX_TIME(temperature) FROM root.ln.wf01.wt01 WHERE root.ln.wf01.wt01.temperature > 24
diff --git a/docs/Documentation/UserGuide/5-Operation Manual/4-SQL Reference.md b/docs/Documentation/UserGuide/5-Operation Manual/4-SQL Reference.md
index b20d3fa..0f5d4b6 100644
--- a/docs/Documentation/UserGuide/5-Operation Manual/4-SQL Reference.md
+++ b/docs/Documentation/UserGuide/5-Operation Manual/4-SQL Reference.md
@@ -183,8 +183,9 @@ Note: This statement can be used in IoTDB Client and JDBC.
SHOW CHILD PATHS <Path>
Eg: IoTDB > SHOW CHILD PATHS root
Eg: IoTDB > SHOW CHILD PATHS root.ln
-Eg: IoTDB > SHOW CHILD PATHS root.ln.wf01
-Note: The path can only be prefix path.
+Eg: IoTDB > SHOW CHILD PATHS root.*.wf01
+Eg: IoTDB > SHOW CHILD PATHS root.ln.wf*
+Note: The path can be prefix path or star path, the nodes can be in a "prefix + star" format.
Note: This statement can be used in IoTDB Client and JDBC.
```
### Data Management Statement
@@ -243,6 +244,7 @@ SensorExpr : (<Timeseries> | <Path>) PrecedenceEqualOperator <PointValue>
Eg: IoTDB > SELECT status, temperature FROM root.ln.wf01.wt01 WHERE temperature < 24 and time > 2017-11-1 0:13:00
Eg. IoTDB > SELECT * FROM root
Eg. IoTDB > SELECT * FROM root where time > now() - 5m
+Eg. IoTDB > SELECT * FROM root.ln.*.wf*
Eg. IoTDB > SELECT COUNT(temperature) FROM root.ln.wf01.wt01 WHERE root.ln.wf01.wt01.temperature < 25
Eg. IoTDB > SELECT MIN_TIME(temperature) FROM root.ln.wf01.wt01 WHERE root.ln.wf01.wt01.temperature < 25
Eg. IoTDB > SELECT MAX_TIME(temperature) FROM root.ln.wf01.wt01 WHERE root.ln.wf01.wt01.temperature > 24
diff --git a/server/src/main/antlr4/org/apache/iotdb/db/qp/strategy/SqlBase.g4 b/server/src/main/antlr4/org/apache/iotdb/db/qp/strategy/SqlBase.g4
index 1a8558f..6a92ea8 100644
--- a/server/src/main/antlr4/org/apache/iotdb/db/qp/strategy/SqlBase.g4
+++ b/server/src/main/antlr4/org/apache/iotdb/db/qp/strategy/SqlBase.g4
@@ -31,11 +31,6 @@ statement
| DELETE FROM prefixPath (COMMA prefixPath)* (whereClause)? #deleteStatement
| SET STORAGE GROUP TO prefixPath #setStorageGroup
| DELETE STORAGE GROUP prefixPath (COMMA prefixPath)* #deleteStorageGroup
- | CREATE PROPERTY ID #createProperty
- | ADD LABEL label=ID TO PROPERTY propertyName=ID #addLabel
- | DELETE LABEL label=ID FROM PROPERTY propertyName=ID #deleteLabel
- | LINK prefixPath TO propertyLabelPair #linkPath
- | UNLINK prefixPath FROM propertyLabelPair #unlinkPath
| SHOW METADATA #showMetadata // not support yet
| DESCRIBE prefixPath #describePath // not support yet
| CREATE INDEX ON timeseriesPath USING function=ID indexWithClause? whereClause? #createIndex //not support yet
@@ -292,6 +287,7 @@ nodeName
: ID
| INT
| STAR
+ | ID STAR
| STRING_LITERAL
;
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
index 649a267..0ef29eb 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
@@ -558,7 +558,7 @@ public class MManager {
* wildcard can only match one level, otherwise it can match to the tail.
* @return A HashSet instance which stores devices names with given prefixPath.
*/
- public List<String> getDevices(String prefixPath) throws MetadataException {
+ public Set<String> getDevices(String prefixPath) throws MetadataException {
lock.readLock().lock();
try {
return mtree.getDevices(prefixPath);
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java b/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
index 9be1476..a4b015c 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
@@ -33,6 +33,8 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.TreeSet;
+import java.util.regex.Pattern;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.metadata.IllegalPathException;
@@ -503,13 +505,16 @@ public class MTree implements Serializable {
return;
}
String nodeReg = MetaUtils.getNodeRegByIdx(idx, nodes);
- if (!(PATH_WILDCARD).equals(nodeReg)) {
+ if (!nodeReg.contains(PATH_WILDCARD)) {
if (node.hasChild(nodeReg)) {
findPath(node.getChild(nodeReg), nodes, idx + 1, parent + node.getName() + PATH_SEPARATOR,
timeseriesSchemaList);
}
} else {
for (MNode child : node.getChildren().values()) {
+ if (!Pattern.matches(nodeReg.replace("*", ".*"), child.getName())) {
+ return;
+ }
findPath(child, nodes, idx + 1, parent + node.getName() + PATH_SEPARATOR,
timeseriesSchemaList);
}
@@ -525,17 +530,52 @@ public class MTree implements Serializable {
* @return All child nodes' seriesPath(s) of given seriesPath.
*/
Set<String> getChildNodePathInNextLevel(String path) throws MetadataException {
- MNode node = getNodeByPath(path);
-
- if (node instanceof LeafMNode) {
- return new HashSet<>();
+ String[] nodes = MetaUtils.getNodeNames(path);
+ if (nodes.length == 0 || !nodes[0].equals(root.getName())) {
+ throw new IllegalPathException(path);
}
+ Set<String> childNodePaths = new TreeSet<>();
+ findChildNodePathInNextLevel(root, nodes, 1, "", childNodePaths, nodes.length + 1);
+ return childNodePaths;
+ }
- Set<String> childPaths = new HashSet<>();
- for (MNode child : node.getChildren().values()) {
- childPaths.add(path + PATH_SEPARATOR + child.getName());
+ /**
+ * Traverse the MTree to match all child node path in next level
+ *
+ * @param node the current traversing node
+ * @param nodes split the prefix path with '.'
+ * @param idx the current index of array nodes
+ * @param parent store the node string having traversed
+ * @param res store all matched device names
+ * @param length expected length of path
+ */
+ private void findChildNodePathInNextLevel(MNode node, String[] nodes, int idx, String parent,
+ Set<String> res, int length) {
+ String nodeReg = MetaUtils.getNodeRegByIdx(idx, nodes);
+ if (!nodeReg.contains(PATH_WILDCARD)) {
+ if (idx == length) {
+ res.add(parent + node.getName());
+ } else {
+ findChildNodePathInNextLevel(node.getChild(nodeReg), nodes, idx + 1,
+ parent + node.getName() + PATH_SEPARATOR, res, length);
+ }
+ } else {
+ if (node instanceof InternalMNode) {
+ for (MNode child : node.getChildren().values()) {
+ if (!Pattern.matches(nodeReg.replace("*", ".*"), child.getName())) {
+ return;
+ }
+ if (idx == length) {
+ res.add(parent + node.getName());
+ } else {
+ findChildNodePathInNextLevel(child, nodes, idx + 1,
+ parent + node.getName() + PATH_SEPARATOR, res, length);
+ }
+ }
+ } else if (idx == length) {
+ res.add(parent + node.getName());
+ }
}
- return childPaths;
}
/**
@@ -543,12 +583,12 @@ public class MTree implements Serializable {
*
* @return a list contains all distinct devices names
*/
- List<String> getDevices(String prefixPath) throws MetadataException {
+ Set<String> getDevices(String prefixPath) throws MetadataException {
String[] nodes = MetaUtils.getNodeNames(prefixPath);
if (nodes.length == 0 || !nodes[0].equals(root.getName())) {
throw new IllegalPathException(prefixPath);
}
- List<String> devices = new ArrayList<>();
+ Set<String> devices = new TreeSet<>();
findDevices(root, nodes, 1, "", devices);
return devices;
}
@@ -562,7 +602,7 @@ public class MTree implements Serializable {
* @param parent store the node string having traversed
* @param res store all matched device names
*/
- private void findDevices(MNode node, String[] nodes, int idx, String parent, List<String> res) {
+ private void findDevices(MNode node, String[] nodes, int idx, String parent, Set<String> res) {
String nodeReg = MetaUtils.getNodeRegByIdx(idx, nodes);
if (!(PATH_WILDCARD).equals(nodeReg)) {
if (node.hasChild(nodeReg)) {
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
index 2cf135d..8956d4c 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
@@ -333,8 +333,7 @@ public class PlanExecutor implements IPlanExecutor {
throws MetadataException {
ListDataSet listDataSet = new ListDataSet(Collections.singletonList(new Path(COLUMN_DEVICES)),
Collections.singletonList(TSDataType.TEXT));
- List<String> devices;
- devices = MManager.getInstance().getDevices(showDevicesPlan.getPath().toString());
+ Set<String> devices = MManager.getInstance().getDevices(showDevicesPlan.getPath().toString());
for (String s : devices) {
RowRecord record = new RowRecord(0);
Field field = new Field(TSDataType.TEXT);
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java b/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
index ee44e56..ea1897a 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
@@ -281,8 +281,9 @@ public class PhysicalGenerator {
for (String device : devices) { // per device in FROM after deduplication
Path fullPath = Path.addPrefixPath(suffixPath, device);
try {
+ // remove stars in SELECT to get actual paths
List<String> actualPaths = MManager.getInstance()
- .getAllTimeseriesName(fullPath.getFullPath()); // remove stars in SELECT to get actual paths
+ .getAllTimeseriesName(fullPath.getFullPath());
// for actual non exist path
if (actualPaths.isEmpty() && originAggregations.isEmpty()) {
@@ -426,9 +427,7 @@ public class PhysicalGenerator {
Set<String> deviceSet = new LinkedHashSet<>();
try {
for (Path path : paths) {
- List<String> tempDS;
- tempDS = MManager.getInstance().getDevices(path.getFullPath());
-
+ Set<String> tempDS = MManager.getInstance().getDevices(path.getFullPath());
deviceSet.addAll(tempDS);
}
retDevices = new ArrayList<>(deviceSet);
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBAlignByDeviceIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBAlignByDeviceIT.java
index fb7c396..86aadaf 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBAlignByDeviceIT.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBAlignByDeviceIT.java
@@ -595,9 +595,9 @@ public class IoTDBAlignByDeviceIT {
@Test
public void unusualCaseTest1() throws ClassNotFoundException {
String[] retArray = new String[]{
+ "root.other.d1,1,",
"root.vehicle.d0,11,",
- "root.vehicle.d1,2,",
- "root.other.d1,1,"
+ "root.vehicle.d1,2,"
};
Class.forName(Config.JDBC_DRIVER_NAME);
diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/MManagerAdvancedTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/MManagerAdvancedTest.java
index 33962ae..3b90be8 100644
--- a/server/src/test/java/org/apache/iotdb/db/metadata/MManagerAdvancedTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/metadata/MManagerAdvancedTest.java
@@ -84,10 +84,16 @@ public class MManagerAdvancedTest {
assertEquals("root.vehicle.d0", mmanager.getStorageGroupName("root.vehicle.d0.s1"));
List<String> pathList = mmanager.getAllTimeseriesName("root.vehicle.d1.*");
assertEquals(6, pathList.size());
- List<String> paths = mmanager.getAllTimeseriesName("root.vehicle.d0");
- assertEquals(6, paths.size());
- paths = mmanager.getAllTimeseriesName("root.vehicle.d2");
- assertEquals(0, paths.size());
+ pathList = mmanager.getAllTimeseriesName("root.vehicle.d0");
+ assertEquals(6, pathList.size());
+ pathList = mmanager.getAllTimeseriesName("root.vehicle.d*");
+ assertEquals(12, pathList.size());
+ pathList = mmanager.getAllTimeseriesName("root.ve*.*");
+ assertEquals(12, pathList.size());
+ pathList = mmanager.getAllTimeseriesName("root.vehicle*.d*.s1");
+ assertEquals(2, pathList.size());
+ pathList = mmanager.getAllTimeseriesName("root.vehicle.d2");
+ assertEquals(0, pathList.size());
} catch (MetadataException e) {
e.printStackTrace();
fail(e.getMessage());
diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java
index 819e0c5..da87dfa 100644
--- a/server/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java
@@ -26,6 +26,8 @@ import static org.junit.Assert.fail;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.exception.query.PathException;
@@ -303,7 +305,7 @@ public class MManagerBasicTest {
CompressionType.GZIP, null);
manager.createTimeseries("root.laptop.d2.s1", TSDataType.INT32, TSEncoding.PLAIN,
CompressionType.GZIP, null);
- List<String> devices = new ArrayList<>();
+ Set<String> devices = new TreeSet<>();
devices.add("root.laptop.d1");
devices.add("root.laptop.d2");
// usual condition
@@ -319,4 +321,57 @@ public class MManagerBasicTest {
fail(e.getMessage());
}
}
+
+ @Test
+ public void testGetChildNodePathInNextLevel() {
+ MManager manager = MManager.getInstance();
+ String[] res = new String[]{
+ "[root.laptop, root.vehicle]",
+ "[root.laptop.b1, root.laptop.b2]",
+ "[root.laptop.b1.d1, root.laptop.b1.d2]",
+ "[root.laptop.b1, root.laptop.b2, root.vehicle.b1, root.vehicle.b2]",
+ "[root.laptop.b1.d1, root.laptop.b1.d2, root.vehicle.b1.d0, root.vehicle.b1.d2, root.vehicle.b1.d3]",
+ "[root.laptop.b1.d1, root.laptop.b1.d2]",
+ "[root.laptop.b1.d1, root.laptop.b1.d2, root.laptop.b2.d1, root.laptop.b2.d2]",
+ "[root.laptop.b1.d1.s0, root.laptop.b1.d1.s1, root.laptop.b1.d2.s0, root.laptop.b2.d1.s1, root.laptop.b2.d1.s3, root.laptop.b2.d2.s2]"
+ };
+
+ try {
+ manager.setStorageGroup("root.laptop");
+ manager.setStorageGroup("root.vehicle");
+
+ manager.createTimeseries("root.laptop.b1.d1.s0", TSDataType.INT32, TSEncoding.PLAIN,
+ CompressionType.GZIP, null);
+ manager.createTimeseries("root.laptop.b1.d1.s1", TSDataType.INT32, TSEncoding.PLAIN,
+ CompressionType.GZIP, null);
+ manager.createTimeseries("root.laptop.b1.d2.s0", TSDataType.INT32, TSEncoding.PLAIN,
+ CompressionType.GZIP, null);
+ manager.createTimeseries("root.laptop.b2.d1.s1", TSDataType.INT32, TSEncoding.PLAIN,
+ CompressionType.GZIP, null);
+ manager.createTimeseries("root.laptop.b2.d1.s3", TSDataType.INT32, TSEncoding.PLAIN,
+ CompressionType.GZIP, null);
+ manager.createTimeseries("root.laptop.b2.d2.s2", TSDataType.INT32, TSEncoding.PLAIN,
+ CompressionType.GZIP, null);
+ manager.createTimeseries("root.vehicle.b1.d0.s0", TSDataType.INT32, TSEncoding.PLAIN,
+ CompressionType.GZIP, null);
+ manager.createTimeseries("root.vehicle.b2.d0.s1", TSDataType.INT32, TSEncoding.PLAIN,
+ CompressionType.GZIP, null);
+ manager.createTimeseries("root.vehicle.b1.d2.s2", TSDataType.INT32, TSEncoding.PLAIN,
+ CompressionType.GZIP, null);
+ manager.createTimeseries("root.vehicle.b1.d3.s3", TSDataType.INT32, TSEncoding.PLAIN,
+ CompressionType.GZIP, null);
+
+ assertEquals(res[0], manager.getChildNodePathInNextLevel("root").toString());
+ assertEquals(res[1], manager.getChildNodePathInNextLevel("root.laptop").toString());
+ assertEquals(res[2], manager.getChildNodePathInNextLevel("root.laptop.b1").toString());
+ assertEquals(res[3], manager.getChildNodePathInNextLevel("root.*").toString());
+ assertEquals(res[4], manager.getChildNodePathInNextLevel("root.*.b1").toString());
+ assertEquals(res[5], manager.getChildNodePathInNextLevel("root.l*.b1").toString());
+ assertEquals(res[6], manager.getChildNodePathInNextLevel("root.l*.*").toString());
+ assertEquals(res[7], manager.getChildNodePathInNextLevel("root.l*.b*.*").toString());
+ } catch (MetadataException e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
}