You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by su...@apache.org on 2020/02/25 11:32:49 UTC
[incubator-iotdb] branch jira_444 created (now 447a648)
This is an automated email from the ASF dual-hosted git repository.
sunzesong pushed a change to branch jira_444
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.
at 447a648 [IOTDB-444] Enhanced wildcard query on device path
This branch includes the following new commits:
new 447a648 [IOTDB-444] Enhanced wildcard query on device path
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-iotdb] 01/01: [IOTDB-444] Enhanced wildcard query on
device path
Posted by su...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sunzesong pushed a commit to branch jira_444
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 447a648093e16888a68908857d743b6f2a4e650d
Author: samperson1997 <sz...@mails.tsinghua.edu.cn>
AuthorDate: Tue Feb 25 19:32:22 2020 +0800
[IOTDB-444] Enhanced wildcard query on device path
---
.../org/apache/iotdb/db/qp/strategy/SqlBase.g4 | 6 +-
.../org/apache/iotdb/db/metadata/MManager.java | 2 +-
.../java/org/apache/iotdb/db/metadata/MTree.java | 71 ++++++++++++++++++----
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 3 +-
.../iotdb/db/qp/strategy/PhysicalGenerator.java | 7 +--
.../iotdb/db/integration/IoTDBAlignByDeviceIT.java | 4 +-
.../iotdb/db/metadata/MManagerAdvancedTest.java | 14 +++--
.../iotdb/db/metadata/MManagerBasicTest.java | 57 ++++++++++++++++-
8 files changed, 133 insertions(+), 31 deletions(-)
diff --git a/server/src/main/antlr4/org/apache/iotdb/db/qp/strategy/SqlBase.g4 b/server/src/main/antlr4/org/apache/iotdb/db/qp/strategy/SqlBase.g4
index 1a8558f..6a92ea8 100644
--- a/server/src/main/antlr4/org/apache/iotdb/db/qp/strategy/SqlBase.g4
+++ b/server/src/main/antlr4/org/apache/iotdb/db/qp/strategy/SqlBase.g4
@@ -31,11 +31,6 @@ statement
| DELETE FROM prefixPath (COMMA prefixPath)* (whereClause)? #deleteStatement
| SET STORAGE GROUP TO prefixPath #setStorageGroup
| DELETE STORAGE GROUP prefixPath (COMMA prefixPath)* #deleteStorageGroup
- | CREATE PROPERTY ID #createProperty
- | ADD LABEL label=ID TO PROPERTY propertyName=ID #addLabel
- | DELETE LABEL label=ID FROM PROPERTY propertyName=ID #deleteLabel
- | LINK prefixPath TO propertyLabelPair #linkPath
- | UNLINK prefixPath FROM propertyLabelPair #unlinkPath
| SHOW METADATA #showMetadata // not support yet
| DESCRIBE prefixPath #describePath // not support yet
| CREATE INDEX ON timeseriesPath USING function=ID indexWithClause? whereClause? #createIndex //not support yet
@@ -292,6 +287,7 @@ nodeName
: ID
| INT
| STAR
+ | ID STAR
| STRING_LITERAL
;
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
index 649a267..0ef29eb 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
@@ -558,7 +558,7 @@ public class MManager {
* wildcard can only match one level, otherwise it can match to the tail.
* @return A HashSet instance which stores devices names with given prefixPath.
*/
- public List<String> getDevices(String prefixPath) throws MetadataException {
+ public Set<String> getDevices(String prefixPath) throws MetadataException {
lock.readLock().lock();
try {
return mtree.getDevices(prefixPath);
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java b/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
index 9be1476..a56e52c 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
@@ -33,6 +33,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.TreeSet;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.metadata.IllegalPathException;
@@ -503,13 +504,20 @@ public class MTree implements Serializable {
return;
}
String nodeReg = MetaUtils.getNodeRegByIdx(idx, nodes);
- if (!(PATH_WILDCARD).equals(nodeReg)) {
+ if (!nodeReg.contains(PATH_WILDCARD)) {
if (node.hasChild(nodeReg)) {
findPath(node.getChild(nodeReg), nodes, idx + 1, parent + node.getName() + PATH_SEPARATOR,
timeseriesSchemaList);
}
} else {
for (MNode child : node.getChildren().values()) {
+ String prefix = "";
+ if (!nodeReg.equals(PATH_WILDCARD)) {
+ prefix = nodeReg.split("\\*")[0];
+ }
+ if (!child.getName().startsWith(prefix)) {
+ return;
+ }
findPath(child, nodes, idx + 1, parent + node.getName() + PATH_SEPARATOR,
timeseriesSchemaList);
}
@@ -525,17 +533,56 @@ public class MTree implements Serializable {
* @return All child nodes' seriesPath(s) of given seriesPath.
*/
Set<String> getChildNodePathInNextLevel(String path) throws MetadataException {
- MNode node = getNodeByPath(path);
-
- if (node instanceof LeafMNode) {
- return new HashSet<>();
+ String[] nodes = MetaUtils.getNodeNames(path);
+ if (nodes.length == 0 || !nodes[0].equals(root.getName())) {
+ throw new IllegalPathException(path);
}
+ Set<String> childNodePaths = new TreeSet<>();
+ findChildNodePathInNextLevel(root, nodes, 1, "", childNodePaths, nodes.length + 1);
+ return childNodePaths;
+ }
- Set<String> childPaths = new HashSet<>();
- for (MNode child : node.getChildren().values()) {
- childPaths.add(path + PATH_SEPARATOR + child.getName());
+ /**
+ * Traverse the MTree to match all child node path in next level
+ *
+ * @param node the current traversing node
+ * @param nodes split the prefix path with '.'
+ * @param idx the current index of array nodes
+ * @param parent store the node string having traversed
+ * @param res store all matched device names
+ * @param length expected length of path
+ */
+ private void findChildNodePathInNextLevel(MNode node, String[] nodes, int idx, String parent,
+ Set<String> res, int length) {
+ String nodeReg = MetaUtils.getNodeRegByIdx(idx, nodes);
+ if (!nodeReg.contains(PATH_WILDCARD)) {
+ if (idx == length) {
+ res.add(parent + node.getName());
+ } else {
+ findChildNodePathInNextLevel(node.getChild(nodeReg), nodes, idx + 1,
+ parent + node.getName() + PATH_SEPARATOR, res, length);
+ }
+ } else {
+ if (node instanceof InternalMNode) {
+ for (MNode child : node.getChildren().values()) {
+ String prefix = "";
+ if (!nodeReg.equals(PATH_WILDCARD)) {
+ prefix = nodeReg.split("\\*")[0];
+ }
+ if (!child.getName().startsWith(prefix)) {
+ return;
+ }
+ if (idx == length) {
+ res.add(parent + node.getName());
+ } else {
+ findChildNodePathInNextLevel(child, nodes, idx + 1,
+ parent + node.getName() + PATH_SEPARATOR, res, length);
+ }
+ }
+ } else if (idx == length) {
+ res.add(parent + node.getName());
+ }
}
- return childPaths;
}
/**
@@ -543,12 +590,12 @@ public class MTree implements Serializable {
*
* @return a list contains all distinct devices names
*/
- List<String> getDevices(String prefixPath) throws MetadataException {
+ Set<String> getDevices(String prefixPath) throws MetadataException {
String[] nodes = MetaUtils.getNodeNames(prefixPath);
if (nodes.length == 0 || !nodes[0].equals(root.getName())) {
throw new IllegalPathException(prefixPath);
}
- List<String> devices = new ArrayList<>();
+ Set<String> devices = new TreeSet<>();
findDevices(root, nodes, 1, "", devices);
return devices;
}
@@ -562,7 +609,7 @@ public class MTree implements Serializable {
* @param parent store the node string having traversed
* @param res store all matched device names
*/
- private void findDevices(MNode node, String[] nodes, int idx, String parent, List<String> res) {
+ private void findDevices(MNode node, String[] nodes, int idx, String parent, Set<String> res) {
String nodeReg = MetaUtils.getNodeRegByIdx(idx, nodes);
if (!(PATH_WILDCARD).equals(nodeReg)) {
if (node.hasChild(nodeReg)) {
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
index 2cf135d..8956d4c 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
@@ -333,8 +333,7 @@ public class PlanExecutor implements IPlanExecutor {
throws MetadataException {
ListDataSet listDataSet = new ListDataSet(Collections.singletonList(new Path(COLUMN_DEVICES)),
Collections.singletonList(TSDataType.TEXT));
- List<String> devices;
- devices = MManager.getInstance().getDevices(showDevicesPlan.getPath().toString());
+ Set<String> devices = MManager.getInstance().getDevices(showDevicesPlan.getPath().toString());
for (String s : devices) {
RowRecord record = new RowRecord(0);
Field field = new Field(TSDataType.TEXT);
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java b/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
index ee44e56..ea1897a 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
@@ -281,8 +281,9 @@ public class PhysicalGenerator {
for (String device : devices) { // per device in FROM after deduplication
Path fullPath = Path.addPrefixPath(suffixPath, device);
try {
+ // remove stars in SELECT to get actual paths
List<String> actualPaths = MManager.getInstance()
- .getAllTimeseriesName(fullPath.getFullPath()); // remove stars in SELECT to get actual paths
+ .getAllTimeseriesName(fullPath.getFullPath());
// for actual non exist path
if (actualPaths.isEmpty() && originAggregations.isEmpty()) {
@@ -426,9 +427,7 @@ public class PhysicalGenerator {
Set<String> deviceSet = new LinkedHashSet<>();
try {
for (Path path : paths) {
- List<String> tempDS;
- tempDS = MManager.getInstance().getDevices(path.getFullPath());
-
+ Set<String> tempDS = MManager.getInstance().getDevices(path.getFullPath());
deviceSet.addAll(tempDS);
}
retDevices = new ArrayList<>(deviceSet);
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBAlignByDeviceIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBAlignByDeviceIT.java
index fb7c396..86aadaf 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBAlignByDeviceIT.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBAlignByDeviceIT.java
@@ -595,9 +595,9 @@ public class IoTDBAlignByDeviceIT {
@Test
public void unusualCaseTest1() throws ClassNotFoundException {
String[] retArray = new String[]{
+ "root.other.d1,1,",
"root.vehicle.d0,11,",
- "root.vehicle.d1,2,",
- "root.other.d1,1,"
+ "root.vehicle.d1,2,"
};
Class.forName(Config.JDBC_DRIVER_NAME);
diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/MManagerAdvancedTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/MManagerAdvancedTest.java
index 33962ae..3b90be8 100644
--- a/server/src/test/java/org/apache/iotdb/db/metadata/MManagerAdvancedTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/metadata/MManagerAdvancedTest.java
@@ -84,10 +84,16 @@ public class MManagerAdvancedTest {
assertEquals("root.vehicle.d0", mmanager.getStorageGroupName("root.vehicle.d0.s1"));
List<String> pathList = mmanager.getAllTimeseriesName("root.vehicle.d1.*");
assertEquals(6, pathList.size());
- List<String> paths = mmanager.getAllTimeseriesName("root.vehicle.d0");
- assertEquals(6, paths.size());
- paths = mmanager.getAllTimeseriesName("root.vehicle.d2");
- assertEquals(0, paths.size());
+ pathList = mmanager.getAllTimeseriesName("root.vehicle.d0");
+ assertEquals(6, pathList.size());
+ pathList = mmanager.getAllTimeseriesName("root.vehicle.d*");
+ assertEquals(12, pathList.size());
+ pathList = mmanager.getAllTimeseriesName("root.ve*.*");
+ assertEquals(12, pathList.size());
+ pathList = mmanager.getAllTimeseriesName("root.vehicle*.d*.s1");
+ assertEquals(2, pathList.size());
+ pathList = mmanager.getAllTimeseriesName("root.vehicle.d2");
+ assertEquals(0, pathList.size());
} catch (MetadataException e) {
e.printStackTrace();
fail(e.getMessage());
diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java
index 819e0c5..da87dfa 100644
--- a/server/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java
@@ -26,6 +26,8 @@ import static org.junit.Assert.fail;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.exception.query.PathException;
@@ -303,7 +305,7 @@ public class MManagerBasicTest {
CompressionType.GZIP, null);
manager.createTimeseries("root.laptop.d2.s1", TSDataType.INT32, TSEncoding.PLAIN,
CompressionType.GZIP, null);
- List<String> devices = new ArrayList<>();
+ Set<String> devices = new TreeSet<>();
devices.add("root.laptop.d1");
devices.add("root.laptop.d2");
// usual condition
@@ -319,4 +321,57 @@ public class MManagerBasicTest {
fail(e.getMessage());
}
}
+
+ @Test
+ public void testGetChildNodePathInNextLevel() {
+ MManager manager = MManager.getInstance();
+ String[] res = new String[]{
+ "[root.laptop, root.vehicle]",
+ "[root.laptop.b1, root.laptop.b2]",
+ "[root.laptop.b1.d1, root.laptop.b1.d2]",
+ "[root.laptop.b1, root.laptop.b2, root.vehicle.b1, root.vehicle.b2]",
+ "[root.laptop.b1.d1, root.laptop.b1.d2, root.vehicle.b1.d0, root.vehicle.b1.d2, root.vehicle.b1.d3]",
+ "[root.laptop.b1.d1, root.laptop.b1.d2]",
+ "[root.laptop.b1.d1, root.laptop.b1.d2, root.laptop.b2.d1, root.laptop.b2.d2]",
+ "[root.laptop.b1.d1.s0, root.laptop.b1.d1.s1, root.laptop.b1.d2.s0, root.laptop.b2.d1.s1, root.laptop.b2.d1.s3, root.laptop.b2.d2.s2]"
+ };
+
+ try {
+ manager.setStorageGroup("root.laptop");
+ manager.setStorageGroup("root.vehicle");
+
+ manager.createTimeseries("root.laptop.b1.d1.s0", TSDataType.INT32, TSEncoding.PLAIN,
+ CompressionType.GZIP, null);
+ manager.createTimeseries("root.laptop.b1.d1.s1", TSDataType.INT32, TSEncoding.PLAIN,
+ CompressionType.GZIP, null);
+ manager.createTimeseries("root.laptop.b1.d2.s0", TSDataType.INT32, TSEncoding.PLAIN,
+ CompressionType.GZIP, null);
+ manager.createTimeseries("root.laptop.b2.d1.s1", TSDataType.INT32, TSEncoding.PLAIN,
+ CompressionType.GZIP, null);
+ manager.createTimeseries("root.laptop.b2.d1.s3", TSDataType.INT32, TSEncoding.PLAIN,
+ CompressionType.GZIP, null);
+ manager.createTimeseries("root.laptop.b2.d2.s2", TSDataType.INT32, TSEncoding.PLAIN,
+ CompressionType.GZIP, null);
+ manager.createTimeseries("root.vehicle.b1.d0.s0", TSDataType.INT32, TSEncoding.PLAIN,
+ CompressionType.GZIP, null);
+ manager.createTimeseries("root.vehicle.b2.d0.s1", TSDataType.INT32, TSEncoding.PLAIN,
+ CompressionType.GZIP, null);
+ manager.createTimeseries("root.vehicle.b1.d2.s2", TSDataType.INT32, TSEncoding.PLAIN,
+ CompressionType.GZIP, null);
+ manager.createTimeseries("root.vehicle.b1.d3.s3", TSDataType.INT32, TSEncoding.PLAIN,
+ CompressionType.GZIP, null);
+
+ assertEquals(res[0], manager.getChildNodePathInNextLevel("root").toString());
+ assertEquals(res[1], manager.getChildNodePathInNextLevel("root.laptop").toString());
+ assertEquals(res[2], manager.getChildNodePathInNextLevel("root.laptop.b1").toString());
+ assertEquals(res[3], manager.getChildNodePathInNextLevel("root.*").toString());
+ assertEquals(res[4], manager.getChildNodePathInNextLevel("root.*.b1").toString());
+ assertEquals(res[5], manager.getChildNodePathInNextLevel("root.l*.b1").toString());
+ assertEquals(res[6], manager.getChildNodePathInNextLevel("root.l*.*").toString());
+ assertEquals(res[7], manager.getChildNodePathInNextLevel("root.l*.b*.*").toString());
+ } catch (MetadataException e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
}