You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2020/04/23 02:57:22 UTC
[incubator-iotdb] branch pushdown_limit created (now 6e84993)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a change to branch pushdown_limit
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.
at 6e84993 push limit from MManager to MTree
This branch includes the following new commits:
new 6e84993 push limit from MManager to MTree
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-iotdb] 01/01: push limit from MManager to MTree
Posted by qi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch pushdown_limit
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 6e8499355e2771186817bd366d196962793eee4e
Author: qiaojialin <64...@qq.com>
AuthorDate: Thu Apr 23 10:57:05 2020 +0800
push limit from MManager to MTree
---
.../org/apache/iotdb/db/metadata/MManager.java | 31 ++++----------
.../java/org/apache/iotdb/db/metadata/MTree.java | 48 ++++++++++++++++++----
.../db/qp/physical/sys/ShowTimeSeriesPlan.java | 15 ++++---
.../iotdb/db/qp/strategy/PhysicalGenerator.java | 2 +-
4 files changed, 59 insertions(+), 37 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
index a18f3d5..356992a 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
@@ -704,9 +704,11 @@ public class MManager {
int offset = plan.getOffset();
for (LeafMNode leaf : allMatchedNodes) {
if (match(leaf.getFullPath(), prefixNodes)) {
- curOffset ++;
- if (curOffset < offset) {
- continue;
+ if (limit != 0 || offset != 0) {
+ curOffset ++;
+ if (curOffset < offset || count == limit) {
+ continue;
+ }
}
try {
Pair<Map<String, String>, Map<String, String>> pair =
@@ -717,9 +719,8 @@ public class MManager {
getStorageGroupName(leaf.getFullPath()), measurementSchema.getType().toString(),
measurementSchema.getEncodingType().toString(),
measurementSchema.getCompressor().toString(), pair.left));
- count ++;
- if (count == limit) {
- return res;
+ if (limit != 0 || offset != 0) {
+ count++;
}
} catch (IOException e) {
throw new MetadataException(
@@ -757,17 +758,9 @@ public class MManager {
public List<ShowTimeSeriesResult> showTimeseries(ShowTimeSeriesPlan plan) throws MetadataException {
lock.readLock().lock();
try {
- List<String[]> ans = mtree.getAllMeasurementSchema(plan.getPath().getFullPath());
- int count = 0;
- int offset = plan.getOffset();
+ List<String[]> ans = mtree.getAllMeasurementSchema(plan);
List<ShowTimeSeriesResult> res = new LinkedList<>();
- for (int i = 0; i < ans.size(); i++) {
- if (i < offset) {
- continue;
- }
-
- String[] ansString = ans.get(i);
-
+ for (String[] ansString : ans) {
long tagFileOffset = Long.parseLong(ansString[6]);
try {
if (tagFileOffset < 0) {
@@ -782,12 +775,6 @@ public class MManager {
res.add(new ShowTimeSeriesResult(ansString[0], ansString[1], ansString[2],
ansString[3], ansString[4], ansString[5], pair.left));
}
-
- count ++;
- if (count == plan.getLimit()) {
- return res;
- }
-
} catch (IOException e) {
throw new MetadataException(
"Something went wrong while deserialize tag info of " + ansString[0], e);
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java b/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
index 3949aee..80e5d99 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
@@ -28,10 +28,12 @@ import org.apache.iotdb.db.metadata.mnode.InternalMNode;
import org.apache.iotdb.db.metadata.mnode.LeafMNode;
import org.apache.iotdb.db.metadata.mnode.MNode;
import org.apache.iotdb.db.metadata.mnode.StorageGroupMNode;
+import org.apache.iotdb.db.qp.physical.sys.ShowTimeSeriesPlan;
import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
@@ -51,6 +53,11 @@ public class MTree implements Serializable {
private static final long serialVersionUID = -4200394435237291964L;
private MNode root;
+ private int limit = 0;
+ private int offset = 0;
+ private int count = 0;
+ private int curOffset = -1;
+
MTree() {
this.root = new InternalMNode(null, IoTDBConstant.PATH_ROOT);
}
@@ -466,7 +473,8 @@ public class MTree implements Serializable {
* @param prefixPath a prefix path or a full path, may contain '*'.
*/
List<String> getAllTimeseriesName(String prefixPath) throws MetadataException {
- List<String[]> res = getAllMeasurementSchema(prefixPath);
+ ShowTimeSeriesPlan plan = new ShowTimeSeriesPlan(new Path(prefixPath));
+ List<String[]> res = getAllMeasurementSchema(plan);
List<String> paths = new ArrayList<>();
for (String[] p : res) {
paths.add(p[0]);
@@ -477,27 +485,45 @@ public class MTree implements Serializable {
/**
* Get all time series schema under the given path
*
- * MeasurementSchema: [name, alias, storage group, dataType, encoding, compression, offset]
+ * result: [name, alias, storage group, dataType, encoding, compression, offset]
*/
- List<String[]> getAllMeasurementSchema(String prefixPath) throws MetadataException {
+ List<String[]> getAllMeasurementSchema(ShowTimeSeriesPlan plan) throws MetadataException {
List<String[]> res = new ArrayList<>();
- String[] nodes = MetaUtils.getNodeNames(prefixPath);
+ String[] nodes = MetaUtils.getNodeNames(plan.getPath().getFullPath());
if (nodes.length == 0 || !nodes[0].equals(root.getName())) {
- throw new IllegalPathException(prefixPath);
+ throw new IllegalPathException(plan.getPath().getFullPath());
+ }
+ this.limit = plan.getLimit();
+ this.offset = plan.getOffset();
+ if (offset != 0 || limit != 0) {
+ findPath(root, nodes, 1, "", res, true);
+ resetCountCurOffset();
+ } else {
+ findPath(root, nodes, 1, "", res, false);
}
- findPath(root, nodes, 1, "", res);
return res;
}
+ private void resetCountCurOffset() {
+ curOffset = -1;
+ count = 0;
+ }
+
/**
* Iterate through MTree to fetch metadata info of all leaf nodes under the given seriesPath
*
* @param timeseriesSchemaList List<timeseriesSchema>
*/
private void findPath(MNode node, String[] nodes, int idx, String parent,
- List<String[]> timeseriesSchemaList) throws MetadataException {
+ List<String[]> timeseriesSchemaList, boolean hasLimit) throws MetadataException {
if (node instanceof LeafMNode) {
if (nodes.length <= idx) {
+ if (hasLimit) {
+ curOffset++;
+ if (curOffset < offset || count == limit) {
+ return;
+ }
+ }
String nodeName;
if (node.getName().contains(TsFileConstant.PATH_SEPARATOR)) {
nodeName = "\"" + node + "\"";
@@ -515,6 +541,10 @@ public class MTree implements Serializable {
tsRow[5] = measurementSchema.getCompressor().toString();
tsRow[6] = String.valueOf(((LeafMNode) node).getOffset());
timeseriesSchemaList.add(tsRow);
+
+ if (hasLimit) {
+ count++;
+ }
}
return;
}
@@ -522,7 +552,7 @@ public class MTree implements Serializable {
if (!nodeReg.contains(PATH_WILDCARD)) {
if (node.hasChild(nodeReg)) {
findPath(node.getChild(nodeReg), nodes, idx + 1, parent + node.getName() + PATH_SEPARATOR,
- timeseriesSchemaList);
+ timeseriesSchemaList, hasLimit);
}
} else {
for (MNode child : node.getChildren().values()) {
@@ -530,7 +560,7 @@ public class MTree implements Serializable {
continue;
}
findPath(child, nodes, idx + 1, parent + node.getName() + PATH_SEPARATOR,
- timeseriesSchemaList);
+ timeseriesSchemaList, hasLimit);
}
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/ShowTimeSeriesPlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/ShowTimeSeriesPlan.java
index 48a6491..fbf8fcb 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/ShowTimeSeriesPlan.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/ShowTimeSeriesPlan.java
@@ -28,12 +28,17 @@ public class ShowTimeSeriesPlan extends ShowPlan {
private boolean isContains;
private String key;
private String value;
- private int limit;
- private int offset;
+ private int limit = 0;
+ private int offset = 0;
- public ShowTimeSeriesPlan(ShowContentType showContentType, Path path, boolean isContains,
- String key, String value, int limit, int offset) {
- super(showContentType);
+ public ShowTimeSeriesPlan(Path path) {
+ super(ShowContentType.TIMESERIES);
+ this.path = path;
+ }
+
+ public ShowTimeSeriesPlan(Path path, boolean isContains, String key, String value, int limit,
+ int offset) {
+ super(ShowContentType.TIMESERIES);
this.path = path;
this.isContains = isContains;
this.key = key;
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java b/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
index 5cfc547..ff80539 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
@@ -126,7 +126,7 @@ public class PhysicalGenerator {
return new ShowPlan(ShowContentType.VERSION);
case SQLConstant.TOK_TIMESERIES:
ShowTimeSeriesOperator showTimeSeriesOperator = (ShowTimeSeriesOperator) operator;
- return new ShowTimeSeriesPlan(ShowContentType.TIMESERIES,
+ return new ShowTimeSeriesPlan(
showTimeSeriesOperator.getPath(), showTimeSeriesOperator.isContains(),
showTimeSeriesOperator.getKey(), showTimeSeriesOperator.getValue(),
showTimeSeriesOperator.getLimit(), showTimeSeriesOperator.getOffset());