You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2019/09/16 06:16:43 UTC
[incubator-iotdb] branch dev_TTL updated: add sql interface for ttl
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch dev_TTL
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/dev_TTL by this push:
new 73603eb add sql interface for ttl
73603eb is described below
commit 73603ebec09939cf5c61ca648d381fecfb9311d4
Author: jt <jt...@163.com>
AuthorDate: Mon Sep 16 14:05:40 2019 +0800
add sql interface for ttl
---
.../org/apache/iotdb/db/engine/StorageEngine.java | 4 ++
.../java/org/apache/iotdb/db/metadata/MGraph.java | 4 +-
.../org/apache/iotdb/db/metadata/MManager.java | 20 +++++++
.../java/org/apache/iotdb/db/metadata/MTree.java | 27 +++------
.../iotdb/db/metadata/MetadataOperationType.java | 1 +
.../org/apache/iotdb/db/qp/QueryProcessor.java | 1 +
.../iotdb/db/qp/executor/QueryProcessExecutor.java | 15 ++++-
.../org/apache/iotdb/db/qp/logical/Operator.java | 3 +-
.../logical/sys/TTLOperator.java} | 44 +++++++++------
.../apache/iotdb/db/qp/physical/sys/TTLPlan.java | 65 ++++++++++++++++++++++
.../iotdb/db/qp/strategy/LogicalGenerator.java | 38 ++++++++++++-
.../iotdb/db/qp/strategy/PhysicalGenerator.java | 10 ++++
12 files changed, 190 insertions(+), 42 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
index 3bbc84f..76c6072 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
@@ -334,4 +334,8 @@ public class StorageEngine implements IService {
return true;
}
+ public void setTTL(String storageGroup, long dataTTL) throws StorageEngineException {
+ StorageGroupProcessor storageGroupProcessor = getProcessor(storageGroup);
+ storageGroupProcessor.setDataTTL(dataTTL);
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MGraph.java b/server/src/main/java/org/apache/iotdb/db/metadata/MGraph.java
index 6310628..63e581f 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MGraph.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MGraph.java
@@ -331,7 +331,7 @@ public class MGraph implements Serializable {
}
MNode getNodeByPath(String path) throws PathErrorException {
- return mtree.getNodeByPath(path);
+ return mtree.getNode(path);
}
MNode getNodeByPathWithCheck(String path) throws PathErrorException {
@@ -390,7 +390,7 @@ public class MGraph implements Serializable {
Map<String, Integer> res = new HashMap<>();
List<String> storageGroups = this.getAllStorageGroup();
for (String sg : storageGroups) {
- MNode node = mtree.getNodeByPath(sg);
+ MNode node = mtree.getNode(sg);
res.put(sg, node.getLeafCount());
}
return res;
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
index e0b4d47..c70286e 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
@@ -28,6 +28,7 @@ import org.apache.iotdb.db.exception.ConfigAdjusterException;
import org.apache.iotdb.db.exception.MetadataErrorException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.monitor.MonitorConstants;
+import org.apache.iotdb.db.qp.logical.Operator.OperatorType;
import org.apache.iotdb.db.utils.RandomDeleteCache;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.exception.cache.CacheException;
@@ -225,6 +226,9 @@ public class MManager {
case MetadataOperationType.UNLINK_MNODE_FROM_PTREE:
unlinkMNodeFromPTree(args[1], args[2]);
break;
+ case MetadataOperationType.SET_TTL:
+ setTTL(args[1], Long.parseLong(args[2]));
+ break;
default:
logger.error("Unrecognizable command {}", cmd);
}
@@ -1233,4 +1237,20 @@ public class MManager {
return dataType;
}
}
+
+ public void setTTL(String storageGroup, long dataTTL) throws PathErrorException, IOException {
+ lock.writeLock().lock();
+ try {
+ MNode sgNode = getNodeByPath(storageGroup);
+ sgNode.setDataTTL(dataTTL);
+ if (writeToLog) {
+ BufferedWriter writer = getLogWriter();
+ writer.write(String.format("%s,%s,%s", OperatorType.TTL, storageGroup, dataTTL));
+ writer.newLine();
+ writer.flush();
+ }
+ } finally {
+ lock.writeLock().unlock();
+ }
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java b/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
index 69acdc6..c133b04 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
@@ -361,7 +361,7 @@ public class MTree implements Serializable {
}
private MNode getLeafByPath(String path) throws PathErrorException {
- checkPath(path);
+ getNode(path);
String[] node = path.split(DOUB_SEPARATOR);
MNode cur = getRoot();
for (int i = 1; i < node.length; i++) {
@@ -427,19 +427,6 @@ public class MTree implements Serializable {
}
/**
- * function for getting node by path.
- */
- MNode getNodeByPath(String path) throws PathErrorException {
- checkPath(path);
- String[] node = path.split(DOUB_SEPARATOR);
- MNode cur = getRoot();
- for (int i = 1; i < node.length; i++) {
- cur = cur.getChild(node[i]);
- }
- return cur;
- }
-
- /**
* function for getting node by path with file level check.
*/
MNode getNodeByPathWithFileLevelCheck(String path) throws PathErrorException {
@@ -472,7 +459,7 @@ public class MTree implements Serializable {
* @return String represents the deviceId
*/
String getDeviceTypeByPath(String path) throws PathErrorException {
- checkPath(path);
+ getNode(path);
String[] nodes = path.split(DOUB_SEPARATOR);
if (nodes.length < 2) {
throw new PathErrorException(
@@ -482,11 +469,11 @@ public class MTree implements Serializable {
}
/**
- * Check whether a seriesPath is available.
+ * find and return a seriesPath specified by the path
*
- * @return last node in given seriesPath if current seriesPath is available
+ * @return last node in given seriesPath
*/
- private MNode checkPath(String path) throws PathErrorException {
+ MNode getNode(String path) throws PathErrorException {
String[] nodes = path.split(DOUB_SEPARATOR);
if (nodes.length < 2 || !nodes[0].equals(getRoot().getName())) {
throw new PathErrorException(String.format(SERIES_NOT_CORRECT, path));
@@ -691,7 +678,7 @@ public class MTree implements Serializable {
*/
List<String> getLeafNodePathInNextLevel(String path) throws PathErrorException {
List<String> ret = new ArrayList<>();
- MNode cur = checkPath(path);
+ MNode cur = getNode(path);
for (MNode child : cur.getChildren().values()) {
if (child.isLeaf()) {
ret.add(path + "." + child.getName());
@@ -817,7 +804,7 @@ public class MTree implements Serializable {
*/
ArrayList<String> getDeviceForOneType(String type) throws PathErrorException {
String path = getRoot().getName() + "." + type;
- checkPath(path);
+ getNode(path);
HashMap<String, Integer> deviceMap = new HashMap<>();
MNode typeNode = getRoot().getChild(type);
putDeviceToMap(getRoot().getName(), typeNode, deviceMap);
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MetadataOperationType.java b/server/src/main/java/org/apache/iotdb/db/metadata/MetadataOperationType.java
index 2e52eff..7c876e5 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MetadataOperationType.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MetadataOperationType.java
@@ -34,4 +34,5 @@ public class MetadataOperationType {
public static final String UNLINK_MNODE_FROM_PTREE = "7";
public static final String ADD_INDEX_TO_PATH = "8";
public static final String DELETE_INDEX_FROM_PATH = "9";
+ public static final String SET_TTL = "10";
}
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/QueryProcessor.java b/server/src/main/java/org/apache/iotdb/db/qp/QueryProcessor.java
index ebb30c2..a3397bb 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/QueryProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/QueryProcessor.java
@@ -140,6 +140,7 @@ public class QueryProcessor {
case INDEXQUERY:
case GRANT_WATERMARK_EMBEDDING:
case REVOKE_WATERMARK_EMBEDDING:
+ case TTL:
return operator;
case QUERY:
case UPDATE:
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/executor/QueryProcessExecutor.java b/server/src/main/java/org/apache/iotdb/db/qp/executor/QueryProcessExecutor.java
index a7b5128..52ed70a 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/executor/QueryProcessExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/executor/QueryProcessExecutor.java
@@ -54,6 +54,7 @@ import org.apache.iotdb.db.qp.physical.sys.AuthorPlan;
import org.apache.iotdb.db.qp.physical.sys.DataAuthPlan;
import org.apache.iotdb.db.qp.physical.sys.MetadataPlan;
import org.apache.iotdb.db.qp.physical.sys.PropertyPlan;
+import org.apache.iotdb.db.qp.physical.sys.TTLPlan;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.dataset.AuthDataSet;
import org.apache.iotdb.db.query.fill.IFill;
@@ -119,9 +120,21 @@ public class QueryProcessExecutor extends AbstractQueryProcessExecutor {
case PROPERTY:
PropertyPlan property = (PropertyPlan) plan;
return operateProperty(property);
+ case TTL:
+ operateTTL((TTLPlan) plan);
+ return true;
default:
throw new UnsupportedOperationException(
- String.format("operation %s does not support", plan.getOperatorType()));
+ String.format("operation %s is not supported", plan.getOperatorType()));
+ }
+ }
+
+ private void operateTTL(TTLPlan plan) throws ProcessorException {
+ try {
+ MManager.getInstance().setTTL(plan.getStorageGroup(), plan.getDataTTL());
+ StorageEngine.getInstance().setTTL(plan.getStorageGroup(), plan.getDataTTL());
+ } catch (PathErrorException | IOException | StorageEngineException e) {
+ throw new ProcessorException(e);
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/logical/Operator.java b/server/src/main/java/org/apache/iotdb/db/qp/logical/Operator.java
index cab4ed4..e2bdccd 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/logical/Operator.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/logical/Operator.java
@@ -73,6 +73,7 @@ public abstract class Operator {
GRANT_USER_PRIVILEGE, REVOKE_USER_PRIVILEGE, GRANT_USER_ROLE, REVOKE_USER_ROLE, CREATE_ROLE,
DELETE_ROLE, GRANT_ROLE_PRIVILEGE, REVOKE_ROLE_PRIVILEGE, LIST_USER, LIST_ROLE,
LIST_USER_PRIVILEGE, LIST_ROLE_PRIVILEGE, LIST_USER_ROLES, LIST_ROLE_USERS,
- GRANT_WATERMARK_EMBEDDING, REVOKE_WATERMARK_EMBEDDING
+ GRANT_WATERMARK_EMBEDDING, REVOKE_WATERMARK_EMBEDDING,
+ TTL
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MetadataOperationType.java b/server/src/main/java/org/apache/iotdb/db/qp/logical/sys/TTLOperator.java
similarity index 53%
copy from server/src/main/java/org/apache/iotdb/db/metadata/MetadataOperationType.java
copy to server/src/main/java/org/apache/iotdb/db/qp/logical/sys/TTLOperator.java
index 2e52eff..0fbcac4 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MetadataOperationType.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/logical/sys/TTLOperator.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -15,23 +15,35 @@
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
+ *
*/
-package org.apache.iotdb.db.metadata;
-public class MetadataOperationType {
+package org.apache.iotdb.db.qp.logical.sys;
+
+import org.apache.iotdb.db.qp.logical.RootOperator;
+
+public class TTLOperator extends RootOperator {
+
+ private String storageGroup;
+ private long dataTTL;
+
+ public TTLOperator(int tokenIntType) {
+ super(tokenIntType);
+ }
- private MetadataOperationType(){
- //allowed to do nothing
+ public String getStorageGroup() {
+ return storageGroup;
}
- public static final String ADD_PATH_TO_MTREE = "0";
- public static final String DELETE_PATH_FROM_MTREE = "1";
- public static final String SET_STORAGE_LEVEL_TO_MTREE = "2";
- public static final String ADD_A_PTREE = "3";
- public static final String ADD_A_PATH_TO_PTREE = "4";
- public static final String DELETE_PATH_FROM_PTREE = "5";
- public static final String LINK_MNODE_TO_PTREE = "6";
- public static final String UNLINK_MNODE_FROM_PTREE = "7";
- public static final String ADD_INDEX_TO_PATH = "8";
- public static final String DELETE_INDEX_FROM_PATH = "9";
-}
+ public void setStorageGroup(String storageGroup) {
+ this.storageGroup = storageGroup;
+ }
+
+ public long getDataTTL() {
+ return dataTTL;
+ }
+
+ public void setDataTTL(long dataTTL) {
+ this.dataTTL = dataTTL;
+ }
+}
\ No newline at end of file
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/TTLPlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/TTLPlan.java
new file mode 100644
index 0000000..30c0388
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/TTLPlan.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.iotdb.db.qp.physical.sys;
+
+import java.util.List;
+import org.apache.iotdb.db.qp.logical.Operator.OperatorType;
+import org.apache.iotdb.db.qp.physical.PhysicalPlan;
+import org.apache.iotdb.tsfile.read.common.Path;
+
+public class TTLPlan extends PhysicalPlan {
+
+ private String storageGroup;
+ private long dataTTL;
+
+ public TTLPlan(String storageGroup, long dataTTL) {
+ // set TTL
+ super(false, OperatorType.TTL);
+ this.storageGroup = storageGroup;
+ this.dataTTL = dataTTL;
+ }
+
+ public TTLPlan(String storageGroup) {
+ // unset TTL
+ this(storageGroup, Long.MAX_VALUE);
+ }
+
+ @Override
+ public List<Path> getPaths() {
+ return null;
+ }
+
+ public String getStorageGroup() {
+ return storageGroup;
+ }
+
+ public void setStorageGroup(String storageGroup) {
+ this.storageGroup = storageGroup;
+ }
+
+ public long getDataTTL() {
+ return dataTTL;
+ }
+
+ public void setDataTTL(long dataTTL) {
+ this.dataTTL = dataTTL;
+ }
+}
\ No newline at end of file
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java b/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java
index e83b3c1..65be2b5 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java
@@ -51,6 +51,7 @@ import org.apache.iotdb.db.qp.logical.sys.DataAuthOperator;
import org.apache.iotdb.db.qp.logical.sys.LoadDataOperator;
import org.apache.iotdb.db.qp.logical.sys.MetadataOperator;
import org.apache.iotdb.db.qp.logical.sys.PropertyOperator;
+import org.apache.iotdb.db.qp.logical.sys.TTLOperator;
import org.apache.iotdb.db.query.fill.IFill;
import org.apache.iotdb.db.query.fill.LinearFill;
import org.apache.iotdb.db.query.fill.PreviousFill;
@@ -82,7 +83,7 @@ public class LogicalGenerator {
}
public RootOperator getLogicalPlan(AstNode astNode)
- throws QueryProcessorException, ArgsErrorException, MetadataErrorException {
+ throws QueryProcessorException, MetadataErrorException {
analyze(astNode);
return initializedOperator;
}
@@ -94,7 +95,7 @@ public class LogicalGenerator {
* @throws ArgsErrorException args error
*/
private void analyze(AstNode astNode)
- throws QueryProcessorException, ArgsErrorException, MetadataErrorException {
+ throws QueryProcessorException, MetadataErrorException {
Token token = astNode.getToken();
if (token == null) {
throw new QueryProcessorException("given token is null");
@@ -210,6 +211,9 @@ public class LogicalGenerator {
case TSParser.TOK_SOFFSET:
analyzeSoffset(astNode);
return;
+ case TSParser.TOK_TTL:
+ analyzeTTL(astNode);
+ return;
default:
throw new QueryProcessorException("Not supported TSParser type" + tokenIntType);
}
@@ -218,6 +222,36 @@ public class LogicalGenerator {
}
}
+ private void analyzeTTL(AstNode astNode) throws QueryProcessorException {
+ int tokenType = astNode.getChild(0).getToken().getType();
+ switch (tokenType) {
+ case TSParser.TOK_SET:
+ analyzeSetTTL(astNode);
+ break;
+ case TSParser.TOK_UNSET:
+ analyzeUnsetTTL(astNode);
+ break;
+ default:
+ throw new QueryProcessorException("Not supported TSParser type in TTL:" + tokenType);
+ }
+ }
+
+ private void analyzeSetTTL(AstNode astNode) {
+ String path = astNode.getChild(1).getText();
+ long dataTTL = Long.parseLong(astNode.getChild(2).getText());
+ TTLOperator operator = new TTLOperator(TSParser.TOK_SET);
+ initializedOperator = operator;
+ operator.setStorageGroup(path);
+ operator.setDataTTL(dataTTL);
+ }
+
+ private void analyzeUnsetTTL(AstNode astNode) {
+ String path = astNode.getChild(1).getText();
+ TTLOperator operator = new TTLOperator(TSParser.TOK_UNSET);
+ initializedOperator = operator;
+ operator.setStorageGroup(path);
+ }
+
private void analyzeSlimit(AstNode astNode) throws LogicalOperatorException {
AstNode unit = astNode.getChild(0);
int seriesLimit;
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java b/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
index 819dba2..26ece61 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
@@ -35,6 +35,7 @@ import org.apache.iotdb.db.qp.logical.sys.DataAuthOperator;
import org.apache.iotdb.db.qp.logical.sys.LoadDataOperator;
import org.apache.iotdb.db.qp.logical.sys.MetadataOperator;
import org.apache.iotdb.db.qp.logical.sys.PropertyOperator;
+import org.apache.iotdb.db.qp.logical.sys.TTLOperator;
import org.apache.iotdb.db.qp.physical.PhysicalPlan;
import org.apache.iotdb.db.qp.physical.crud.AggregationPlan;
import org.apache.iotdb.db.qp.physical.crud.DeletePlan;
@@ -47,6 +48,8 @@ import org.apache.iotdb.db.qp.physical.sys.DataAuthPlan;
import org.apache.iotdb.db.qp.physical.sys.LoadDataPlan;
import org.apache.iotdb.db.qp.physical.sys.MetadataPlan;
import org.apache.iotdb.db.qp.physical.sys.PropertyPlan;
+import org.apache.iotdb.db.qp.physical.sys.TTLPlan;
+import org.apache.iotdb.db.sql.parse.TSParser;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.expression.IExpression;
@@ -121,6 +124,13 @@ public class PhysicalGenerator {
case QUERY:
QueryOperator query = (QueryOperator) operator;
return transformQuery(query);
+ case TTL:
+ TTLOperator ttlOperator = (TTLOperator) operator;
+ if (ttlOperator.getTokenIntType() == TSParser.TOK_SET) {
+ return new TTLPlan(ttlOperator.getStorageGroup(), ttlOperator.getDataTTL());
+ } else {
+ return new TTLPlan(ttlOperator.getStorageGroup());
+ }
default:
throw new LogicalOperatorException("not supported operator type: " + operator.getType());
}