You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by su...@apache.org on 2020/06/23 15:15:04 UTC
[incubator-iotdb] 01/01: [IOTDB-775] Add SQL of creating snapshot
manually
This is an automated email from the ASF dual-hosted git repository.
sunzesong pushed a commit to branch jira_775
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 7221bf83468aa164cb4e1e0ca5287b02fdce3703
Author: samperson1997 <sz...@mails.tsinghua.edu.cn>
AuthorDate: Tue Jun 23 23:14:22 2020 +0800
[IOTDB-775] Add SQL of creating snapshot manually
---
.../org/apache/iotdb/db/qp/strategy/SqlBase.g4 | 13 ++++++++
.../org/apache/iotdb/db/metadata/MManager.java | 39 ++++++++++++----------
.../main/java/org/apache/iotdb/db/qp/Planner.java | 1 +
.../apache/iotdb/db/qp/constant/SQLConstant.java | 2 ++
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 16 ++++++---
.../org/apache/iotdb/db/qp/logical/Operator.java | 2 +-
.../db/qp/logical/sys/CreateSnapshotOperator.java | 30 +++++++++++++++++
.../db/qp/physical/sys/CreateSnapshotPlan.java | 37 ++++++++++++++++++++
.../iotdb/db/qp/strategy/LogicalGenerator.java | 12 ++++---
.../iotdb/db/qp/strategy/PhysicalGenerator.java | 3 ++
10 files changed, 129 insertions(+), 26 deletions(-)
diff --git a/server/src/main/antlr4/org/apache/iotdb/db/qp/strategy/SqlBase.g4 b/server/src/main/antlr4/org/apache/iotdb/db/qp/strategy/SqlBase.g4
index 8170130..f78e854 100644
--- a/server/src/main/antlr4/org/apache/iotdb/db/qp/strategy/SqlBase.g4
+++ b/server/src/main/antlr4/org/apache/iotdb/db/qp/strategy/SqlBase.g4
@@ -80,6 +80,7 @@ statement
| LOAD STRING_LITERAL autoCreateSchema? #loadFiles
| REMOVE STRING_LITERAL #removeFile
| MOVE STRING_LITERAL STRING_LITERAL #moveFile
+ | CREATE SNAPSHOT FOR SCHEMA #createSnapshot
| SELECT INDEX func=ID //not support yet
LR_BRACKET
p1=fullPath COMMA p2=fullPath COMMA n1=timeValue COMMA n2=timeValue COMMA
@@ -883,6 +884,18 @@ LATEST
: L A T E S T
;
+SNAPSHOT
+ : S N A P S H O T
+ ;
+
+FOR
+ : F O R
+ ;
+
+SCHEMA
+ : S C H E M A
+ ;
+
//============================
// End of the keywords list
//============================
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
index 16847ca..1506d2b 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
@@ -1780,24 +1780,29 @@ public class MManager {
"MTree snapshot need not be created. Current mlog line number: {}, last snapshot line number: {}",
logWriter.getLineNumber(), lastSnapshotLogLineNumber);
} else {
- lock.readLock().lock();
- logger.info("Start creating MTree snapshot. This may take a while...");
- try {
- mtree.serializeTo(mtreeSnapshotTmpPath, logWriter.getLineNumber());
- lastSnapshotLogLineNumber = logWriter.getLineNumber();
- File tmpFile = SystemFileFactory.INSTANCE.getFile(mtreeSnapshotTmpPath);
- File snapshotFile = SystemFileFactory.INSTANCE.getFile(mtreeSnapshotPath);
- if (snapshotFile.exists()) {
- Files.delete(snapshotFile.toPath());
- }
- if (tmpFile.renameTo(snapshotFile)) {
- logger.info("Finish creating MTree snapshot to {}.", mtreeSnapshotPath);
- }
- } catch (IOException e) {
- logger.warn("Failed to create MTree snapshot to {}", mtreeSnapshotPath, e);
- } finally {
- lock.readLock().unlock();
+ createMTreeSnapshot();
+ }
+ }
+
+ public void createMTreeSnapshot() {
+ lock.readLock().lock();
+ logger.info("Start creating MTree snapshot. This may take a while...");
+ try {
+ mtree.serializeTo(mtreeSnapshotTmpPath, logWriter.getLineNumber());
+ lastSnapshotLogLineNumber = logWriter.getLineNumber();
+ File tmpFile = SystemFileFactory.INSTANCE.getFile(mtreeSnapshotTmpPath);
+ File snapshotFile = SystemFileFactory.INSTANCE.getFile(mtreeSnapshotPath);
+ if (snapshotFile.exists()) {
+ Files.delete(snapshotFile.toPath());
+ }
+ if (tmpFile.renameTo(snapshotFile)) {
+ logger.info("Finish creating MTree snapshot to {}, to mlog line number: {}",
+ mtreeSnapshotPath, lastSnapshotLogLineNumber);
}
+ } catch (IOException e) {
+ logger.warn("Failed to create MTree snapshot to {}", mtreeSnapshotPath, e);
+ } finally {
+ lock.readLock().unlock();
}
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/Planner.java b/server/src/main/java/org/apache/iotdb/db/qp/Planner.java
index 3b31bbe..74f6aa8 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/Planner.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/Planner.java
@@ -100,6 +100,7 @@ public class Planner {
case CLEAR_CACHE:
case NULL:
case SHOW_MERGE_STATUS:
+ case CREATE_SNAPSHOT:
return operator;
case QUERY:
case UPDATE:
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/constant/SQLConstant.java b/server/src/main/java/org/apache/iotdb/db/qp/constant/SQLConstant.java
index 1acf1a7..ca075a6 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/constant/SQLConstant.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/constant/SQLConstant.java
@@ -152,6 +152,8 @@ public class SQLConstant {
public static final int TOK_SHOW_MERGE_STATUS = 87;
+ public static final int TOK_CREATE_SNAPSHOT = 88;
+
public static final Map<Integer, String> tokenSymbol = new HashMap<>();
public static final Map<Integer, String> tokenNames = new HashMap<>();
public static final Map<Integer, Integer> reverseWords = new HashMap<>();
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
index 3cf570e..3e11619 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
@@ -106,6 +106,7 @@ import org.apache.iotdb.db.qp.physical.sys.AlterTimeSeriesPlan;
import org.apache.iotdb.db.qp.physical.sys.AuthorPlan;
import org.apache.iotdb.db.qp.physical.sys.ClearCachePlan;
import org.apache.iotdb.db.qp.physical.sys.CountPlan;
+import org.apache.iotdb.db.qp.physical.sys.CreateSnapshotPlan;
import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan;
import org.apache.iotdb.db.qp.physical.sys.DataAuthPlan;
import org.apache.iotdb.db.qp.physical.sys.DeleteStorageGroupPlan;
@@ -261,6 +262,9 @@ public class PlanExecutor implements IPlanExecutor {
case CLEAR_CACHE:
operateClearCache((ClearCachePlan) plan);
return true;
+ case CREATE_SNAPSHOT:
+ operateCreateSnapshot((CreateSnapshotPlan) plan);
+ return true;
default:
throw new UnsupportedOperationException(
String.format("operation %s is not supported", plan.getOperatorType()));
@@ -282,6 +286,10 @@ public class PlanExecutor implements IPlanExecutor {
TimeSeriesMetadataCache.getInstance().clear();
}
+ private void operateCreateSnapshot(CreateSnapshotPlan plan) {
+ mManager.createMTreeSnapshot();
+ }
+
private void operateFlush(FlushPlan plan) throws StorageGroupNotSetException {
if (plan.getPaths() == null) {
StorageEngine.getInstance().syncCloseAllProcessor();
@@ -1124,9 +1132,9 @@ public class PlanExecutor implements IPlanExecutor {
if (measurementNode.getSchema().getType() != insertTabletPlan.getDataTypes()[i]) {
if (!enablePartialInsert) {
throw new QueryProcessException(String.format(
- "Datatype mismatch, Insert measurement %s type %s, metadata tree type %s",
- measurement, insertTabletPlan.getDataTypes()[i],
- measurementNode.getSchema().getType()));
+ "Datatype mismatch, Insert measurement %s type %s, metadata tree type %s",
+ measurement, insertTabletPlan.getDataTypes()[i],
+ measurementNode.getSchema().getType()));
} else {
insertTabletPlan.markMeasurementInsertionFailed(i);
continue;
@@ -1140,7 +1148,7 @@ public class PlanExecutor implements IPlanExecutor {
StorageEngine.getInstance().insertTablet(insertTabletPlan);
if (insertTabletPlan.getFailedMeasurements() != null) {
throw new StorageEngineException(
- "failed to insert measurements " + insertTabletPlan.getFailedMeasurements());
+ "failed to insert measurements " + insertTabletPlan.getFailedMeasurements());
}
} catch (StorageEngineException | MetadataException e) {
throw new QueryProcessException(e);
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/logical/Operator.java b/server/src/main/java/org/apache/iotdb/db/qp/logical/Operator.java
index 746326c..ead36f6 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/logical/Operator.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/logical/Operator.java
@@ -76,6 +76,6 @@ public abstract class Operator {
GRANT_WATERMARK_EMBEDDING, REVOKE_WATERMARK_EMBEDDING,
TTL, DELETE_STORAGE_GROUP, LOAD_CONFIGURATION, SHOW, LOAD_FILES, REMOVE_FILE, MOVE_FILE, LAST, GROUP_BY_FILL,
ALTER_TIMESERIES, FLUSH, MERGE, FULL_MERGE, CLEAR_CACHE,
- SHOW_MERGE_STATUS
+ SHOW_MERGE_STATUS, CREATE_SNAPSHOT
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/logical/sys/CreateSnapshotOperator.java b/server/src/main/java/org/apache/iotdb/db/qp/logical/sys/CreateSnapshotOperator.java
new file mode 100644
index 0000000..341eb03
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/qp/logical/sys/CreateSnapshotOperator.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.qp.logical.sys;
+
+import org.apache.iotdb.db.qp.logical.RootOperator;
+
+public class CreateSnapshotOperator extends RootOperator {
+
+ public CreateSnapshotOperator(int tokenIntType) {
+ super(tokenIntType);
+ operatorType = OperatorType.CREATE_SNAPSHOT;
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/CreateSnapshotPlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/CreateSnapshotPlan.java
new file mode 100644
index 0000000..3e46a54
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/CreateSnapshotPlan.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.qp.physical.sys;
+
+import java.util.List;
+import org.apache.iotdb.db.qp.logical.Operator.OperatorType;
+import org.apache.iotdb.db.qp.physical.PhysicalPlan;
+import org.apache.iotdb.tsfile.read.common.Path;
+
+public class CreateSnapshotPlan extends PhysicalPlan {
+
+ public CreateSnapshotPlan() {
+ super(false, OperatorType.CREATE_SNAPSHOT);
+ }
+
+ @Override
+ public List<Path> getPaths() {
+ return null;
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java b/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java
index ebd41f0..f27e02f 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java
@@ -48,6 +48,7 @@ import org.apache.iotdb.db.qp.logical.sys.AuthorOperator;
import org.apache.iotdb.db.qp.logical.sys.AuthorOperator.AuthorType;
import org.apache.iotdb.db.qp.logical.sys.ClearCacheOperator;
import org.apache.iotdb.db.qp.logical.sys.CountOperator;
+import org.apache.iotdb.db.qp.logical.sys.CreateSnapshotOperator;
import org.apache.iotdb.db.qp.logical.sys.CreateTimeSeriesOperator;
import org.apache.iotdb.db.qp.logical.sys.DataAuthOperator;
import org.apache.iotdb.db.qp.logical.sys.DeleteStorageGroupOperator;
@@ -78,6 +79,7 @@ import org.apache.iotdb.db.qp.strategy.SqlBaseParser.ConstantContext;
import org.apache.iotdb.db.qp.strategy.SqlBaseParser.CountNodesContext;
import org.apache.iotdb.db.qp.strategy.SqlBaseParser.CountTimeseriesContext;
import org.apache.iotdb.db.qp.strategy.SqlBaseParser.CreateRoleContext;
+import org.apache.iotdb.db.qp.strategy.SqlBaseParser.CreateSnapshotContext;
import org.apache.iotdb.db.qp.strategy.SqlBaseParser.CreateTimeseriesContext;
import org.apache.iotdb.db.qp.strategy.SqlBaseParser.CreateUserContext;
import org.apache.iotdb.db.qp.strategy.SqlBaseParser.DateExpressionContext;
@@ -1481,10 +1483,7 @@ public class LogicalGenerator extends SqlBaseBaseListener {
/**
* parse time expression, which is addition and subtraction expression of duration time, now() or
- * DataTimeFormat time.
- * <p>
- * eg. now() + 1d - 2h
- * </p>
+ * DataTimeFormat time. <p> eg. now() + 1d - 2h </p>
*/
private Long parseDateExpression(DateExpressionContext ctx) {
long time;
@@ -1622,4 +1621,9 @@ public class LogicalGenerator extends SqlBaseBaseListener {
super.enterShowMergeStatus(ctx);
initializedOperator = new ShowMergeStatusOperator(SQLConstant.TOK_SHOW_MERGE_STATUS);
}
+
+ public void enterCreateSnapshot(CreateSnapshotContext ctx) {
+ super.enterCreateSnapshot(ctx);
+ initializedOperator = new CreateSnapshotOperator(SQLConstant.TOK_CREATE_SNAPSHOT);
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java b/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
index d050e5d..2ba9a20 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
@@ -76,6 +76,7 @@ import org.apache.iotdb.db.qp.physical.sys.AlterTimeSeriesPlan;
import org.apache.iotdb.db.qp.physical.sys.AuthorPlan;
import org.apache.iotdb.db.qp.physical.sys.ClearCachePlan;
import org.apache.iotdb.db.qp.physical.sys.CountPlan;
+import org.apache.iotdb.db.qp.physical.sys.CreateSnapshotPlan;
import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan;
import org.apache.iotdb.db.qp.physical.sys.DataAuthPlan;
import org.apache.iotdb.db.qp.physical.sys.DeleteStorageGroupPlan;
@@ -258,6 +259,8 @@ public class PhysicalGenerator {
return new ClearCachePlan();
case SHOW_MERGE_STATUS:
return new ShowMergeStatusPlan();
+ case CREATE_SNAPSHOT:
+ return new CreateSnapshotPlan();
default:
throw new LogicalOperatorException(operator.getType().toString(), "");
}