You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2020/04/10 08:10:28 UTC
[incubator-iotdb] branch fix_recover_delete_bug created (now
897543f)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a change to branch fix_recover_delete_bug
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.
at 897543f fix recover delete bug
This branch includes the following new commits:
new 897543f fix recover delete bug
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-iotdb] 01/01: fix recover delete bug
Posted by qi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch fix_recover_delete_bug
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 897543f4720002fc9edc3abfb6367be2d571c69f
Author: qiaojialin <64...@qq.com>
AuthorDate: Fri Apr 10 16:10:06 2020 +0800
fix recover delete bug
---
.../writelog/recover/TsFileRecoverPerformer.java | 1 +
.../iotdb/db/integration/IoTDBRestartIT.java | 43 ++++++++++++++++++++++
.../iotdb/tsfile/read/TsFileSequenceReader.java | 16 ++------
.../apache/iotdb/tsfile/utils/VersionUtils.java | 36 ++++++++++++++++++
.../write/writer/RestorableTsFileIOWriter.java | 4 ++
.../iotdb/tsfile/write/writer/TsFileIOWriter.java | 2 +-
6 files changed, 88 insertions(+), 14 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java b/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
index aff32c8..71726cb 100644
--- a/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
+++ b/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
@@ -192,6 +192,7 @@ public class TsFileRecoverPerformer {
private void redoLogs(RestorableTsFileIOWriter restorableTsFileIOWriter)
throws StorageGroupProcessorException {
IMemTable recoverMemTable = new PrimitiveMemTable();
+ recoverMemTable.setVersion(versionController.nextVersion());
LogReplayer logReplayer = new LogReplayer(logNodePrefix, filePath, resource.getModFile(),
versionController, resource, recoverMemTable, acceptUnseq);
logReplayer.replayLogs();
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBRestartIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBRestartIT.java
index 931ecb2..a9854cd 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBRestartIT.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBRestartIT.java
@@ -84,4 +84,47 @@ public class IoTDBRestartIT {
EnvironmentUtils.cleanEnv();
}
+
+
+ @Test
+ public void testRestartDelete()
+ throws SQLException, ClassNotFoundException, IOException, StorageEngineException {
+ EnvironmentUtils.envSetUp();
+ Class.forName(Config.JDBC_DRIVER_NAME);
+
+ try(Connection connection = DriverManager
+ .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root",
+ "root");
+ Statement statement = connection.createStatement()){
+ statement.execute("insert into root.turbine.d1(timestamp,s1) values(1,1)");
+ statement.execute("insert into root.turbine.d1(timestamp,s1) values(2,2)");
+ statement.execute("insert into root.turbine.d1(timestamp,s1) values(3,3)");
+ }
+
+ EnvironmentUtils.restartDaemon();
+
+ try(Connection connection = DriverManager
+ .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root",
+ "root");
+ Statement statement = connection.createStatement()){
+ statement.execute("delete from root.turbine.d1.s1 where time<=1");
+ statement.execute("flush");
+ statement.execute("delete from root.turbine.d1.s1 where time<=2");
+
+ boolean hasResultSet = statement.execute("SELECT s1 FROM root.turbine.d1");
+ assertTrue(hasResultSet);
+ String[] exp = new String[]{
+ "3,3"
+ };
+ ResultSet resultSet = statement.getResultSet();
+ int cnt = 0;
+ while (resultSet.next()) {
+ String result = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(2);
+ assertEquals(exp[cnt], result);
+ cnt++;
+ }
+ }
+
+ EnvironmentUtils.cleanEnv();
+ }
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index 999ac28..79ccd96 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -40,6 +40,7 @@ import org.apache.iotdb.tsfile.read.controller.MetadataQuerierByFileImpl;
import org.apache.iotdb.tsfile.read.reader.TsFileInput;
import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+import org.apache.iotdb.tsfile.utils.VersionUtils;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -347,22 +348,11 @@ public class TsFileSequenceReader implements AutoCloseable {
// set version in ChunkMetadata
List<Pair<Long, Long>> versionInfo = tsFileMetaData.getVersionInfo();
for (Entry<String, List<ChunkMetadata>> entry : seriesMetadata.entrySet()) {
- setVersion(entry.getValue(), versionInfo);
+ VersionUtils.applyVersion(entry.getValue(), versionInfo);
}
return seriesMetadata;
}
-
- private void setVersion(List<ChunkMetadata> chunkMetadataList, List<Pair<Long, Long>> versionInfo) {
- int versionIndex = 0;
- for (ChunkMetadata chunkMetadata : chunkMetadataList) {
- while (chunkMetadata.getOffsetOfChunkHeader() >= versionInfo.get(versionIndex).left) {
- versionIndex++;
- }
- chunkMetadata.setVersion(versionInfo.get(versionIndex).right);
- }
- }
-
/**
* this function return all timeseries names in this file
*
@@ -721,7 +711,7 @@ public class TsFileSequenceReader implements AutoCloseable {
chunkMetadataList.add(ChunkMetadata.deserializeFrom(buffer));
}
- setVersion(chunkMetadataList, versionInfo);
+ VersionUtils.applyVersion(chunkMetadataList, versionInfo);
return chunkMetadataList;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/VersionUtils.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/VersionUtils.java
new file mode 100644
index 0000000..4948f68
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/VersionUtils.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.tsfile.utils;
+
+import java.util.List;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
+
+public class VersionUtils {
+
+ public static void applyVersion(List<ChunkMetadata> chunkMetadataList, List<Pair<Long, Long>> versionInfo) {
+ int versionIndex = 0;
+ for (ChunkMetadata chunkMetadata : chunkMetadataList) {
+ while (chunkMetadata.getOffsetOfChunkHeader() >= versionInfo.get(versionIndex).left) {
+ versionIndex++;
+ }
+ chunkMetadata.setVersion(versionInfo.get(versionIndex).right);
+ }
+ }
+
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java
index 3af18fc..f2755f5 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java
@@ -39,6 +39,7 @@ import org.apache.iotdb.tsfile.read.TsFileCheckStatus;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.utils.Pair;
+import org.apache.iotdb.tsfile.utils.VersionUtils;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -195,6 +196,9 @@ public class RestorableTsFileIOWriter extends TsFileIOWriter {
if (!newlyFlushedMetadataList.isEmpty()) {
for (ChunkGroupMetadata chunkGroupMetadata : newlyFlushedMetadataList) {
List<ChunkMetadata> rowMetaDataList = chunkGroupMetadata.getChunkMetadataList();
+
+ VersionUtils.applyVersion(rowMetaDataList, versionInfo);
+
String device = chunkGroupMetadata.getDevice();
for (ChunkMetadata chunkMetaData : rowMetaDataList) {
String measurementId = chunkMetaData.getMeasurementUid();
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
index 6f0ed79..87ee259 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
@@ -83,7 +83,7 @@ public class TsFileIOWriter {
private long markedPosition;
private String currentChunkGroupDeviceId;
private long currentChunkGroupStartOffset;
- private List<Pair<Long, Long>> versionInfo = new ArrayList<>();
+ protected List<Pair<Long, Long>> versionInfo = new ArrayList<>();
/**
* empty construct function.