You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2020/04/10 08:10:29 UTC
[incubator-iotdb] 01/01: fix recover delete bug
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch fix_recover_delete_bug
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 897543f4720002fc9edc3abfb6367be2d571c69f
Author: qiaojialin <64...@qq.com>
AuthorDate: Fri Apr 10 16:10:06 2020 +0800
fix recover delete bug
---
.../writelog/recover/TsFileRecoverPerformer.java | 1 +
.../iotdb/db/integration/IoTDBRestartIT.java | 43 ++++++++++++++++++++++
.../iotdb/tsfile/read/TsFileSequenceReader.java | 16 ++------
.../apache/iotdb/tsfile/utils/VersionUtils.java | 36 ++++++++++++++++++
.../write/writer/RestorableTsFileIOWriter.java | 4 ++
.../iotdb/tsfile/write/writer/TsFileIOWriter.java | 2 +-
6 files changed, 88 insertions(+), 14 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java b/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
index aff32c8..71726cb 100644
--- a/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
+++ b/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
@@ -192,6 +192,7 @@ public class TsFileRecoverPerformer {
private void redoLogs(RestorableTsFileIOWriter restorableTsFileIOWriter)
throws StorageGroupProcessorException {
IMemTable recoverMemTable = new PrimitiveMemTable();
+ recoverMemTable.setVersion(versionController.nextVersion());
LogReplayer logReplayer = new LogReplayer(logNodePrefix, filePath, resource.getModFile(),
versionController, resource, recoverMemTable, acceptUnseq);
logReplayer.replayLogs();
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBRestartIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBRestartIT.java
index 931ecb2..a9854cd 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBRestartIT.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBRestartIT.java
@@ -84,4 +84,47 @@ public class IoTDBRestartIT {
EnvironmentUtils.cleanEnv();
}
+
+
+ @Test
+ public void testRestartDelete()
+ throws SQLException, ClassNotFoundException, IOException, StorageEngineException {
+ EnvironmentUtils.envSetUp();
+ Class.forName(Config.JDBC_DRIVER_NAME);
+
+ try(Connection connection = DriverManager
+ .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root",
+ "root");
+ Statement statement = connection.createStatement()){
+ statement.execute("insert into root.turbine.d1(timestamp,s1) values(1,1)");
+ statement.execute("insert into root.turbine.d1(timestamp,s1) values(2,2)");
+ statement.execute("insert into root.turbine.d1(timestamp,s1) values(3,3)");
+ }
+
+ EnvironmentUtils.restartDaemon();
+
+ try(Connection connection = DriverManager
+ .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root",
+ "root");
+ Statement statement = connection.createStatement()){
+ statement.execute("delete from root.turbine.d1.s1 where time<=1");
+ statement.execute("flush");
+ statement.execute("delete from root.turbine.d1.s1 where time<=2");
+
+ boolean hasResultSet = statement.execute("SELECT s1 FROM root.turbine.d1");
+ assertTrue(hasResultSet);
+ String[] exp = new String[]{
+ "3,3"
+ };
+ ResultSet resultSet = statement.getResultSet();
+ int cnt = 0;
+ while (resultSet.next()) {
+ String result = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(2);
+ assertEquals(exp[cnt], result);
+ cnt++;
+ }
+ }
+
+ EnvironmentUtils.cleanEnv();
+ }
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index 999ac28..79ccd96 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -40,6 +40,7 @@ import org.apache.iotdb.tsfile.read.controller.MetadataQuerierByFileImpl;
import org.apache.iotdb.tsfile.read.reader.TsFileInput;
import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+import org.apache.iotdb.tsfile.utils.VersionUtils;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -347,22 +348,11 @@ public class TsFileSequenceReader implements AutoCloseable {
// set version in ChunkMetadata
List<Pair<Long, Long>> versionInfo = tsFileMetaData.getVersionInfo();
for (Entry<String, List<ChunkMetadata>> entry : seriesMetadata.entrySet()) {
- setVersion(entry.getValue(), versionInfo);
+ VersionUtils.applyVersion(entry.getValue(), versionInfo);
}
return seriesMetadata;
}
-
- private void setVersion(List<ChunkMetadata> chunkMetadataList, List<Pair<Long, Long>> versionInfo) {
- int versionIndex = 0;
- for (ChunkMetadata chunkMetadata : chunkMetadataList) {
- while (chunkMetadata.getOffsetOfChunkHeader() >= versionInfo.get(versionIndex).left) {
- versionIndex++;
- }
- chunkMetadata.setVersion(versionInfo.get(versionIndex).right);
- }
- }
-
/**
* this function return all timeseries names in this file
*
@@ -721,7 +711,7 @@ public class TsFileSequenceReader implements AutoCloseable {
chunkMetadataList.add(ChunkMetadata.deserializeFrom(buffer));
}
- setVersion(chunkMetadataList, versionInfo);
+ VersionUtils.applyVersion(chunkMetadataList, versionInfo);
return chunkMetadataList;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/VersionUtils.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/VersionUtils.java
new file mode 100644
index 0000000..4948f68
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/VersionUtils.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.tsfile.utils;
+
+import java.util.List;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
+
+public class VersionUtils {
+
+ public static void applyVersion(List<ChunkMetadata> chunkMetadataList, List<Pair<Long, Long>> versionInfo) {
+ int versionIndex = 0;
+ for (ChunkMetadata chunkMetadata : chunkMetadataList) {
+ while (chunkMetadata.getOffsetOfChunkHeader() >= versionInfo.get(versionIndex).left) {
+ versionIndex++;
+ }
+ chunkMetadata.setVersion(versionInfo.get(versionIndex).right);
+ }
+ }
+
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java
index 3af18fc..f2755f5 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java
@@ -39,6 +39,7 @@ import org.apache.iotdb.tsfile.read.TsFileCheckStatus;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.utils.Pair;
+import org.apache.iotdb.tsfile.utils.VersionUtils;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -195,6 +196,9 @@ public class RestorableTsFileIOWriter extends TsFileIOWriter {
if (!newlyFlushedMetadataList.isEmpty()) {
for (ChunkGroupMetadata chunkGroupMetadata : newlyFlushedMetadataList) {
List<ChunkMetadata> rowMetaDataList = chunkGroupMetadata.getChunkMetadataList();
+
+ VersionUtils.applyVersion(rowMetaDataList, versionInfo);
+
String device = chunkGroupMetadata.getDevice();
for (ChunkMetadata chunkMetaData : rowMetaDataList) {
String measurementId = chunkMetaData.getMeasurementUid();
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
index 6f0ed79..87ee259 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
@@ -83,7 +83,7 @@ public class TsFileIOWriter {
private long markedPosition;
private String currentChunkGroupDeviceId;
private long currentChunkGroupStartOffset;
- private List<Pair<Long, Long>> versionInfo = new ArrayList<>();
+ protected List<Pair<Long, Long>> versionInfo = new ArrayList<>();
/**
* empty construct function.