You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2020/11/11 04:45:12 UTC
[iotdb] 01/01: fix recover bug
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch RecoverBug
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit a82f08eb1fe367d13898cb9dc02210ff51ec1156
Author: JackieTien97 <Ja...@foxmail.com>
AuthorDate: Wed Nov 11 12:44:43 2020 +0800
fix recover bug
---
.../engine/storagegroup/StorageGroupProcessor.java | 44 +++++++++++-----------
.../writelog/recover/TsFileRecoverPerformer.java | 18 +++++----
.../recover/RecoverResourceFromReaderTest.java | 2 +-
.../db/writelog/recover/SeqTsFileRecoverTest.java | 4 +-
.../writelog/recover/UnseqTsFileRecoverTest.java | 2 +-
5 files changed, 37 insertions(+), 33 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index f581489..618e020 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -86,7 +86,6 @@ import org.apache.iotdb.db.writelog.recover.TsFileRecoverPerformer;
import org.apache.iotdb.rpc.RpcUtils;
import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.service.rpc.thrift.TSStatus;
-import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
import org.apache.iotdb.tsfile.fileSystem.fsFactory.FSFactory;
@@ -316,19 +315,6 @@ public class StorageGroupProcessor {
recoverTsFiles(value, false);
}
- for (TsFileResource resource : tsFileManagement.getTsFileList(true)) {
- long partitionNum = resource.getTimePartition();
- partitionDirectFileVersions.computeIfAbsent(partitionNum, p -> new HashSet<>())
- .addAll(resource.getHistoricalVersions());
- updatePartitionFileVersion(partitionNum, Collections.max(resource.getHistoricalVersions()));
- }
- for (TsFileResource resource : tsFileManagement.getTsFileList(false)) {
- long partitionNum = resource.getTimePartition();
- partitionDirectFileVersions.computeIfAbsent(partitionNum, p -> new HashSet<>())
- .addAll(resource.getHistoricalVersions());
- updatePartitionFileVersion(partitionNum, Collections.max(resource.getHistoricalVersions()));
- }
-
String taskName = storageGroupName + "-" + System.currentTimeMillis();
File mergingMods = SystemFileFactory.INSTANCE.getFile(storageGroupSysDir,
MERGING_MODIFICATION_FILE_NAME);
@@ -349,6 +335,18 @@ public class StorageGroupProcessor {
}
tsFileManagement.recover();
+ for (TsFileResource resource : tsFileManagement.getTsFileList(true)) {
+ long partitionNum = resource.getTimePartition();
+ partitionDirectFileVersions.computeIfAbsent(partitionNum, p -> new HashSet<>())
+ .addAll(resource.getHistoricalVersions());
+ updatePartitionFileVersion(partitionNum, Collections.max(resource.getHistoricalVersions()));
+ }
+ for (TsFileResource resource : tsFileManagement.getTsFileList(false)) {
+ long partitionNum = resource.getTimePartition();
+ partitionDirectFileVersions.computeIfAbsent(partitionNum, p -> new HashSet<>())
+ .addAll(resource.getHistoricalVersions());
+ updatePartitionFileVersion(partitionNum, Collections.max(resource.getHistoricalVersions()));
+ }
updateLatestFlushedTime();
} catch (IOException | MetadataException e) {
throw new StorageGroupProcessorException(e);
@@ -560,12 +558,6 @@ public class StorageGroupProcessor {
private void recoverTsFiles(List<TsFileResource> tsFiles, boolean isSeq) {
for (int i = 0; i < tsFiles.size(); i++) {
TsFileResource tsFileResource = tsFiles.get(i);
- // this tsfile is not zero level, no need to perform recovery here
- if (LevelCompactionTsFileManagement.getMergeLevel(tsFileResource.getTsFile()) > 0) {
- tsFileResource.setClosed(true);
- tsFileManagement.add(tsFileResource, isSeq);
- continue;
- }
long timePartitionId = tsFileResource.getTimePartition();
TsFileRecoverPerformer recoverPerformer = new TsFileRecoverPerformer(
@@ -575,13 +567,23 @@ public class StorageGroupProcessor {
RestorableTsFileIOWriter writer;
try {
- writer = recoverPerformer.recover();
+ // this tsfile is not zero level, no need to perform redo wal
+ if (LevelCompactionTsFileManagement.getMergeLevel(tsFileResource.getTsFile()) > 0) {
+ recoverPerformer.recover(false);
+ tsFileResource.setClosed(true);
+ tsFileManagement.add(tsFileResource, isSeq);
+ continue;
+ } else {
+ writer = recoverPerformer.recover(true);
+ }
} catch (StorageGroupProcessorException e) {
logger.warn("Skip TsFile: {} because of error in recover: ", tsFileResource.getTsFilePath(),
e);
continue;
}
+
+
if (i != tsFiles.size() - 1 || !writer.canWrite()) {
// not the last file or cannot write, just close it
tsFileResource.setClosed(true);
diff --git a/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java b/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
index 4f1be33..f2ccfe6 100644
--- a/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
+++ b/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
@@ -85,7 +85,7 @@ public class TsFileRecoverPerformer {
* writing
*/
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
- public RestorableTsFileIOWriter recover()
+ public RestorableTsFileIOWriter recover(boolean needRedoWal)
throws StorageGroupProcessorException {
File file = FSFactoryProducer.getFSFactory().getFile(filePath);
@@ -123,14 +123,16 @@ public class TsFileRecoverPerformer {
recoverResourceFromWriter(restorableTsFileIOWriter);
// redo logs
- redoLogs(restorableTsFileIOWriter);
+ if (needRedoWal) {
+ redoLogs(restorableTsFileIOWriter);
- // clean logs
- try {
- MultiFileLogNodeManager.getInstance()
- .deleteNode(logNodePrefix + SystemFileFactory.INSTANCE.getFile(filePath).getName());
- } catch (IOException e) {
- throw new StorageGroupProcessorException(e);
+ // clean logs
+ try {
+ MultiFileLogNodeManager.getInstance()
+ .deleteNode(logNodePrefix + SystemFileFactory.INSTANCE.getFile(filePath).getName());
+ } catch (IOException e) {
+ throw new StorageGroupProcessorException(e);
+ }
}
return restorableTsFileIOWriter;
}
diff --git a/server/src/test/java/org/apache/iotdb/db/writelog/recover/RecoverResourceFromReaderTest.java b/server/src/test/java/org/apache/iotdb/db/writelog/recover/RecoverResourceFromReaderTest.java
index 0283cfd..94d4feb 100644
--- a/server/src/test/java/org/apache/iotdb/db/writelog/recover/RecoverResourceFromReaderTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/writelog/recover/RecoverResourceFromReaderTest.java
@@ -183,7 +183,7 @@ public class RecoverResourceFromReaderTest {
TsFileRecoverPerformer performer = new TsFileRecoverPerformer(logNodePrefix, versionController,
resource, false, false);
- performer.recover().close();
+ performer.recover(true).close();
assertEquals(1, resource.getStartTime("root.sg.device99"));
assertEquals(300, resource.getEndTime("root.sg.device99"));
for (int i = 0; i < 10; i++) {
diff --git a/server/src/test/java/org/apache/iotdb/db/writelog/recover/SeqTsFileRecoverTest.java b/server/src/test/java/org/apache/iotdb/db/writelog/recover/SeqTsFileRecoverTest.java
index aaa080c..d550f60 100644
--- a/server/src/test/java/org/apache/iotdb/db/writelog/recover/SeqTsFileRecoverTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/writelog/recover/SeqTsFileRecoverTest.java
@@ -171,7 +171,7 @@ public class SeqTsFileRecoverTest {
public void testNonLastRecovery() throws StorageGroupProcessorException, IOException {
TsFileRecoverPerformer performer = new TsFileRecoverPerformer(logNodePrefix, versionController,
resource, false, false);
- RestorableTsFileIOWriter writer = performer.recover();
+ RestorableTsFileIOWriter writer = performer.recover(true);
assertFalse(writer.canWrite());
writer.close();
@@ -220,7 +220,7 @@ public class SeqTsFileRecoverTest {
public void testLastRecovery() throws StorageGroupProcessorException, IOException {
TsFileRecoverPerformer performer = new TsFileRecoverPerformer(logNodePrefix, versionController,
resource, false, true);
- RestorableTsFileIOWriter writer = performer.recover();
+ RestorableTsFileIOWriter writer = performer.recover(true);
writer.makeMetadataVisible();
assertEquals(11, writer.getMetadatasForQuery().size());
diff --git a/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java b/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java
index e9c411c..8caa93a 100644
--- a/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java
@@ -180,7 +180,7 @@ public class UnseqTsFileRecoverTest {
public void test() throws StorageGroupProcessorException, IOException {
TsFileRecoverPerformer performer = new TsFileRecoverPerformer(logNodePrefix,
versionController, resource, false, false);
- performer.recover().close();
+ performer.recover(true).close();
assertEquals(1, resource.getStartTime("root.sg.device99"));
assertEquals(300, resource.getEndTime("root.sg.device99"));