You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2020/11/11 04:45:11 UTC
[iotdb] branch RecoverBug created (now a82f08e)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a change to branch RecoverBug
in repository https://gitbox.apache.org/repos/asf/iotdb.git.
at a82f08e fix recover bug
This branch includes the following new commits:
new a82f08e fix recover bug
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: fix recover bug
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch RecoverBug
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit a82f08eb1fe367d13898cb9dc02210ff51ec1156
Author: JackieTien97 <Ja...@foxmail.com>
AuthorDate: Wed Nov 11 12:44:43 2020 +0800
fix recover bug
---
.../engine/storagegroup/StorageGroupProcessor.java | 44 +++++++++++-----------
.../writelog/recover/TsFileRecoverPerformer.java | 18 +++++----
.../recover/RecoverResourceFromReaderTest.java | 2 +-
.../db/writelog/recover/SeqTsFileRecoverTest.java | 4 +-
.../writelog/recover/UnseqTsFileRecoverTest.java | 2 +-
5 files changed, 37 insertions(+), 33 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index f581489..618e020 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -86,7 +86,6 @@ import org.apache.iotdb.db.writelog.recover.TsFileRecoverPerformer;
import org.apache.iotdb.rpc.RpcUtils;
import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.service.rpc.thrift.TSStatus;
-import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
import org.apache.iotdb.tsfile.fileSystem.fsFactory.FSFactory;
@@ -316,19 +315,6 @@ public class StorageGroupProcessor {
recoverTsFiles(value, false);
}
- for (TsFileResource resource : tsFileManagement.getTsFileList(true)) {
- long partitionNum = resource.getTimePartition();
- partitionDirectFileVersions.computeIfAbsent(partitionNum, p -> new HashSet<>())
- .addAll(resource.getHistoricalVersions());
- updatePartitionFileVersion(partitionNum, Collections.max(resource.getHistoricalVersions()));
- }
- for (TsFileResource resource : tsFileManagement.getTsFileList(false)) {
- long partitionNum = resource.getTimePartition();
- partitionDirectFileVersions.computeIfAbsent(partitionNum, p -> new HashSet<>())
- .addAll(resource.getHistoricalVersions());
- updatePartitionFileVersion(partitionNum, Collections.max(resource.getHistoricalVersions()));
- }
-
String taskName = storageGroupName + "-" + System.currentTimeMillis();
File mergingMods = SystemFileFactory.INSTANCE.getFile(storageGroupSysDir,
MERGING_MODIFICATION_FILE_NAME);
@@ -349,6 +335,18 @@ public class StorageGroupProcessor {
}
tsFileManagement.recover();
+ for (TsFileResource resource : tsFileManagement.getTsFileList(true)) {
+ long partitionNum = resource.getTimePartition();
+ partitionDirectFileVersions.computeIfAbsent(partitionNum, p -> new HashSet<>())
+ .addAll(resource.getHistoricalVersions());
+ updatePartitionFileVersion(partitionNum, Collections.max(resource.getHistoricalVersions()));
+ }
+ for (TsFileResource resource : tsFileManagement.getTsFileList(false)) {
+ long partitionNum = resource.getTimePartition();
+ partitionDirectFileVersions.computeIfAbsent(partitionNum, p -> new HashSet<>())
+ .addAll(resource.getHistoricalVersions());
+ updatePartitionFileVersion(partitionNum, Collections.max(resource.getHistoricalVersions()));
+ }
updateLatestFlushedTime();
} catch (IOException | MetadataException e) {
throw new StorageGroupProcessorException(e);
@@ -560,12 +558,6 @@ public class StorageGroupProcessor {
private void recoverTsFiles(List<TsFileResource> tsFiles, boolean isSeq) {
for (int i = 0; i < tsFiles.size(); i++) {
TsFileResource tsFileResource = tsFiles.get(i);
- // this tsfile is not zero level, no need to perform recovery here
- if (LevelCompactionTsFileManagement.getMergeLevel(tsFileResource.getTsFile()) > 0) {
- tsFileResource.setClosed(true);
- tsFileManagement.add(tsFileResource, isSeq);
- continue;
- }
long timePartitionId = tsFileResource.getTimePartition();
TsFileRecoverPerformer recoverPerformer = new TsFileRecoverPerformer(
@@ -575,13 +567,23 @@ public class StorageGroupProcessor {
RestorableTsFileIOWriter writer;
try {
- writer = recoverPerformer.recover();
+ // this tsfile is not zero level, no need to perform redo wal
+ if (LevelCompactionTsFileManagement.getMergeLevel(tsFileResource.getTsFile()) > 0) {
+ recoverPerformer.recover(false);
+ tsFileResource.setClosed(true);
+ tsFileManagement.add(tsFileResource, isSeq);
+ continue;
+ } else {
+ writer = recoverPerformer.recover(true);
+ }
} catch (StorageGroupProcessorException e) {
logger.warn("Skip TsFile: {} because of error in recover: ", tsFileResource.getTsFilePath(),
e);
continue;
}
+
+
if (i != tsFiles.size() - 1 || !writer.canWrite()) {
// not the last file or cannot write, just close it
tsFileResource.setClosed(true);
diff --git a/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java b/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
index 4f1be33..f2ccfe6 100644
--- a/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
+++ b/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
@@ -85,7 +85,7 @@ public class TsFileRecoverPerformer {
* writing
*/
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
- public RestorableTsFileIOWriter recover()
+ public RestorableTsFileIOWriter recover(boolean needRedoWal)
throws StorageGroupProcessorException {
File file = FSFactoryProducer.getFSFactory().getFile(filePath);
@@ -123,14 +123,16 @@ public class TsFileRecoverPerformer {
recoverResourceFromWriter(restorableTsFileIOWriter);
// redo logs
- redoLogs(restorableTsFileIOWriter);
+ if (needRedoWal) {
+ redoLogs(restorableTsFileIOWriter);
- // clean logs
- try {
- MultiFileLogNodeManager.getInstance()
- .deleteNode(logNodePrefix + SystemFileFactory.INSTANCE.getFile(filePath).getName());
- } catch (IOException e) {
- throw new StorageGroupProcessorException(e);
+ // clean logs
+ try {
+ MultiFileLogNodeManager.getInstance()
+ .deleteNode(logNodePrefix + SystemFileFactory.INSTANCE.getFile(filePath).getName());
+ } catch (IOException e) {
+ throw new StorageGroupProcessorException(e);
+ }
}
return restorableTsFileIOWriter;
}
diff --git a/server/src/test/java/org/apache/iotdb/db/writelog/recover/RecoverResourceFromReaderTest.java b/server/src/test/java/org/apache/iotdb/db/writelog/recover/RecoverResourceFromReaderTest.java
index 0283cfd..94d4feb 100644
--- a/server/src/test/java/org/apache/iotdb/db/writelog/recover/RecoverResourceFromReaderTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/writelog/recover/RecoverResourceFromReaderTest.java
@@ -183,7 +183,7 @@ public class RecoverResourceFromReaderTest {
TsFileRecoverPerformer performer = new TsFileRecoverPerformer(logNodePrefix, versionController,
resource, false, false);
- performer.recover().close();
+ performer.recover(true).close();
assertEquals(1, resource.getStartTime("root.sg.device99"));
assertEquals(300, resource.getEndTime("root.sg.device99"));
for (int i = 0; i < 10; i++) {
diff --git a/server/src/test/java/org/apache/iotdb/db/writelog/recover/SeqTsFileRecoverTest.java b/server/src/test/java/org/apache/iotdb/db/writelog/recover/SeqTsFileRecoverTest.java
index aaa080c..d550f60 100644
--- a/server/src/test/java/org/apache/iotdb/db/writelog/recover/SeqTsFileRecoverTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/writelog/recover/SeqTsFileRecoverTest.java
@@ -171,7 +171,7 @@ public class SeqTsFileRecoverTest {
public void testNonLastRecovery() throws StorageGroupProcessorException, IOException {
TsFileRecoverPerformer performer = new TsFileRecoverPerformer(logNodePrefix, versionController,
resource, false, false);
- RestorableTsFileIOWriter writer = performer.recover();
+ RestorableTsFileIOWriter writer = performer.recover(true);
assertFalse(writer.canWrite());
writer.close();
@@ -220,7 +220,7 @@ public class SeqTsFileRecoverTest {
public void testLastRecovery() throws StorageGroupProcessorException, IOException {
TsFileRecoverPerformer performer = new TsFileRecoverPerformer(logNodePrefix, versionController,
resource, false, true);
- RestorableTsFileIOWriter writer = performer.recover();
+ RestorableTsFileIOWriter writer = performer.recover(true);
writer.makeMetadataVisible();
assertEquals(11, writer.getMetadatasForQuery().size());
diff --git a/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java b/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java
index e9c411c..8caa93a 100644
--- a/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java
@@ -180,7 +180,7 @@ public class UnseqTsFileRecoverTest {
public void test() throws StorageGroupProcessorException, IOException {
TsFileRecoverPerformer performer = new TsFileRecoverPerformer(logNodePrefix,
versionController, resource, false, false);
- performer.recover().close();
+ performer.recover(true).close();
assertEquals(1, resource.getStartTime("root.sg.device99"));
assertEquals(300, resource.getEndTime("root.sg.device99"));