You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2020/11/11 04:52:39 UTC
[iotdb] branch RecoverBug11 created (now d6a559a)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a change to branch RecoverBug11
in repository https://gitbox.apache.org/repos/asf/iotdb.git.
at d6a559a fix recover bug
This branch includes the following new commits:
new d6a559a fix recover bug
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: fix recover bug
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch RecoverBug11
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit d6a559ac23f57123d3309aa7abf8979d5fb9a66c
Author: JackieTien97 <Ja...@foxmail.com>
AuthorDate: Wed Nov 11 12:52:09 2020 +0800
fix recover bug
---
.../engine/storagegroup/StorageGroupProcessor.java | 44 +++++++++++-----------
.../writelog/recover/TsFileRecoverPerformer.java | 20 +++++-----
.../recover/RecoverResourceFromReaderTest.java | 2 +-
.../db/writelog/recover/SeqTsFileRecoverTest.java | 4 +-
.../writelog/recover/UnseqTsFileRecoverTest.java | 2 +-
5 files changed, 37 insertions(+), 35 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index 2199f77..5f82625 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -86,7 +86,6 @@ import org.apache.iotdb.db.writelog.recover.TsFileRecoverPerformer;
import org.apache.iotdb.rpc.RpcUtils;
import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.service.rpc.thrift.TSStatus;
-import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
import org.apache.iotdb.tsfile.fileSystem.fsFactory.FSFactory;
@@ -316,19 +315,6 @@ public class StorageGroupProcessor {
recoverTsFiles(value, false);
}
- for (TsFileResource resource : tsFileManagement.getTsFileList(true)) {
- long partitionNum = resource.getTimePartition();
- partitionDirectFileVersions.computeIfAbsent(partitionNum, p -> new HashSet<>())
- .addAll(resource.getHistoricalVersions());
- updatePartitionFileVersion(partitionNum, Collections.max(resource.getHistoricalVersions()));
- }
- for (TsFileResource resource : tsFileManagement.getTsFileList(false)) {
- long partitionNum = resource.getTimePartition();
- partitionDirectFileVersions.computeIfAbsent(partitionNum, p -> new HashSet<>())
- .addAll(resource.getHistoricalVersions());
- updatePartitionFileVersion(partitionNum, Collections.max(resource.getHistoricalVersions()));
- }
-
String taskName = storageGroupName + "-" + System.currentTimeMillis();
File mergingMods = SystemFileFactory.INSTANCE.getFile(storageGroupSysDir,
MERGING_MODIFICATION_FILE_NAME);
@@ -349,6 +335,19 @@ public class StorageGroupProcessor {
}
tsFileManagement.recover();
+ for (TsFileResource resource : tsFileManagement.getTsFileList(true)) {
+ long partitionNum = resource.getTimePartition();
+ partitionDirectFileVersions.computeIfAbsent(partitionNum, p -> new HashSet<>())
+ .addAll(resource.getHistoricalVersions());
+ updatePartitionFileVersion(partitionNum, Collections.max(resource.getHistoricalVersions()));
+ }
+ for (TsFileResource resource : tsFileManagement.getTsFileList(false)) {
+ long partitionNum = resource.getTimePartition();
+ partitionDirectFileVersions.computeIfAbsent(partitionNum, p -> new HashSet<>())
+ .addAll(resource.getHistoricalVersions());
+ updatePartitionFileVersion(partitionNum, Collections.max(resource.getHistoricalVersions()));
+ }
+
updateLatestFlushedTime();
} catch (IOException | MetadataException e) {
throw new StorageGroupProcessorException(e);
@@ -560,13 +559,6 @@ public class StorageGroupProcessor {
private void recoverTsFiles(List<TsFileResource> tsFiles, boolean isSeq) {
for (int i = 0; i < tsFiles.size(); i++) {
TsFileResource tsFileResource = tsFiles.get(i);
- // this tsfile is not zero level, no need to perform recovery here
- if (LevelCompactionTsFileManagement.getMergeLevel(tsFileResource.getTsFile()) > 0) {
- tsFileResource.setClosed(true);
- tsFileManagement.add(tsFileResource, isSeq);
- continue;
- }
-
long timePartitionId = tsFileResource.getTimePartition();
TsFileRecoverPerformer recoverPerformer = new TsFileRecoverPerformer(
storageGroupName + FILE_NAME_SEPARATOR,
@@ -575,7 +567,15 @@ public class StorageGroupProcessor {
RestorableTsFileIOWriter writer;
try {
- writer = recoverPerformer.recover();
+ // this tsfile is not zero level, no need to perform redo wal
+ if (LevelCompactionTsFileManagement.getMergeLevel(tsFileResource.getTsFile()) > 0) {
+ recoverPerformer.recover(false);
+ tsFileResource.setClosed(true);
+ tsFileManagement.add(tsFileResource, isSeq);
+ continue;
+ } else {
+ writer = recoverPerformer.recover(true);
+ }
} catch (StorageGroupProcessorException e) {
logger.warn("Skip TsFile: {} because of error in recover: ", tsFileResource.getTsFilePath(),
e);
diff --git a/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java b/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
index 4f1be33..b397afc 100644
--- a/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
+++ b/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
@@ -85,7 +85,7 @@ public class TsFileRecoverPerformer {
* writing
*/
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
- public RestorableTsFileIOWriter recover()
+ public RestorableTsFileIOWriter recover(boolean needRedoWal)
throws StorageGroupProcessorException {
File file = FSFactoryProducer.getFSFactory().getFile(filePath);
@@ -122,15 +122,17 @@ public class TsFileRecoverPerformer {
// map must be updated first to avoid duplicated insertion
recoverResourceFromWriter(restorableTsFileIOWriter);
- // redo logs
- redoLogs(restorableTsFileIOWriter);
+ if (needRedoWal) {
+ // redo logs
+ redoLogs(restorableTsFileIOWriter);
- // clean logs
- try {
- MultiFileLogNodeManager.getInstance()
- .deleteNode(logNodePrefix + SystemFileFactory.INSTANCE.getFile(filePath).getName());
- } catch (IOException e) {
- throw new StorageGroupProcessorException(e);
+ // clean logs
+ try {
+ MultiFileLogNodeManager.getInstance()
+ .deleteNode(logNodePrefix + SystemFileFactory.INSTANCE.getFile(filePath).getName());
+ } catch (IOException e) {
+ throw new StorageGroupProcessorException(e);
+ }
}
return restorableTsFileIOWriter;
}
diff --git a/server/src/test/java/org/apache/iotdb/db/writelog/recover/RecoverResourceFromReaderTest.java b/server/src/test/java/org/apache/iotdb/db/writelog/recover/RecoverResourceFromReaderTest.java
index 0283cfd..94d4feb 100644
--- a/server/src/test/java/org/apache/iotdb/db/writelog/recover/RecoverResourceFromReaderTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/writelog/recover/RecoverResourceFromReaderTest.java
@@ -183,7 +183,7 @@ public class RecoverResourceFromReaderTest {
TsFileRecoverPerformer performer = new TsFileRecoverPerformer(logNodePrefix, versionController,
resource, false, false);
- performer.recover().close();
+ performer.recover(true).close();
assertEquals(1, resource.getStartTime("root.sg.device99"));
assertEquals(300, resource.getEndTime("root.sg.device99"));
for (int i = 0; i < 10; i++) {
diff --git a/server/src/test/java/org/apache/iotdb/db/writelog/recover/SeqTsFileRecoverTest.java b/server/src/test/java/org/apache/iotdb/db/writelog/recover/SeqTsFileRecoverTest.java
index aaa080c..d550f60 100644
--- a/server/src/test/java/org/apache/iotdb/db/writelog/recover/SeqTsFileRecoverTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/writelog/recover/SeqTsFileRecoverTest.java
@@ -171,7 +171,7 @@ public class SeqTsFileRecoverTest {
public void testNonLastRecovery() throws StorageGroupProcessorException, IOException {
TsFileRecoverPerformer performer = new TsFileRecoverPerformer(logNodePrefix, versionController,
resource, false, false);
- RestorableTsFileIOWriter writer = performer.recover();
+ RestorableTsFileIOWriter writer = performer.recover(true);
assertFalse(writer.canWrite());
writer.close();
@@ -220,7 +220,7 @@ public class SeqTsFileRecoverTest {
public void testLastRecovery() throws StorageGroupProcessorException, IOException {
TsFileRecoverPerformer performer = new TsFileRecoverPerformer(logNodePrefix, versionController,
resource, false, true);
- RestorableTsFileIOWriter writer = performer.recover();
+ RestorableTsFileIOWriter writer = performer.recover(true);
writer.makeMetadataVisible();
assertEquals(11, writer.getMetadatasForQuery().size());
diff --git a/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java b/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java
index e9c411c..8caa93a 100644
--- a/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java
@@ -180,7 +180,7 @@ public class UnseqTsFileRecoverTest {
public void test() throws StorageGroupProcessorException, IOException {
TsFileRecoverPerformer performer = new TsFileRecoverPerformer(logNodePrefix,
versionController, resource, false, false);
- performer.recover().close();
+ performer.recover(true).close();
assertEquals(1, resource.getStartTime("root.sg.device99"));
assertEquals(300, resource.getEndTime("root.sg.device99"));