You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2024/08/14 04:14:24 UTC
(iotdb) 03/03: add more tests and fix wal with broken v3 tsfile
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch adapt_wal_1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 4003a3448cb1bf9c5991145bffa7d923f7d85ff3
Author: HTHou <hh...@outlook.com>
AuthorDate: Wed Aug 14 12:13:53 2024 +0800
add more tests and fix wal with broken v3 tsfile
---
.../file/AbstractTsFileRecoverPerformer.java | 5 ++++
.../wal/recover/WALRecoverManagerTest.java | 32 +++++++++++++++++----
.../src/test/resources/oldwal/1723544967972-1-0-0 | Bin 0 -> 237 bytes
.../resources/oldwal/{2147483647 => }/_0-0-0.wal | Bin
.../oldwal/{2147483647 => }/_0.checkpoint | Bin
.../resources/oldwal/{2147483647 => }/_1-0-0.wal | Bin
.../resources/oldwal/{2147483647 => }/_2-0-0.wal | Bin
.../resources/oldwal/{2147483647 => }/_3-0-1.wal | Bin
8 files changed, 31 insertions(+), 6 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/AbstractTsFileRecoverPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/AbstractTsFileRecoverPerformer.java
index 0880697b02c..cec93e0ead4 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/AbstractTsFileRecoverPerformer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/AbstractTsFileRecoverPerformer.java
@@ -35,6 +35,7 @@ import org.slf4j.LoggerFactory;
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
+import java.nio.file.Files;
/** This class is used to help recover TsFile. */
public abstract class AbstractTsFileRecoverPerformer implements Closeable {
@@ -85,6 +86,10 @@ public abstract class AbstractTsFileRecoverPerformer implements Closeable {
boolean result = tsFile.delete();
logger.warn(
"TsFile {} is incompatible. Try to delete it and delete result is {}", tsFile, result);
+ // if the broken TsFile is v3, we can recover the all data from wal
+ // to support it, we can regenerate an empty file here
+ Files.createFile(tsFile.toPath());
+ writer = new RestorableTsFileIOWriter(tsFile);
throw new DataRegionException(e);
} catch (IOException e) {
throw new DataRegionException(e);
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverManagerTest.java
index b2d462e10f3..50233122779 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverManagerTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverManagerTest.java
@@ -354,19 +354,30 @@ public class WALRecoverManagerTest {
}
@Test
- public void testMemTableSnapshotFromOldVersion() throws Exception {
+ public void testRecoverOldWalWithEmptyTsFile() throws Exception {
// old version of wal is generated by prepareCheckpointAndWALFileForSnapshot() in v1.3
- String oldWalPathStr =
- this.getClass().getClassLoader().getResource("oldwal/2147483647").getFile();
+ String oldWalPathStr = this.getClass().getClassLoader().getResource("oldwal").getFile();
File oldWalFileDir = new File(oldWalPathStr);
FileUtils.copyDir(oldWalFileDir, new File(WAL_NODE_FOLDER));
walBuffer = new WALBuffer(WAL_NODE_IDENTIFIER, WAL_NODE_FOLDER);
checkpointManager = walBuffer.getCheckpointManager();
WALRecoverManager.getInstance().clear();
- recoverFromOldWalAndCheck();
+ recoverFromOldWalAndCheck(false);
}
- private void recoverFromOldWalAndCheck() throws Exception {
+ @Test
+ public void testRecoverOldWalWithBrokenTsFile() throws Exception {
+ // old version of wal is generated by prepareCheckpointAndWALFileForSnapshot() in v1.3
+ String oldWalPathStr = this.getClass().getClassLoader().getResource("oldwal").getFile();
+ File oldWalFileDir = new File(oldWalPathStr);
+ FileUtils.copyDir(oldWalFileDir, new File(WAL_NODE_FOLDER));
+ walBuffer = new WALBuffer(WAL_NODE_IDENTIFIER, WAL_NODE_FOLDER);
+ checkpointManager = walBuffer.getCheckpointManager();
+ WALRecoverManager.getInstance().clear();
+ recoverFromOldWalAndCheck(true);
+ }
+
+ private void recoverFromOldWalAndCheck(boolean withBrokenTsFile) throws Exception {
// prepare tsFiles
List<WALRecoverListener> recoverListeners = new ArrayList<>();
@@ -374,7 +385,16 @@ public class WALRecoverManagerTest {
File fileWithWALDir = new File(FILE_WITH_WAL_NAME).getParentFile();
Files.createDirectory(fileWithWALDir.toPath());
File fileWithWAL = new File(fileWithWALDir, "1723544967972-1-0-0.tsfile");
- Files.createFile(fileWithWAL.toPath());
+ ;
+ if (withBrokenTsFile) {
+ // copy a broken TsFileV3
+ String oldWalPathStr = this.getClass().getClassLoader().getResource("oldwal").getFile();
+ File oldWalFileDir = new File(oldWalPathStr);
+ FileUtils.copyFile(new File(oldWalFileDir, "1723544967972-1-0-0"), fileWithWAL);
+ } else {
+ // create an empty tsfile
+ Files.createFile(fileWithWAL.toPath());
+ }
tsFileWithWALResource = new TsFileResource(fileWithWAL);
UnsealedTsFileRecoverPerformer recoverPerformer =
new UnsealedTsFileRecoverPerformer(
diff --git a/iotdb-core/datanode/src/test/resources/oldwal/1723544967972-1-0-0 b/iotdb-core/datanode/src/test/resources/oldwal/1723544967972-1-0-0
new file mode 100644
index 00000000000..f41e64de04b
Binary files /dev/null and b/iotdb-core/datanode/src/test/resources/oldwal/1723544967972-1-0-0 differ
diff --git a/iotdb-core/datanode/src/test/resources/oldwal/2147483647/_0-0-0.wal b/iotdb-core/datanode/src/test/resources/oldwal/_0-0-0.wal
similarity index 100%
rename from iotdb-core/datanode/src/test/resources/oldwal/2147483647/_0-0-0.wal
rename to iotdb-core/datanode/src/test/resources/oldwal/_0-0-0.wal
diff --git a/iotdb-core/datanode/src/test/resources/oldwal/2147483647/_0.checkpoint b/iotdb-core/datanode/src/test/resources/oldwal/_0.checkpoint
similarity index 100%
rename from iotdb-core/datanode/src/test/resources/oldwal/2147483647/_0.checkpoint
rename to iotdb-core/datanode/src/test/resources/oldwal/_0.checkpoint
diff --git a/iotdb-core/datanode/src/test/resources/oldwal/2147483647/_1-0-0.wal b/iotdb-core/datanode/src/test/resources/oldwal/_1-0-0.wal
similarity index 100%
rename from iotdb-core/datanode/src/test/resources/oldwal/2147483647/_1-0-0.wal
rename to iotdb-core/datanode/src/test/resources/oldwal/_1-0-0.wal
diff --git a/iotdb-core/datanode/src/test/resources/oldwal/2147483647/_2-0-0.wal b/iotdb-core/datanode/src/test/resources/oldwal/_2-0-0.wal
similarity index 100%
rename from iotdb-core/datanode/src/test/resources/oldwal/2147483647/_2-0-0.wal
rename to iotdb-core/datanode/src/test/resources/oldwal/_2-0-0.wal
diff --git a/iotdb-core/datanode/src/test/resources/oldwal/2147483647/_3-0-1.wal b/iotdb-core/datanode/src/test/resources/oldwal/_3-0-1.wal
similarity index 100%
rename from iotdb-core/datanode/src/test/resources/oldwal/2147483647/_3-0-1.wal
rename to iotdb-core/datanode/src/test/resources/oldwal/_3-0-1.wal