You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hx...@apache.org on 2021/06/05 03:32:21 UTC
[iotdb] branch rel/0.12 updated: [To rel/0.12] fix bug caused by
incorrect file name (#3352)
This is an automated email from the ASF dual-hosted git repository.
hxd pushed a commit to branch rel/0.12
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/0.12 by this push:
new 521db04 [To rel/0.12] fix bug caused by incorrect file name (#3352)
521db04 is described below
commit 521db0490a99f14dc5c4c35702734365a0f6cac3
Author: zhanglingzhe0820 <44...@qq.com>
AuthorDate: Sat Jun 5 11:31:52 2021 +0800
[To rel/0.12] fix bug caused by incorrect file name (#3352)
* fix file name bug and update log
* fix cluster test file name bug
---
.../org/apache/iotdb/cluster/common/TestUtils.java | 2 +-
.../cluster/server/member/DataGroupMemberTest.java | 2 +-
.../db/engine/compaction/TsFileManagement.java | 5 +-
.../level/LevelCompactionTsFileManagement.java | 6 +--
.../engine/storagegroup/StorageGroupProcessor.java | 10 ++--
.../db/engine/storagegroup/TsFileResource.java | 55 ++++++++++------------
.../engine/compaction/LevelCompactionLogTest.java | 2 +-
.../compaction/LevelCompactionSelectorTest.java | 3 +-
.../LevelCompactionTsFileManagementTest.java | 4 +-
9 files changed, 42 insertions(+), 47 deletions(-)
diff --git a/cluster/src/test/java/org/apache/iotdb/cluster/common/TestUtils.java b/cluster/src/test/java/org/apache/iotdb/cluster/common/TestUtils.java
index d8aebfc..07861b7 100644
--- a/cluster/src/test/java/org/apache/iotdb/cluster/common/TestUtils.java
+++ b/cluster/src/test/java/org/apache/iotdb/cluster/common/TestUtils.java
@@ -377,7 +377,7 @@ public class TestUtils {
+ File.separator
+ 0
+ File.separator
- + "0-%d-0"
+ + "0-%d-0-0"
+ TsFileConstant.TSFILE_SUFFIX,
i);
if (asHardLink) {
diff --git a/cluster/src/test/java/org/apache/iotdb/cluster/server/member/DataGroupMemberTest.java b/cluster/src/test/java/org/apache/iotdb/cluster/server/member/DataGroupMemberTest.java
index f6d5e5d..05f4c60 100644
--- a/cluster/src/test/java/org/apache/iotdb/cluster/server/member/DataGroupMemberTest.java
+++ b/cluster/src/test/java/org/apache/iotdb/cluster/server/member/DataGroupMemberTest.java
@@ -1016,7 +1016,7 @@ public class DataGroupMemberTest extends BaseMember {
+ File.separator
+ "0-"
+ serialNum
- + "-0.tsfile";
+ + "-0-0.tsfile";
if (asHardLink) {
fileName = fileName + ".0_0";
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/TsFileManagement.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/TsFileManagement.java
index fffbd3b..4ae9f1f 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/TsFileManagement.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/TsFileManagement.java
@@ -108,13 +108,14 @@ public abstract class TsFileManagement {
public abstract void removeAll(List<TsFileResource> tsFileResourceList, boolean sequence);
/** add one TsFile to list */
- public abstract void add(TsFileResource tsFileResource, boolean sequence);
+ public abstract void add(TsFileResource tsFileResource, boolean sequence) throws IOException;
/** add one TsFile to list for recover */
public abstract void addRecover(TsFileResource tsFileResource, boolean sequence);
/** add some TsFiles to list */
- public abstract void addAll(List<TsFileResource> tsFileResourceList, boolean sequence);
+ public abstract void addAll(List<TsFileResource> tsFileResourceList, boolean sequence)
+ throws IOException;
/** is one TsFile contained in list */
public abstract boolean contains(TsFileResource tsFileResource, boolean sequence);
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
index 4f4aa15..f013669 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
@@ -259,7 +259,7 @@ public class LevelCompactionTsFileManagement extends TsFileManagement {
}
@Override
- public void add(TsFileResource tsFileResource, boolean sequence) {
+ public void add(TsFileResource tsFileResource, boolean sequence) throws IOException {
writeLock();
try {
long timePartitionId = tsFileResource.getTimePartition();
@@ -308,7 +308,7 @@ public class LevelCompactionTsFileManagement extends TsFileManagement {
}
@Override
- public void addAll(List<TsFileResource> tsFileResourceList, boolean sequence) {
+ public void addAll(List<TsFileResource> tsFileResourceList, boolean sequence) throws IOException {
writeLock();
try {
for (TsFileResource tsFileResource : tsFileResourceList) {
@@ -676,7 +676,7 @@ public class LevelCompactionTsFileManagement extends TsFileManagement {
i,
toMergeTsFiles.size());
for (TsFileResource toMergeTsFile : toMergeTsFiles) {
- logger.debug(
+ logger.info(
"{} [Compaction] start to merge TsFile {}", storageGroupName, toMergeTsFile);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index 8c2d95d..8beca06 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -690,7 +690,7 @@ public class StorageGroupProcessor {
}
}
- private void recoverTsFiles(List<TsFileResource> tsFiles, boolean isSeq) {
+ private void recoverTsFiles(List<TsFileResource> tsFiles, boolean isSeq) throws IOException {
for (int i = 0; i < tsFiles.size(); i++) {
TsFileResource tsFileResource = tsFiles.get(i);
long timePartitionId = tsFileResource.getTimePartition();
@@ -722,7 +722,7 @@ public class StorageGroupProcessor {
writer =
recoverPerformer.recover(true, this::getWalDirectByteBuffer, this::releaseWalBuffer);
}
- } catch (StorageGroupProcessorException e) {
+ } catch (StorageGroupProcessorException | IOException e) {
logger.warn(
"Skip TsFile: {} because of error in recover: ", tsFileResource.getTsFilePath(), e);
continue;
@@ -2163,7 +2163,7 @@ public class StorageGroupProcessor {
tsfileToBeInserted.getParentFile().getName());
IoTDBDescriptor.getInstance().getConfig().setReadOnly(true);
throw new LoadFileException(e);
- } catch (IllegalPathException | WriteProcessException e) {
+ } catch (IllegalPathException | WriteProcessException | IOException e) {
logger.error(
"Failed to reset last cache when loading file {}", newTsFileResource.getTsFilePath());
throw new LoadFileException(e);
@@ -2270,7 +2270,7 @@ public class StorageGroupProcessor {
tsfileToBeInserted.getParentFile().getName());
IoTDBDescriptor.getInstance().getConfig().setReadOnly(true);
throw new LoadFileException(e);
- } catch (IllegalPathException | WriteProcessException e) {
+ } catch (IllegalPathException | WriteProcessException | IOException e) {
logger.error(
"Failed to reset last cache when loading file {}", newTsFileResource.getTsFilePath());
throw new LoadFileException(e);
@@ -2560,7 +2560,7 @@ public class StorageGroupProcessor {
*/
private boolean loadTsFileByType(
LoadTsFileType type, File syncedTsFile, TsFileResource tsFileResource, long filePartitionId)
- throws LoadFileException, DiskSpaceInsufficientException {
+ throws LoadFileException, DiskSpaceInsufficientException, IOException {
File targetFile;
switch (type) {
case LOAD_UNSEQUENCE:
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
index 5c67e04..16cdb54 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
@@ -32,8 +32,7 @@ import org.apache.iotdb.db.service.UpgradeSevice;
import org.apache.iotdb.db.utils.FilePathUtils;
import org.apache.iotdb.db.utils.TestOnly;
import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
-import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
-import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
+import org.apache.iotdb.tsfile.file.metadata.*;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
@@ -51,21 +50,9 @@ import java.io.OutputStream;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Random;
-import java.util.Set;
-
-import static org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SEPARATOR;
-import static org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SUFFIX_INDEX;
-import static org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SUFFIX_MERGECNT_INDEX;
-import static org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SUFFIX_SEPARATOR;
-import static org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SUFFIX_TIME_INDEX;
-import static org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SUFFIX_UNSEQMERGECNT_INDEX;
-import static org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SUFFIX_VERSION_INDEX;
+import java.util.*;
+
+import static org.apache.iotdb.db.conf.IoTDBConstant.*;
import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.TSFILE_SUFFIX;
@SuppressWarnings("java:S1135") // ignore todos
@@ -772,25 +759,31 @@ public class TsFileResource {
+ TSFILE_SUFFIX;
}
- public static TsFileName getTsFileName(String FileName) {
- String[] fileName =
- FileName.split(FILE_NAME_SUFFIX_SEPARATOR)[FILE_NAME_SUFFIX_INDEX].split(
+ public static TsFileName getTsFileName(String fileName) throws IOException {
+ String[] fileNameParts =
+ fileName.split(FILE_NAME_SUFFIX_SEPARATOR)[FILE_NAME_SUFFIX_INDEX].split(
FILE_NAME_SEPARATOR);
- TsFileName tsFileName =
- new TsFileName(
- Long.parseLong(fileName[FILE_NAME_SUFFIX_TIME_INDEX]),
- Long.parseLong(fileName[FILE_NAME_SUFFIX_VERSION_INDEX]),
- Integer.parseInt(fileName[FILE_NAME_SUFFIX_MERGECNT_INDEX]),
- Integer.parseInt(fileName[FILE_NAME_SUFFIX_UNSEQMERGECNT_INDEX]));
- return tsFileName;
+ if (fileNameParts.length != 4) {
+ throw new IOException("tsfile file name format is incorrect:" + fileName);
+ }
+ try {
+ return new TsFileName(
+ Long.parseLong(fileNameParts[FILE_NAME_SUFFIX_TIME_INDEX]),
+ Long.parseLong(fileNameParts[FILE_NAME_SUFFIX_VERSION_INDEX]),
+ Integer.parseInt(fileNameParts[FILE_NAME_SUFFIX_MERGECNT_INDEX]),
+ Integer.parseInt(fileNameParts[FILE_NAME_SUFFIX_UNSEQMERGECNT_INDEX]));
+ } catch (NumberFormatException e) {
+ throw new IOException("tsfile file name format is incorrect:" + fileName);
+ }
}
- public static int getMergeLevel(String fileName) {
+ public static int getMergeLevel(String fileName) throws IOException {
TsFileName tsFileName = getTsFileName(fileName);
return tsFileName.mergeCnt;
}
- public static TsFileResource modifyTsFileNameUnseqMergCnt(TsFileResource tsFileResource) {
+ public static TsFileResource modifyTsFileNameUnseqMergCnt(TsFileResource tsFileResource)
+ throws IOException {
File tsFile = tsFileResource.getTsFile();
String path = tsFile.getParent();
TsFileName tsFileName = getTsFileName(tsFileResource.getTsFile().getName());
@@ -809,7 +802,7 @@ public class TsFileResource {
return tsFileResource;
}
- public static File modifyTsFileNameUnseqMergCnt(File tsFile) {
+ public static File modifyTsFileNameUnseqMergCnt(File tsFile) throws IOException {
String path = tsFile.getParent();
TsFileName tsFileName = getTsFileName(tsFile.getName());
tsFileName.setUnSeqMergeCnt(tsFileName.getUnSeqMergeCnt() + 1);
@@ -825,7 +818,7 @@ public class TsFileResource {
+ TSFILE_SUFFIX);
}
- public static File modifyTsFileNameMergeCnt(File tsFile) {
+ public static File modifyTsFileNameMergeCnt(File tsFile) throws IOException {
String path = tsFile.getParent();
TsFileName tsFileName = getTsFileName(tsFile.getName());
tsFileName.setMergeCnt(tsFileName.getMergeCnt() + 1);
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionLogTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionLogTest.java
index 2170fbd..f599f05 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionLogTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionLogTest.java
@@ -59,7 +59,7 @@ public class LevelCompactionLogTest extends LevelCompactionTest {
}
@Test
- public void testCompactionLog() {
+ public void testCompactionLog() throws IOException {
LevelCompactionTsFileManagement levelCompactionTsFileManagement =
new LevelCompactionTsFileManagement(COMPACTION_TEST_SG, tempSGDir.getPath());
levelCompactionTsFileManagement.addAll(seqResources, true);
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionSelectorTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionSelectorTest.java
index baa2502..4237e33 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionSelectorTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionSelectorTest.java
@@ -59,7 +59,8 @@ public class LevelCompactionSelectorTest extends LevelCompactionTest {
/** just compaction once */
@Test
- public void testCompactionSelector() throws NoSuchFieldException, IllegalAccessException {
+ public void testCompactionSelector()
+ throws NoSuchFieldException, IllegalAccessException, IOException {
LevelCompactionTsFileManagement levelCompactionTsFileManagement =
new LevelCompactionTsFileManagement(COMPACTION_TEST_SG, tempSGDir.getPath());
levelCompactionTsFileManagement.addAll(seqResources, true);
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionTsFileManagementTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionTsFileManagementTest.java
index 7375bad..20602c7 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionTsFileManagementTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionTsFileManagementTest.java
@@ -61,7 +61,7 @@ public class LevelCompactionTsFileManagementTest extends LevelCompactionTest {
/** just compaction once */
@Test
- public void testAddRemoveAndIterator() {
+ public void testAddRemoveAndIterator() throws IOException {
LevelCompactionTsFileManagement levelCompactionTsFileManagement =
new LevelCompactionTsFileManagement(COMPACTION_TEST_SG, tempSGDir.getPath());
for (TsFileResource tsFileResource : seqResources) {
@@ -158,7 +158,7 @@ public class LevelCompactionTsFileManagementTest extends LevelCompactionTest {
}
@Test
- public void testIteratorRemove() {
+ public void testIteratorRemove() throws IOException {
LevelCompactionTsFileManagement levelCompactionTsFileManagement =
new LevelCompactionTsFileManagement(COMPACTION_TEST_SG, tempSGDir.getPath());
for (TsFileResource tsFileResource : seqResources) {