You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2021/09/21 14:20:42 UTC
[iotdb] branch rel/0.12 updated: [To rel/0.12][IOTDB-1578] Set
unsequnce when loading TsFile with the same establish time (#3983)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch rel/0.12
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/0.12 by this push:
new 6cf9dbd [To rel/0.12][IOTDB-1578] Set unsequnce when loading TsFile with the same establish time (#3983)
6cf9dbd is described below
commit 6cf9dbda30653ccd0151d5b4297f41ef1346701c
Author: yschengzi <87...@users.noreply.github.com>
AuthorDate: Tue Sep 21 22:20:14 2021 +0800
[To rel/0.12][IOTDB-1578] Set unsequnce when loading TsFile with the same establish time (#3983)
---
.../engine/storagegroup/StorageGroupProcessor.java | 94 ++++++++++------------
.../db/integration/IoTDBLoadExternalTsfileIT.java | 27 ++++---
2 files changed, 59 insertions(+), 62 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index 3648fb9..c6399ef 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -2250,27 +2250,14 @@ public class StorageGroupProcessor {
tsFileManagement.getTsFileListByTimePartition(true, newFilePartitionId);
int insertPos = findInsertionPosition(newTsFileResource, sequenceList);
- String newFileName, renameInfo;
- LoadTsFileType tsFileType;
-
- // loading tsfile by type
- if (insertPos == POS_OVERLAP) {
- newFileName =
- getNewTsFileName(
- System.currentTimeMillis(),
- getAndSetNewVersion(newFilePartitionId, newTsFileResource),
- 0,
- 0);
- renameInfo = IoTDBConstant.UNSEQUENCE_FLODER_NAME;
- tsFileType = LoadTsFileType.LOAD_UNSEQUENCE;
- newTsFileResource.setSeq(false);
- } else {
- // check whether the file name needs to be renamed.
- newFileName = getFileNameForSequenceLoadingFile(insertPos, newTsFileResource, sequenceList);
- renameInfo = IoTDBConstant.SEQUENCE_FLODER_NAME;
- tsFileType = LoadTsFileType.LOAD_SEQUENCE;
- newTsFileResource.setSeq(true);
- }
+ LoadTsFileType tsFileType = getLoadingTsFileType(insertPos, sequenceList);
+ String renameInfo =
+ (tsFileType == LoadTsFileType.LOAD_SEQUENCE)
+ ? IoTDBConstant.SEQUENCE_FLODER_NAME
+ : IoTDBConstant.UNSEQUENCE_FLODER_NAME;
+ newTsFileResource.setSeq(tsFileType == LoadTsFileType.LOAD_SEQUENCE);
+ String newFileName =
+ getLoadingTsFileName(tsFileType, insertPos, newTsFileResource, sequenceList);
if (!newFileName.equals(tsfileToBeInserted.getName())) {
logger.info(
@@ -2323,6 +2310,26 @@ public class StorageGroupProcessor {
return Math.max(oldVersion, newVersion);
}
+ private Long getTsFileResourceEstablishTime(TsFileResource tsFileResource) {
+ String tsFileName = tsFileResource.getTsFile().getName();
+ return Long.parseLong(tsFileName.split(FILE_NAME_SEPARATOR)[0]);
+ }
+
+ private LoadTsFileType getLoadingTsFileType(int insertPos, List<TsFileResource> sequenceList) {
+ if (insertPos == POS_OVERLAP) {
+ return LoadTsFileType.LOAD_UNSEQUENCE;
+ }
+ if (insertPos == sequenceList.size() - 1) {
+ return LoadTsFileType.LOAD_SEQUENCE;
+ }
+ long preTime =
+ (insertPos == -1) ? 0 : getTsFileResourceEstablishTime(sequenceList.get(insertPos));
+ long subsequenceTime = getTsFileResourceEstablishTime(sequenceList.get(insertPos + 1));
+ return preTime == subsequenceTime
+ ? LoadTsFileType.LOAD_UNSEQUENCE
+ : LoadTsFileType.LOAD_SEQUENCE;
+ }
+
/**
* Find the position of "newTsFileResource" in the sequence files if it can be inserted into them.
*
@@ -2501,41 +2508,24 @@ public class StorageGroupProcessor {
* 1]
* @return appropriate filename
*/
- private String getFileNameForSequenceLoadingFile(
- int insertIndex, TsFileResource newTsFileResource, List<TsFileResource> sequenceList)
- throws LoadFileException {
- int sequenceListLength = sequenceList.size();
+ private String getLoadingTsFileName(
+ LoadTsFileType tsFileType,
+ int insertIndex,
+ TsFileResource newTsFileResource,
+ List<TsFileResource> sequenceList) {
long timePartitionId = newTsFileResource.getTimePartition();
- long preTime, subsequenceTime;
-
- if (insertIndex == -1) {
- preTime = 0L;
- } else {
- String preName = sequenceList.get(insertIndex).getTsFile().getName();
- preTime = Long.parseLong(preName.split(FILE_NAME_SEPARATOR)[0]);
- }
- if (insertIndex == sequenceListLength - 1) {
- subsequenceTime = preTime + ((System.currentTimeMillis() - preTime) << 1);
- } else {
- String subsequenceName = sequenceList.get(insertIndex + 1).getTsFile().getName();
- subsequenceTime = Long.parseLong(subsequenceName.split(FILE_NAME_SEPARATOR)[0]);
+ if (tsFileType == LoadTsFileType.LOAD_UNSEQUENCE || insertIndex == sequenceList.size() - 1) {
+ return getNewTsFileName(
+ System.currentTimeMillis(),
+ getAndSetNewVersion(timePartitionId, newTsFileResource),
+ 0,
+ 0);
}
+ long preTime =
+ (insertIndex == -1) ? 0 : getTsFileResourceEstablishTime(sequenceList.get(insertIndex));
+ long subsequenceTime = getTsFileResourceEstablishTime(sequenceList.get(insertIndex + 1));
long meanTime = preTime + ((subsequenceTime - preTime) >> 1);
- if (insertIndex != sequenceListLength - 1 && meanTime == subsequenceTime) {
- throw new LoadFileException(
- "can not load TsFile because of can not find suitable location, "
- + "preName: "
- + preTime
- + " subsequenceTime: "
- + subsequenceTime
- + " insertIndex: "
- + insertIndex
- + " partition: "
- + timePartitionId
- + " sequencePartitionListLength: "
- + sequenceListLength);
- }
return getNewTsFileName(
meanTime, getAndSetNewVersion(timePartitionId, newTsFileResource), 0, 0);
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBLoadExternalTsfileIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBLoadExternalTsfileIT.java
index 9a6c9a8..1479a09 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBLoadExternalTsfileIT.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBLoadExternalTsfileIT.java
@@ -459,18 +459,25 @@ public class IoTDBLoadExternalTsfileIT {
.getUnSequenceFileList()
.size());
if (config.getTimeIndexLevel().equals(TimeIndexLevel.DEVICE_TIME_INDEX)) {
- assertEquals(
- 1,
- StorageEngine.getInstance()
+ if (StorageEngine.getInstance()
.getProcessor(new PartialPath("root.test"))
.getUnSequenceFileList()
- .size());
- assertEquals(
- 3,
- StorageEngine.getInstance()
- .getProcessor(new PartialPath("root.test"))
- .getSequenceFileTreeSet()
- .size());
+ .size()
+ == 1) {
+ assertEquals(
+ 3,
+ StorageEngine.getInstance()
+ .getProcessor(new PartialPath("root.test"))
+ .getSequenceFileTreeSet()
+ .size());
+ } else {
+ assertEquals(
+ 2,
+ StorageEngine.getInstance()
+ .getProcessor(new PartialPath("root.test"))
+ .getSequenceFileTreeSet()
+ .size());
+ }
} else if (config.getTimeIndexLevel().equals(TimeIndexLevel.FILE_TIME_INDEX)) {
assertEquals(
2,