You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2021/05/13 06:37:55 UTC
[iotdb] branch upgrade_but1 created (now 4e2d272)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a change to branch upgrade_but1
in repository https://gitbox.apache.org/repos/asf/iotdb.git.
at 4e2d272 [To rel/0.12] Fix upgrade tool cannot load old tsfile if time partition enabled in 0.11 (#3154)
This branch includes the following new commits:
new f8c1fca [To rel/0.12] fix versionInfo NPE when query upgrading 0.11 tsfile (#3178)
new 4e2d272 [To rel/0.12] Fix upgrade tool cannot load old tsfile if time partition enabled in 0.11 (#3154)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 02/02: [To rel/0.12] Fix upgrade tool cannot load old
tsfile if time partition enabled in 0.11 (#3154)
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch upgrade_but1
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 4e2d2724318c71dbd9f8cb9fa54c697aa59b9adf
Author: Haonan <hh...@outlook.com>
AuthorDate: Thu May 13 14:06:40 2021 +0800
[To rel/0.12] Fix upgrade tool cannot load old tsfile if time partition enabled in 0.11 (#3154)
---
.../org/apache/iotdb/db/conf/IoTDBConfigCheck.java | 28 +++++++++++++++++++++-
1 file changed, 27 insertions(+), 1 deletion(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java
index ffbba6e..992ef36 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java
@@ -241,6 +241,14 @@ public class IoTDBConfigCheck {
System.exit(-1);
}
+ // virtual storage group num can only set to 1 when upgrading from old version
+ if (!virtualStorageGroupNum.equals("1")) {
+ logger.error(
+ "virtual storage group num cannot set to {} when upgrading from old version, "
+ + "please set to 1 and restart",
+ virtualStorageGroupNum);
+ System.exit(-1);
+ }
try (FileOutputStream tmpFOS = new FileOutputStream(tmpPropertiesFile.toString())) {
properties.setProperty(PARTITION_INTERVAL_STRING, String.valueOf(partitionInterval));
properties.setProperty(TSFILE_FILE_SYSTEM_STRING, tsfileFileSystem);
@@ -432,6 +440,16 @@ public class IoTDBConfigCheck {
if (!storageGroup.isDirectory()) {
continue;
}
+ // create virtual storage group folder 0
+ File virtualStorageGroupDir = fsFactory.getFile(storageGroup, "0");
+ if (virtualStorageGroupDir.mkdirs()) {
+ logger.info(
+ "virtual storage directory {} doesn't exist, create it",
+ virtualStorageGroupDir.getPath());
+ } else if (!virtualStorageGroupDir.exists()) {
+ logger.error(
+ "Create virtual storage directory {} failed", virtualStorageGroupDir.getPath());
+ }
for (File partitionDir : storageGroup.listFiles()) {
if (!partitionDir.isDirectory()) {
continue;
@@ -450,7 +468,8 @@ public class IoTDBConfigCheck {
if (oldTsfileArray.length + oldResourceFileArray.length + oldModificationFileArray.length
!= 0) {
// create upgrade directory if not exist
- File upgradeFolder = fsFactory.getFile(partitionDir, IoTDBConstant.UPGRADE_FOLDER_NAME);
+ File upgradeFolder =
+ fsFactory.getFile(virtualStorageGroupDir, IoTDBConstant.UPGRADE_FOLDER_NAME);
if (upgradeFolder.mkdirs()) {
logger.info("Upgrade Directory {} doesn't exist, create it", upgradeFolder.getPath());
} else if (!upgradeFolder.exists()) {
@@ -475,6 +494,13 @@ public class IoTDBConfigCheck {
}
}
}
+ if (partitionDir.listFiles().length == 0) {
+ try {
+ Files.delete(partitionDir.toPath());
+ } catch (IOException e) {
+ logger.error("Delete {} failed", partitionDir);
+ }
+ }
}
}
}
[iotdb] 01/02: [To rel/0.12] fix versionInfo NPE when query
upgrading 0.11 tsfile (#3178)
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch upgrade_but1
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit f8c1fcaf80f57e65f55868ff6517437ae3b88490
Author: Haonan <hh...@outlook.com>
AuthorDate: Thu May 13 13:52:52 2021 +0800
[To rel/0.12] fix versionInfo NPE when query upgrading 0.11 tsfile (#3178)
---
.../apache/iotdb/tsfile/v2/file/metadata/TsFileMetadataV2.java | 9 +++++----
.../apache/iotdb/tsfile/v2/read/TsFileSequenceReaderForV2.java | 10 +++++-----
2 files changed, 10 insertions(+), 9 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/v2/file/metadata/TsFileMetadataV2.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/v2/file/metadata/TsFileMetadataV2.java
index 9825a5c..2ebd12e 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/v2/file/metadata/TsFileMetadataV2.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/v2/file/metadata/TsFileMetadataV2.java
@@ -24,6 +24,7 @@ import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
import java.util.List;
public class TsFileMetadataV2 {
@@ -34,12 +35,12 @@ public class TsFileMetadataV2 {
* deserialize data from the buffer.
*
* @param buffer -buffer use to deserialize
- * @return -a instance of TsFileMetaData
+ * @return -a pair of TsFileMetaData and VersionInfo
*/
- public static TsFileMetadata deserializeFrom(
- ByteBuffer buffer, List<Pair<Long, Long>> versionInfo) {
+ public static Pair<TsFileMetadata, List<Pair<Long, Long>>> deserializeFrom(ByteBuffer buffer) {
TsFileMetadata fileMetaData = new TsFileMetadata();
+ List<Pair<Long, Long>> versionInfo = new ArrayList<>();
// metadataIndex
fileMetaData.setMetadataIndex(MetadataIndexNodeV2.deserializeFrom(buffer));
// totalChunkNum
@@ -70,6 +71,6 @@ public class TsFileMetadataV2 {
BloomFilter.buildBloomFilter(bytes, filterSize, hashFunctionSize));
}
- return fileMetaData;
+ return new Pair<>(fileMetaData, versionInfo);
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/v2/read/TsFileSequenceReaderForV2.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/v2/read/TsFileSequenceReaderForV2.java
index d07007b..117da82 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/v2/read/TsFileSequenceReaderForV2.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/v2/read/TsFileSequenceReaderForV2.java
@@ -145,11 +145,11 @@ public class TsFileSequenceReaderForV2 extends TsFileSequenceReader implements A
*/
@Override
public TsFileMetadata readFileMetadata() throws IOException {
- if (tsFileMetaData == null) {
- versionInfo = new ArrayList<>();
- tsFileMetaData =
- TsFileMetadataV2.deserializeFrom(
- readData(fileMetadataPos, fileMetadataSize), versionInfo);
+ if (tsFileMetaData == null || versionInfo == null) {
+ Pair<TsFileMetadata, List<Pair<Long, Long>>> pair =
+ TsFileMetadataV2.deserializeFrom(readData(fileMetadataPos, fileMetadataSize));
+ tsFileMetaData = pair.left;
+ versionInfo = pair.right;
}
return tsFileMetaData;
}