You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by su...@apache.org on 2021/05/15 13:03:06 UTC
[iotdb] branch rel/0.11 updated: [To rel/0.11] Unseq compaction
throws wrong exception if some paths are not in the file (#3197)
This is an automated email from the ASF dual-hosted git repository.
sunzesong pushed a commit to branch rel/0.11
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/0.11 by this push:
new 5c072a1 [To rel/0.11] Unseq compaction throws wrong exception if some paths are not in the file (#3197)
5c072a1 is described below
commit 5c072a198ea0ee82c42963ebd5bcded128ac20de
Author: zhanglingzhe0820 <44...@qq.com>
AuthorDate: Sat May 15 21:02:33 2021 +0800
[To rel/0.11] Unseq compaction throws wrong exception if some paths are not in the file (#3197)
* Unseq compaction throws wrong exception if some paths are not in the file
* fix ci
Co-authored-by: Zesong Sun <sz...@mails.tsinghua.edu.cn>
Co-authored-by: zhanglingzhe <su...@foxmail.com>
---
.../db/engine/cache/TimeSeriesMetadataCache.java | 2 +-
.../iotdb/db/engine/merge/manage/MergeResource.java | 2 +-
.../java/org/apache/iotdb/db/utils/MergeUtils.java | 4 ++--
.../iotdb/tsfile/read/TsFileSequenceReader.java | 21 +++++++++++++++------
.../tsfile/v1/read/TsFileSequenceReaderForV1.java | 5 -----
5 files changed, 19 insertions(+), 15 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
index 9aef0db..c738cf1 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
@@ -122,7 +122,7 @@ public class TimeSeriesMetadataCache {
.contains(key.device + IoTDBConstant.PATH_SEPARATOR + key.measurement)) {
return null;
}
- return reader.readTimeseriesMetadata(new Path(key.device, key.measurement));
+ return reader.readTimeseriesMetadata(new Path(key.device, key.measurement), false);
}
cacheRequestNum.incrementAndGet();
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeResource.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeResource.java
index bb4d069..d0d67e5 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeResource.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeResource.java
@@ -135,7 +135,7 @@ public class MergeResource {
public List<ChunkMetadata> queryChunkMetadata(PartialPath path, TsFileResource seqFile)
throws IOException {
TsFileSequenceReader sequenceReader = getFileReader(seqFile);
- return sequenceReader.getChunkMetadataList(path);
+ return sequenceReader.getChunkMetadataList(path, true);
}
/**
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/MergeUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/MergeUtils.java
index c9070f4..3bd14e6 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/MergeUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/MergeUtils.java
@@ -138,7 +138,7 @@ public class MergeUtils {
List<Path> paths = collectFileSeries(sequenceReader);
for (Path path : paths) {
- List<ChunkMetadata> chunkMetadataList = sequenceReader.getChunkMetadataList(path);
+ List<ChunkMetadata> chunkMetadataList = sequenceReader.getChunkMetadataList(path, true);
totalChunkNum += chunkMetadataList.size();
maxChunkNum = chunkMetadataList.size() > maxChunkNum ? chunkMetadataList.size() : maxChunkNum;
}
@@ -184,7 +184,7 @@ public class MergeUtils {
throws IOException {
for (int i = 0; i < paths.size(); i++) {
PartialPath path = paths.get(i);
- List<ChunkMetadata> metaDataList = tsFileReader.getChunkMetadataList(path);
+ List<ChunkMetadata> metaDataList = tsFileReader.getChunkMetadataList(path, true);
if (metaDataList.isEmpty()) {
continue;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index 8bb293e..f1f254a 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -315,12 +315,16 @@ public class TsFileSequenceReader implements AutoCloseable {
return deviceMetadata;
}
- public TimeseriesMetadata readTimeseriesMetadata(Path path) throws IOException {
+ public TimeseriesMetadata readTimeseriesMetadata(Path path, boolean ignoreNotExists)
+ throws IOException {
readFileMetadata();
MetadataIndexNode deviceMetadataIndexNode = tsFileMetaData.getMetadataIndex();
Pair<MetadataIndexEntry, Long> metadataIndexPair =
getMetadataAndEndOffset(deviceMetadataIndexNode, path.getDevice(), INTERNAL_DEVICE, true);
if (metadataIndexPair == null) {
+ if (ignoreNotExists) {
+ return null;
+ }
throw new IOException("Device {" + path.getDevice() + "} is not in tsFileMetaData");
}
ByteBuffer buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right);
@@ -361,8 +365,8 @@ public class TsFileSequenceReader implements AutoCloseable {
}
/**
- * Find the leaf node that contains path, return all the sensors in that leaf node which are also
- * in allSensors set
+ * getChunkMetadataList Find the leaf node that contains path, return all the sensors in that leaf
+ * node which are also in allSensors set
*/
public List<TimeseriesMetadata> readTimeseriesMetadata(Path path, Set<String> allSensors)
throws IOException {
@@ -1144,13 +1148,14 @@ public class TsFileSequenceReader implements AutoCloseable {
}
/**
- * get ChunkMetaDatas of given path
+ * get ChunkMetaDatas of given path, and throw exception if path not exists
*
* @param path timeseries path
* @return List of ChunkMetaData
*/
- public List<ChunkMetadata> getChunkMetadataList(Path path) throws IOException {
- TimeseriesMetadata timeseriesMetaData = readTimeseriesMetadata(path);
+ public List<ChunkMetadata> getChunkMetadataList(Path path, boolean ignoreNotExists)
+ throws IOException {
+ TimeseriesMetadata timeseriesMetaData = readTimeseriesMetadata(path, ignoreNotExists);
if (timeseriesMetaData == null) {
return Collections.emptyList();
}
@@ -1159,6 +1164,10 @@ public class TsFileSequenceReader implements AutoCloseable {
return chunkMetadataList;
}
+ public List<ChunkMetadata> getChunkMetadataList(Path path) throws IOException {
+ return getChunkMetadataList(path, false);
+ }
+
/**
* get ChunkMetaDatas in given TimeseriesMetaData
*
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/v1/read/TsFileSequenceReaderForV1.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/v1/read/TsFileSequenceReaderForV1.java
index 91990be..26f541b 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/v1/read/TsFileSequenceReaderForV1.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/v1/read/TsFileSequenceReaderForV1.java
@@ -249,11 +249,6 @@ public class TsFileSequenceReaderForV1 extends TsFileSequenceReader {
}
@Override
- public TimeseriesMetadata readTimeseriesMetadata(Path path) throws IOException {
- return getTimeseriesMetadataFromOldFile(path);
- }
-
- @Override
public List<TimeseriesMetadata> readTimeseriesMetadata(String device, Set<String> measurements)
throws IOException {
return getTimeseriesMetadataFromOldFile(device, measurements);