You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2020/04/14 08:05:16 UTC
[incubator-iotdb] 01/01: add metaOffset in TsFileMetadata
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch add_meta_marker_offset
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 839dd1b39ad10f9336beb372d24ca533aea5b824
Author: qiaojialin <64...@qq.com>
AuthorDate: Tue Apr 14 16:04:58 2020 +0800
add metaOffset in TsFileMetadata
---
.../iotdb/tsfile/file/metadata/TsFileMetadata.java | 20 ++++++++++++++++++--
.../tsfile/write/writer/ForceAppendTsFileWriter.java | 20 +++-----------------
.../iotdb/tsfile/write/writer/TsFileIOWriter.java | 3 +++
3 files changed, 24 insertions(+), 19 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java
index feb147e..3195c08 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java
@@ -54,6 +54,8 @@ public class TsFileMetadata {
// offset -> version
private List<Pair<Long, Long>> versionInfo;
+ // offset of MetaMarker.SEPARATOR
+ private long metaOffset;
/**
* deserialize data from the buffer.
@@ -90,6 +92,9 @@ public class TsFileMetadata {
}
fileMetaData.setVersionInfo(versionInfo);
+ // metaOffset
+ long metaOffset = ReadWriteIOUtils.readLong(buffer);
+ fileMetaData.setMetaOffset(metaOffset);
// read bloom filter
if (buffer.hasRemaining()) {
@@ -134,10 +139,13 @@ public class TsFileMetadata {
// versionInfo
byteLen += ReadWriteIOUtils.write(versionInfo.size(), outputStream);
for (Pair<Long, Long> versionPair : versionInfo) {
- byteLen +=ReadWriteIOUtils.write(versionPair.left, outputStream);
- byteLen +=ReadWriteIOUtils.write(versionPair.right, outputStream);
+ byteLen += ReadWriteIOUtils.write(versionPair.left, outputStream);
+ byteLen += ReadWriteIOUtils.write(versionPair.right, outputStream);
}
+ // metaOffset
+ byteLen += ReadWriteIOUtils.write(metaOffset, outputStream);
+
return byteLen;
}
@@ -192,6 +200,14 @@ public class TsFileMetadata {
this.invalidChunkNum = invalidChunkNum;
}
+ public long getMetaOffset() {
+ return metaOffset;
+ }
+
+ public void setMetaOffset(long metaOffset) {
+ this.metaOffset = metaOffset;
+ }
+
public Map<String, Pair<Long, Integer>> getDeviceMetadataIndex() {
return deviceMetadataIndex;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriter.java
index a8627d6..f23e99b 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriter.java
@@ -20,13 +20,9 @@ package org.apache.iotdb.tsfile.write.writer;
import java.io.File;
import java.io.IOException;
-import java.util.Map;
import org.apache.iotdb.tsfile.exception.write.TsFileNotCompleteException;
-import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.file.metadata.TsFileMetadata;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
-import org.apache.iotdb.tsfile.utils.Pair;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -38,11 +34,10 @@ public class ForceAppendTsFileWriter extends TsFileIOWriter {
private long truncatePosition;
private static Logger logger = LoggerFactory.getLogger(ForceAppendTsFileWriter.class);
- private static final Logger resourceLogger = LoggerFactory.getLogger("FileMonitor");
public ForceAppendTsFileWriter(File file) throws IOException {
- if (resourceLogger.isInfoEnabled()) {
- resourceLogger.info("{} writer is opened.", file.getName());
+ if (logger.isDebugEnabled()) {
+ logger.debug("{} writer is opened.", file.getName());
}
this.out = new LocalTsFileOutput(file, true);
this.file = file;
@@ -60,17 +55,8 @@ public class ForceAppendTsFileWriter extends TsFileIOWriter {
"File " + file.getPath() + " is not a complete TsFile");
}
TsFileMetadata tsFileMetadata = reader.readFileMetadata();
- Map<String, Pair<Long, Integer>> deviceMap = tsFileMetadata.getDeviceMetadataIndex();
- long firstChunkMetaPos = Long.MAX_VALUE;
- for (String deviceId : deviceMap.keySet()) {
- Map<String, TimeseriesMetadata> deviceMetadata = reader.readDeviceMetadata(deviceId);
- for (TimeseriesMetadata timeseriesMetadata : deviceMetadata.values()) {
- firstChunkMetaPos = firstChunkMetaPos > timeseriesMetadata.getOffsetOfChunkMetaDataList() ?
- timeseriesMetadata.getOffsetOfChunkMetaDataList() : firstChunkMetaPos;
- }
- }
// truncate metadata and marker
- truncatePosition = firstChunkMetaPos - 1;
+ truncatePosition = tsFileMetadata.getMetaOffset();
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
index d26b1b9..06460b0 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
@@ -218,6 +218,8 @@ public class TsFileIOWriter {
*/
public void endFile() throws IOException {
+ long metaOffset = out.getPosition();
+
// serialize the SEPARATOR of MetaData
ReadWriteIOUtils.write(MetaMarker.SEPARATOR, out.wrapAsStream());
@@ -237,6 +239,7 @@ public class TsFileIOWriter {
tsFileMetaData.setVersionInfo(versionInfo);
tsFileMetaData.setTotalChunkNum(totalChunkNum);
tsFileMetaData.setInvalidChunkNum(invalidChunkNum);
+ tsFileMetaData.setMetaOffset(metaOffset);
long footerIndex = out.getPosition();
if (logger.isDebugEnabled()) {