You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2020/04/14 08:05:15 UTC
[incubator-iotdb] branch add_meta_marker_offset created (now
839dd1b)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a change to branch add_meta_marker_offset
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.
at 839dd1b add metaOffset in TsFileMetadata
This branch includes the following new commits:
new 839dd1b add metaOffset in TsFileMetadata
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-iotdb] 01/01: add metaOffset in TsFileMetadata
Posted by qi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch add_meta_marker_offset
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 839dd1b39ad10f9336beb372d24ca533aea5b824
Author: qiaojialin <64...@qq.com>
AuthorDate: Tue Apr 14 16:04:58 2020 +0800
add metaOffset in TsFileMetadata
---
.../iotdb/tsfile/file/metadata/TsFileMetadata.java | 20 ++++++++++++++++++--
.../tsfile/write/writer/ForceAppendTsFileWriter.java | 20 +++-----------------
.../iotdb/tsfile/write/writer/TsFileIOWriter.java | 3 +++
3 files changed, 24 insertions(+), 19 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java
index feb147e..3195c08 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java
@@ -54,6 +54,8 @@ public class TsFileMetadata {
// offset -> version
private List<Pair<Long, Long>> versionInfo;
+ // offset of MetaMarker.SEPARATOR
+ private long metaOffset;
/**
* deserialize data from the buffer.
@@ -90,6 +92,9 @@ public class TsFileMetadata {
}
fileMetaData.setVersionInfo(versionInfo);
+ // metaOffset
+ long metaOffset = ReadWriteIOUtils.readLong(buffer);
+ fileMetaData.setMetaOffset(metaOffset);
// read bloom filter
if (buffer.hasRemaining()) {
@@ -134,10 +139,13 @@ public class TsFileMetadata {
// versionInfo
byteLen += ReadWriteIOUtils.write(versionInfo.size(), outputStream);
for (Pair<Long, Long> versionPair : versionInfo) {
- byteLen +=ReadWriteIOUtils.write(versionPair.left, outputStream);
- byteLen +=ReadWriteIOUtils.write(versionPair.right, outputStream);
+ byteLen += ReadWriteIOUtils.write(versionPair.left, outputStream);
+ byteLen += ReadWriteIOUtils.write(versionPair.right, outputStream);
}
+ // metaOffset
+ byteLen += ReadWriteIOUtils.write(metaOffset, outputStream);
+
return byteLen;
}
@@ -192,6 +200,14 @@ public class TsFileMetadata {
this.invalidChunkNum = invalidChunkNum;
}
+ public long getMetaOffset() {
+ return metaOffset;
+ }
+
+ public void setMetaOffset(long metaOffset) {
+ this.metaOffset = metaOffset;
+ }
+
public Map<String, Pair<Long, Integer>> getDeviceMetadataIndex() {
return deviceMetadataIndex;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriter.java
index a8627d6..f23e99b 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriter.java
@@ -20,13 +20,9 @@ package org.apache.iotdb.tsfile.write.writer;
import java.io.File;
import java.io.IOException;
-import java.util.Map;
import org.apache.iotdb.tsfile.exception.write.TsFileNotCompleteException;
-import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.file.metadata.TsFileMetadata;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
-import org.apache.iotdb.tsfile.utils.Pair;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -38,11 +34,10 @@ public class ForceAppendTsFileWriter extends TsFileIOWriter {
private long truncatePosition;
private static Logger logger = LoggerFactory.getLogger(ForceAppendTsFileWriter.class);
- private static final Logger resourceLogger = LoggerFactory.getLogger("FileMonitor");
public ForceAppendTsFileWriter(File file) throws IOException {
- if (resourceLogger.isInfoEnabled()) {
- resourceLogger.info("{} writer is opened.", file.getName());
+ if (logger.isDebugEnabled()) {
+ logger.debug("{} writer is opened.", file.getName());
}
this.out = new LocalTsFileOutput(file, true);
this.file = file;
@@ -60,17 +55,8 @@ public class ForceAppendTsFileWriter extends TsFileIOWriter {
"File " + file.getPath() + " is not a complete TsFile");
}
TsFileMetadata tsFileMetadata = reader.readFileMetadata();
- Map<String, Pair<Long, Integer>> deviceMap = tsFileMetadata.getDeviceMetadataIndex();
- long firstChunkMetaPos = Long.MAX_VALUE;
- for (String deviceId : deviceMap.keySet()) {
- Map<String, TimeseriesMetadata> deviceMetadata = reader.readDeviceMetadata(deviceId);
- for (TimeseriesMetadata timeseriesMetadata : deviceMetadata.values()) {
- firstChunkMetaPos = firstChunkMetaPos > timeseriesMetadata.getOffsetOfChunkMetaDataList() ?
- timeseriesMetadata.getOffsetOfChunkMetaDataList() : firstChunkMetaPos;
- }
- }
// truncate metadata and marker
- truncatePosition = firstChunkMetaPos - 1;
+ truncatePosition = tsFileMetadata.getMetaOffset();
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
index d26b1b9..06460b0 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
@@ -218,6 +218,8 @@ public class TsFileIOWriter {
*/
public void endFile() throws IOException {
+ long metaOffset = out.getPosition();
+
// serialize the SEPARATOR of MetaData
ReadWriteIOUtils.write(MetaMarker.SEPARATOR, out.wrapAsStream());
@@ -237,6 +239,7 @@ public class TsFileIOWriter {
tsFileMetaData.setVersionInfo(versionInfo);
tsFileMetaData.setTotalChunkNum(totalChunkNum);
tsFileMetaData.setInvalidChunkNum(invalidChunkNum);
+ tsFileMetaData.setMetaOffset(metaOffset);
long footerIndex = out.getPosition();
if (logger.isDebugEnabled()) {