You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hx...@apache.org on 2019/07/11 12:15:20 UTC
[incubator-iotdb] 01/01: add a function to support appending data
at the end of a completed TsFile
This is an automated email from the ASF dual-hosted git repository.
hxd pushed a commit to branch feature_append_add_on_complete_tsfile
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 8d2f4ac7348d0b948f9ec26ceb079617c17b8001
Author: xiangdong huang <sa...@gmail.com>
AuthorDate: Thu Jul 11 20:14:57 2019 +0800
add a function to support appending data at the end of a completed TsFile
---
.../iotdb/tsfile/read/TsFileSequenceReader.java | 2 +-
.../write/writer/RestorableTsFileIOWriter.java | 41 ++++++++++++++++++++++
.../write/writer/RestorableTsFileIOWriterTest.java | 23 ++++++++++++
3 files changed, 65 insertions(+), 1 deletion(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index 8dc058c..eec73e0 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -142,7 +142,7 @@ public class TsFileSequenceReader implements AutoCloseable {
this.fileMetadataSize = fileMetadataSize;
}
- protected void loadMetadataSize() throws IOException {
+ public void loadMetadataSize() throws IOException {
ByteBuffer metadataSize = ByteBuffer.allocate(Integer.BYTES);
tsFileInput.read(metadataSize,
tsFileInput.size() - TSFileConfig.MAGIC_STRING.length() - Integer.BYTES);
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java
index d5bb0e0..d84f391 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java
@@ -21,6 +21,9 @@ package org.apache.iotdb.tsfile.write.writer;
import java.io.File;
import java.io.IOException;
+import java.nio.channels.FileChannel;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -28,6 +31,8 @@ import java.util.Map;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetaData;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
+import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadataIndex;
+import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.TsFileCheckStatus;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
@@ -179,4 +184,40 @@ public class RestorableTsFileIOWriter extends TsFileIOWriter {
}
+ /**
+ * Given a TsFile, generate a writable RestorableTsFileIOWriter. That is, for a complete TsFile,
+ * the function erases all FileMetadata and supports writing new data; For a incomplete TsFile,
+ * the function supports writing new data directly. However, it is more efficient using the
+ * construction function of RestorableTsFileIOWriter, if the tsfile is incomplete.
+ *
+ * @param file a TsFile
+ * @return a writable RestorableTsFileIOWriter
+ */
+ public static RestorableTsFileIOWriter getWriterForAppendingDataOnCompletedTsFile(File file)
+ throws IOException {
+ long position = file.length();
+
+ try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getAbsolutePath(), false)) {
+ // this tsfile is complete
+ if (reader.isComplete()) {
+ reader.loadMetadataSize();
+ TsFileMetaData metaData = reader.readFileMetadata();
+ for (TsDeviceMetadataIndex deviceMetadata : metaData.getDeviceMap().values()) {
+ if (position > deviceMetadata.getOffset()) {
+ position = deviceMetadata.getOffset();
+ }
+ }
+ }
+ }
+
+ if (position != file.length()) {
+ // if the file is complete, we will remove all file metadatas
+ try (FileChannel channel = FileChannel
+ .open(Paths.get(file.getAbsolutePath()), StandardOpenOption.WRITE)) {
+ channel.truncate(position - 1);//remove the last marker.
+ }
+ }
+ return new RestorableTsFileIOWriter(file);
+ }
+
}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriterTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriterTest.java
index 697d937..ef2f19b 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriterTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriterTest.java
@@ -29,6 +29,7 @@ import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import org.apache.iotdb.tsfile.TsFileSequenceRead;
import org.apache.iotdb.tsfile.file.MetaMarker;
import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadataIndex;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
@@ -327,4 +328,26 @@ public class RestorableTsFileIOWriterTest {
reader.close();
assertTrue(file.delete());
}
+
+ @Test
+ public void testAppendDataOnCompletedFile() throws Exception {
+ File file = new File(FILE_NAME);
+ TsFileWriter writer = new TsFileWriter(file);
+ writer.addMeasurement(new MeasurementSchema("s1", TSDataType.FLOAT, TSEncoding.RLE));
+ writer.addMeasurement(new MeasurementSchema("s2", TSDataType.FLOAT, TSEncoding.RLE));
+ writer.write(new TSRecord(1, "d1").addTuple(new FloatDataPoint("s1", 5))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.write(new TSRecord(2, "d1").addTuple(new FloatDataPoint("s1", 5))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.close();
+
+ long size = file.length();
+ RestorableTsFileIOWriter rWriter = RestorableTsFileIOWriter
+ .getWriterForAppendingDataOnCompletedTsFile(file);
+ TsFileWriter write = new TsFileWriter(rWriter);
+ write.close();
+ assertEquals(size, file.length());
+ assertTrue(file.delete());
+
+ }
}
\ No newline at end of file