You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2020/04/14 05:14:16 UTC
[incubator-iotdb] branch master updated: Fix
ForceAppendTsFileWriter (#1038)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new b233694 Fix ForceAppendTsFileWriter (#1038)
b233694 is described below
commit b233694980302ddb458ef58ce254c8b52be3dff1
Author: Haonan <hh...@outlook.com>
AuthorDate: Tue Apr 14 13:14:08 2020 +0800
Fix ForceAppendTsFileWriter (#1038)
* fix ForceAppendTsFileWriter
---
.../write/writer/ForceAppendTsFileWriter.java | 14 +++--
.../write/writer/ForceAppendTsFileWriterTest.java | 61 ++++++++++++++++++++++
2 files changed, 70 insertions(+), 5 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriter.java
index d1b0114..a8627d6 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriter.java
@@ -22,6 +22,7 @@ import java.io.File;
import java.io.IOException;
import java.util.Map;
import org.apache.iotdb.tsfile.exception.write.TsFileNotCompleteException;
+import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.file.metadata.TsFileMetadata;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.utils.Pair;
@@ -60,13 +61,16 @@ public class ForceAppendTsFileWriter extends TsFileIOWriter {
}
TsFileMetadata tsFileMetadata = reader.readFileMetadata();
Map<String, Pair<Long, Integer>> deviceMap = tsFileMetadata.getDeviceMetadataIndex();
- long firstDeviceMetaPos = Long.MAX_VALUE;
- for (Pair<Long, Integer> deviceMetadataIndex : deviceMap.values()) {
- firstDeviceMetaPos = firstDeviceMetaPos > deviceMetadataIndex.left ?
- deviceMetadataIndex.left : firstDeviceMetaPos;
+ long firstChunkMetaPos = Long.MAX_VALUE;
+ for (String deviceId : deviceMap.keySet()) {
+ Map<String, TimeseriesMetadata> deviceMetadata = reader.readDeviceMetadata(deviceId);
+ for (TimeseriesMetadata timeseriesMetadata : deviceMetadata.values()) {
+ firstChunkMetaPos = firstChunkMetaPos > timeseriesMetadata.getOffsetOfChunkMetaDataList() ?
+ timeseriesMetadata.getOffsetOfChunkMetaDataList() : firstChunkMetaPos;
+ }
}
// truncate metadata and marker
- truncatePosition = firstDeviceMetaPos - 1;
+ truncatePosition = firstChunkMetaPos - 1;
}
}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriterTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriterTest.java
new file mode 100644
index 0000000..694d7c1
--- /dev/null
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriterTest.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.tsfile.write.writer;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.File;
+
+import org.apache.iotdb.tsfile.constant.TestConstant;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
+import org.apache.iotdb.tsfile.fileSystem.fsFactory.FSFactory;
+import org.apache.iotdb.tsfile.read.common.Path;
+import org.apache.iotdb.tsfile.write.TsFileWriter;
+import org.apache.iotdb.tsfile.write.record.TSRecord;
+import org.apache.iotdb.tsfile.write.record.datapoint.FloatDataPoint;
+import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
+import org.junit.Test;
+
+public class ForceAppendTsFileWriterTest {
+ private static final String FILE_NAME = TestConstant.BASE_OUTPUT_PATH.concat("test.tsfile");
+ private static FSFactory fsFactory = FSFactoryProducer.getFSFactory();
+
+ @Test
+ public void test() throws Exception {
+ File file = fsFactory.getFile(FILE_NAME);
+ TsFileWriter writer = new TsFileWriter(file);
+ writer.registerTimeseries(new Path("d1.s1"),
+ new MeasurementSchema("s1", TSDataType.FLOAT, TSEncoding.RLE));
+ writer.registerTimeseries(new Path("d1.s2"),
+ new MeasurementSchema("s2", TSDataType.FLOAT, TSEncoding.RLE));
+ writer.write(new TSRecord(1, "d1").addTuple(new FloatDataPoint("s1", 5))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.write(new TSRecord(2, "d1").addTuple(new FloatDataPoint("s1", 5))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.flushAllChunkGroups();
+
+ long firstMetadataPosition = writer.getIOWriter().getPos();
+ writer.close();
+ ForceAppendTsFileWriter fwriter = new ForceAppendTsFileWriter(file);
+ assertEquals(firstMetadataPosition, fwriter.getTruncatePosition());
+ }
+
+}