You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ma...@apache.org on 2022/10/28 10:54:42 UTC
[iotdb] 01/01: try to accelrate TsFileIOWriter.endFile()
This is an automated email from the ASF dual-hosted git repository.
marklau99 pushed a commit to branch IOTDB-4791
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit f542692ec7b24203e02a46336c015ea42aa7fb9f
Author: Liu Xuxin <li...@outlook.com>
AuthorDate: Fri Oct 28 18:54:22 2022 +0800
try to accelrate TsFileIOWriter.endFile()
---
.../iotdb/tsfile/write/writer/TsFileIOWriter.java | 21 ++++++---------------
.../write/writer/tsmiterator/DiskTSMIterator.java | 7 ++++---
.../write/writer/tsmiterator/TSMIterator.java | 4 ++--
.../writer/TsFileIOWriterMemoryControlTest.java | 11 ++++++-----
4 files changed, 18 insertions(+), 25 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
index a8de080e0e..cebef59af5 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
@@ -111,7 +111,6 @@ public class TsFileIOWriter implements AutoCloseable {
protected File chunkMetadataTempFile;
protected LocalTsFileOutput tempOutput;
protected volatile boolean hasChunkMetadataInDisk = false;
- protected String currentSeries = null;
// record the total num of path in order to make bloom filter
protected int pathCount = 0;
protected boolean enableMemoryControl = false;
@@ -349,6 +348,7 @@ public class TsFileIOWriter implements AutoCloseable {
Queue<MetadataIndexNode> measurementMetadataIndexQueue = new ArrayDeque<>();
String currentDevice = null;
String prevDevice = null;
+ Path currentPath = null;
MetadataIndexNode currentIndexNode =
new MetadataIndexNode(MetadataIndexNodeType.LEAF_MEASUREMENT);
TSFileConfig config = TSFileDescriptor.getInstance().getConfig();
@@ -361,25 +361,16 @@ public class TsFileIOWriter implements AutoCloseable {
while (tsmIterator.hasNext()) {
// read in all chunk metadata of one series
// construct the timeseries metadata for this series
- Pair<String, TimeseriesMetadata> timeseriesMetadataPair = tsmIterator.next();
+ Pair<Path, TimeseriesMetadata> timeseriesMetadataPair = tsmIterator.next();
TimeseriesMetadata timeseriesMetadata = timeseriesMetadataPair.right;
- currentSeries = timeseriesMetadataPair.left;
+ currentPath = timeseriesMetadataPair.left;
indexCount++;
// build bloom filter
- filter.add(currentSeries);
+ filter.add(currentPath.getFullPath());
// construct the index tree node for the series
- Path currentPath = null;
- if (timeseriesMetadata.getTSDataType() == TSDataType.VECTOR) {
- // this series is the time column of the aligned device
- // the full series path will be like "root.sg.d."
- // we remove the last . in the series id here
- currentPath = new Path(currentSeries);
- currentDevice = currentSeries.substring(0, currentSeries.length() - 1);
- } else {
- currentPath = new Path(currentSeries, true);
- currentDevice = currentPath.getDevice();
- }
+
+ currentDevice = currentPath.getDevice();
if (!currentDevice.equals(prevDevice)) {
if (prevDevice != null) {
addCurrentIndexNodeToQueue(currentIndexNode, measurementMetadataIndexQueue, out);
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/tsmiterator/DiskTSMIterator.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/tsmiterator/DiskTSMIterator.java
index fd02f1438a..2ed2b57f08 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/tsmiterator/DiskTSMIterator.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/tsmiterator/DiskTSMIterator.java
@@ -24,6 +24,7 @@ import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.reader.LocalTsFileInput;
import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
@@ -75,7 +76,7 @@ public class DiskTSMIterator extends TSMIterator {
}
@Override
- public Pair<String, TimeseriesMetadata> next() {
+ public Pair<Path, TimeseriesMetadata> next() {
try {
if (remainsInFile) {
// deserialize from file
@@ -90,7 +91,7 @@ public class DiskTSMIterator extends TSMIterator {
}
}
- private Pair<String, TimeseriesMetadata> getTimeSerisMetadataFromFile() throws IOException {
+ private Pair<Path, TimeseriesMetadata> getTimeSerisMetadataFromFile() throws IOException {
if (currentPos == nextEndPosForDevice) {
// deserialize the current device name
currentDevice = ReadWriteIOUtils.readString(input.wrapAsInputStream());
@@ -118,7 +119,7 @@ public class DiskTSMIterator extends TSMIterator {
}
updateCurrentPos();
return new Pair<>(
- currentDevice + "." + measurementUid,
+ new Path(currentDevice, measurementUid),
constructOneTimeseriesMetadata(measurementUid, chunkMetadataList));
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/tsmiterator/TSMIterator.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/tsmiterator/TSMIterator.java
index f11242f296..38425044ac 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/tsmiterator/TSMIterator.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/tsmiterator/TSMIterator.java
@@ -70,10 +70,10 @@ public class TSMIterator {
return iterator.hasNext();
}
- public Pair<String, TimeseriesMetadata> next() throws IOException {
+ public Pair<Path, TimeseriesMetadata> next() throws IOException {
Pair<Path, List<IChunkMetadata>> nextPair = iterator.next();
return new Pair<>(
- nextPair.left.getFullPath(),
+ nextPair.left,
constructOneTimeseriesMetadata(nextPair.left.getMeasurement(), nextPair.right));
}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriterMemoryControlTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriterMemoryControlTest.java
index b7c6ff84db..2152318cc8 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriterMemoryControlTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriterMemoryControlTest.java
@@ -28,6 +28,7 @@ import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.utils.Binary;
import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.tsfile.utils.TsPrimitiveType;
@@ -132,7 +133,7 @@ public class TsFileIOWriterMemoryControlTest {
writer.chunkGroupMetadataList,
writer.endPosInCMTForDevice);
for (int i = 0; iterator.hasNext(); ++i) {
- Pair<String, TimeseriesMetadata> timeseriesMetadataPair = iterator.next();
+ Pair<Path, TimeseriesMetadata> timeseriesMetadataPair = iterator.next();
TimeseriesMetadata timeseriesMetadata = timeseriesMetadataPair.right;
Assert.assertEquals(sortedSeriesId.get(i % 5), timeseriesMetadata.getMeasurementId());
Assert.assertEquals(
@@ -169,8 +170,8 @@ public class TsFileIOWriterMemoryControlTest {
TSMIterator.getTSMIteratorInDisk(
writer.chunkMetadataTempFile, new ArrayList<>(), writer.endPosInCMTForDevice);
for (int i = 0; iterator.hasNext(); ++i) {
- Pair<String, TimeseriesMetadata> timeseriesMetadataPair = iterator.next();
- String fullPath = timeseriesMetadataPair.left;
+ Pair<Path, TimeseriesMetadata> timeseriesMetadataPair = iterator.next();
+ String fullPath = timeseriesMetadataPair.left.getFullPath();
TimeseriesMetadata timeseriesMetadata = timeseriesMetadataPair.right;
Assert.assertEquals(measurementIds.get(i), fullPath);
Assert.assertEquals(
@@ -233,8 +234,8 @@ public class TsFileIOWriterMemoryControlTest {
TSMIterator.getTSMIteratorInDisk(
writer.chunkMetadataTempFile, new ArrayList<>(), writer.endPosInCMTForDevice);
for (int i = 0; i < originChunkMetadataList.size(); ++i) {
- Pair<String, TimeseriesMetadata> timeseriesMetadataPair = iterator.next();
- Assert.assertEquals(seriesIds.get(i), timeseriesMetadataPair.left);
+ Pair<Path, TimeseriesMetadata> timeseriesMetadataPair = iterator.next();
+ Assert.assertEquals(seriesIds.get(i), timeseriesMetadataPair.left.getFullPath());
Assert.assertEquals(
originChunkMetadataList.get(i).getDataType(),
timeseriesMetadataPair.right.getTSDataType());