You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by su...@apache.org on 2021/11/02 07:53:02 UTC
[iotdb] 01/01: Support aligned timeseries in TsFile metadata index
This is an automated email from the ASF dual-hosted git repository.
sunzesong pushed a commit to branch new_vector_jira1910
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 2fa0c3d27465006f503753725a373bcfdfc4d519
Author: samperson1997 <sz...@mails.tsinghua.edu.cn>
AuthorDate: Tue Nov 2 15:51:45 2021 +0800
Support aligned timeseries in TsFile metadata index
---
.../tsfile/file/metadata/AlignedChunkMetadata.java | 10 ----
.../iotdb/tsfile/file/metadata/ChunkMetadata.java | 11 ----
.../iotdb/tsfile/file/metadata/IChunkMetadata.java | 4 --
.../file/metadata/MetadataIndexConstructor.java | 6 +--
.../tsfile/file/metadata/TimeseriesMetadata.java | 9 ----
.../iotdb/tsfile/read/TsFileSequenceReader.java | 35 -------------
.../iotdb/tsfile/write/writer/TsFileIOWriter.java | 61 +++-------------------
7 files changed, 8 insertions(+), 128 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/AlignedChunkMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/AlignedChunkMetadata.java
index f42bb3c..ee4f570 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/AlignedChunkMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/AlignedChunkMetadata.java
@@ -176,16 +176,6 @@ public class AlignedChunkMetadata implements IChunkMetadata {
return 0;
}
- @Override
- public boolean isTimeColumn() {
- return false;
- }
-
- @Override
- public boolean isValueColumn() {
- return false;
- }
-
public Chunk getTimeChunk() throws IOException {
return timeChunkMetadata.getChunkLoader().loadChunk((ChunkMetadata) timeChunkMetadata);
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
index 33908b6..7fe70d2 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
@@ -19,7 +19,6 @@
package org.apache.iotdb.tsfile.file.metadata;
import org.apache.iotdb.tsfile.common.cache.Accountable;
-import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.common.TimeRange;
@@ -335,16 +334,6 @@ public class ChunkMetadata implements Accountable, IChunkMetadata {
return mask;
}
- @Override
- public boolean isTimeColumn() {
- return mask == TsFileConstant.TIME_COLUMN_MASK;
- }
-
- @Override
- public boolean isValueColumn() {
- return mask == TsFileConstant.VALUE_COLUMN_MASK;
- }
-
public void setMask(byte mask) {
this.mask = mask;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IChunkMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IChunkMetadata.java
index 3893d15..1cc819f 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IChunkMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IChunkMetadata.java
@@ -74,8 +74,4 @@ public interface IChunkMetadata {
int serializeTo(OutputStream outputStream, boolean serializeStatistic) throws IOException;
byte getMask();
-
- boolean isTimeColumn();
-
- boolean isValueColumn();
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexConstructor.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexConstructor.java
index de20f40..062ffd6 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexConstructor.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexConstructor.java
@@ -62,21 +62,17 @@ public class MetadataIndexConstructor {
TimeseriesMetadata timeseriesMetadata;
MetadataIndexNode currentIndexNode =
new MetadataIndexNode(MetadataIndexNodeType.LEAF_MEASUREMENT);
- int serializedTimeseriesMetadataNum = 0;
for (int i = 0; i < entry.getValue().size(); i++) {
timeseriesMetadata = entry.getValue().get(i);
- if (serializedTimeseriesMetadataNum == 0
- || serializedTimeseriesMetadataNum >= config.getMaxDegreeOfIndexNode()) {
+ if (i % config.getMaxDegreeOfIndexNode() == 0) {
if (currentIndexNode.isFull()) {
addCurrentIndexNodeToQueue(currentIndexNode, measurementMetadataIndexQueue, out);
currentIndexNode = new MetadataIndexNode(MetadataIndexNodeType.LEAF_MEASUREMENT);
}
currentIndexNode.addEntry(
new MetadataIndexEntry(timeseriesMetadata.getMeasurementId(), out.getPosition()));
- serializedTimeseriesMetadataNum = 0;
}
timeseriesMetadata.serializeTo(out.wrapAsStream());
- serializedTimeseriesMetadataNum++;
}
addCurrentIndexNodeToQueue(currentIndexNode, measurementMetadataIndexQueue, out);
deviceMetadataIndexMap.put(
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
index 2881711..156d220 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
@@ -20,7 +20,6 @@
package org.apache.iotdb.tsfile.file.metadata;
import org.apache.iotdb.tsfile.common.cache.Accountable;
-import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.controller.IChunkMetadataLoader;
@@ -148,14 +147,6 @@ public class TimeseriesMetadata implements Accountable, ITimeSeriesMetadata {
return timeSeriesMetadataType;
}
- public boolean isTimeColumn() {
- return timeSeriesMetadataType == TsFileConstant.TIME_COLUMN_MASK;
- }
-
- public boolean isValueColumn() {
- return timeSeriesMetadataType == TsFileConstant.VALUE_COLUMN_MASK;
- }
-
public void setTimeSeriesMetadataType(byte timeSeriesMetadataType) {
this.timeSeriesMetadataType = timeSeriesMetadataType;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index 7588b84..c546068 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -370,41 +370,6 @@ public class TsFileSequenceReader implements AutoCloseable {
return searchResult >= 0 ? timeseriesMetadataList.get(searchResult) : null;
}
- /**
- * Find the leaf node that contains this vector, return all the needed subSensor and time column
- *
- * @param path path with time column
- * @param subSensorList value columns that needed
- * @return TimeseriesMetadata for the time column and all the needed subSensor, the order of the
- * element in this list should be the same as subSensorList
- */
- public List<TimeseriesMetadata> readTimeseriesMetadata(Path path, List<String> subSensorList)
- throws IOException {
- Pair<MetadataIndexEntry, Long> metadataIndexPair = getLeafMetadataIndexPair(path);
- if (metadataIndexPair == null) {
- return Collections.emptyList();
- }
- Map<String, TimeseriesMetadata> timeseriesMetadataMap = new HashMap<>();
- ByteBuffer buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right);
- while (buffer.hasRemaining()) {
- TimeseriesMetadata timeseriesMetadata;
- try {
- timeseriesMetadata = TimeseriesMetadata.deserializeFrom(buffer, true);
- } catch (BufferOverflowException e) {
- logger.error(
- "Something error happened while deserializing TimeseriesMetadata of file {}", file);
- throw e;
- }
- timeseriesMetadataMap.put(timeseriesMetadata.getMeasurementId(), timeseriesMetadata);
- }
-
- List<TimeseriesMetadata> timeseriesMetadataList = new ArrayList<>();
- for (String subSensor : subSensorList) {
- timeseriesMetadataList.add(timeseriesMetadataMap.get(subSensor));
- }
- return timeseriesMetadataList;
- }
-
/* Find the leaf node that contains path, return all the sensors in that leaf node which are also in allSensors set */
public List<TimeseriesMetadata> readTimeseriesMetadata(Path path, Set<String> allSensors)
throws IOException {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
index 3759780..524cb4f5 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
@@ -253,44 +253,17 @@ public class TsFileIOWriter {
ReadWriteIOUtils.write(MetaMarker.SEPARATOR, out.wrapAsStream());
// group ChunkMetadata by series
- // only contains ordinary path and time column of vector series
Map<Path, List<IChunkMetadata>> chunkMetadataListMap = new TreeMap<>();
- // time column -> ChunkMetadataList TreeMap of value columns in vector
- Map<Path, Map<Path, List<IChunkMetadata>>> vectorToPathsMap = new HashMap<>();
-
for (ChunkGroupMetadata chunkGroupMetadata : chunkGroupMetadataList) {
List<ChunkMetadata> chunkMetadatas = chunkGroupMetadata.getChunkMetadataList();
- int idx = 0;
- while (idx < chunkMetadatas.size()) {
- IChunkMetadata chunkMetadata = chunkMetadatas.get(idx);
- if (chunkMetadata.getMask() == 0) {
- Path series = new Path(chunkGroupMetadata.getDevice(), chunkMetadata.getMeasurementUid());
- chunkMetadataListMap.computeIfAbsent(series, k -> new ArrayList<>()).add(chunkMetadata);
- idx++;
- } else if (chunkMetadata.isTimeColumn()) {
- // time column of a vector series
- Path series = new Path(chunkGroupMetadata.getDevice(), chunkMetadata.getMeasurementUid());
- chunkMetadataListMap.computeIfAbsent(series, k -> new ArrayList<>()).add(chunkMetadata);
- idx++;
- Map<Path, List<IChunkMetadata>> chunkMetadataListMapInVector =
- vectorToPathsMap.computeIfAbsent(series, key -> new TreeMap<>());
-
- // value columns of a vector series
- while (idx < chunkMetadatas.size() && chunkMetadatas.get(idx).isValueColumn()) {
- chunkMetadata = chunkMetadatas.get(idx);
- Path vectorSeries =
- new Path(chunkGroupMetadata.getDevice(), chunkMetadata.getMeasurementUid());
- chunkMetadataListMapInVector
- .computeIfAbsent(vectorSeries, k -> new ArrayList<>())
- .add(chunkMetadata);
- idx++;
- }
- }
+ for (IChunkMetadata chunkMetadata : chunkMetadatas) {
+ Path series = new Path(chunkGroupMetadata.getDevice(), chunkMetadata.getMeasurementUid());
+ chunkMetadataListMap.computeIfAbsent(series, k -> new ArrayList<>()).add(chunkMetadata);
}
}
- MetadataIndexNode metadataIndex = flushMetadataIndex(chunkMetadataListMap, vectorToPathsMap);
+ MetadataIndexNode metadataIndex = flushMetadataIndex(chunkMetadataListMap);
TsFileMetadata tsFileMetaData = new TsFileMetadata();
tsFileMetaData.setMetadataIndex(metadataIndex);
tsFileMetaData.setMetaOffset(metaOffset);
@@ -330,13 +303,9 @@ public class TsFileIOWriter {
* Flush TsFileMetadata, including ChunkMetadataList and TimeseriesMetaData
*
* @param chunkMetadataListMap chunkMetadata that Path.mask == 0
- * @param vectorToPathsMap Map Path to chunkMataList, Key is Path(timeColumn) and Value is it's
- * sub chunkMetadataListMap
* @return MetadataIndexEntry list in TsFileMetadata
*/
- private MetadataIndexNode flushMetadataIndex(
- Map<Path, List<IChunkMetadata>> chunkMetadataListMap,
- Map<Path, Map<Path, List<IChunkMetadata>>> vectorToPathsMap)
+ private MetadataIndexNode flushMetadataIndex(Map<Path, List<IChunkMetadata>> chunkMetadataListMap)
throws IOException {
// convert ChunkMetadataList to this field
@@ -344,7 +313,7 @@ public class TsFileIOWriter {
// create device -> TimeseriesMetaDataList Map
for (Map.Entry<Path, List<IChunkMetadata>> entry : chunkMetadataListMap.entrySet()) {
// for ordinary path
- flushOneChunkMetadata(entry.getKey(), entry.getValue(), vectorToPathsMap);
+ flushOneChunkMetadata(entry.getKey(), entry.getValue());
}
// construct TsFileMetadata and return
@@ -356,12 +325,8 @@ public class TsFileIOWriter {
*
* @param path Path of chunk
* @param chunkMetadataList List of chunkMetadata about path(previous param)
- * @param vectorToPathsMap Key is Path(timeColumn) and Value is it's sub chunkMetadataListMap
*/
- private void flushOneChunkMetadata(
- Path path,
- List<IChunkMetadata> chunkMetadataList,
- Map<Path, Map<Path, List<IChunkMetadata>>> vectorToPathsMap)
+ private void flushOneChunkMetadata(Path path, List<IChunkMetadata> chunkMetadataList)
throws IOException {
// create TimeseriesMetaData
PublicBAOS publicBAOS = new PublicBAOS();
@@ -391,18 +356,6 @@ public class TsFileIOWriter {
deviceTimeseriesMetadataMap
.computeIfAbsent(path.getDevice(), k -> new ArrayList<>())
.add(timeseriesMetadata);
-
- // for VECTOR
- for (IChunkMetadata chunkMetadata : chunkMetadataList) {
- // chunkMetadata is time column of a vector series
- if (chunkMetadata.isTimeColumn()) {
- Map<Path, List<IChunkMetadata>> vectorMap = vectorToPathsMap.get(path);
- for (Map.Entry<Path, List<IChunkMetadata>> entry : vectorMap.entrySet()) {
- flushOneChunkMetadata(entry.getKey(), entry.getValue(), vectorToPathsMap);
- }
- }
- break;
- }
}
/**