You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by su...@apache.org on 2021/11/02 07:53:01 UTC
[iotdb] branch new_vector_jira1910 created (now 2fa0c3d)
This is an automated email from the ASF dual-hosted git repository.
sunzesong pushed a change to branch new_vector_jira1910
in repository https://gitbox.apache.org/repos/asf/iotdb.git.
at 2fa0c3d Support aligned timeseries in TsFile metadata index
This branch includes the following new commits:
new 2fa0c3d Support aligned timeseries in TsFile metadata index
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: Support aligned timeseries in TsFile metadata index
Posted by su...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sunzesong pushed a commit to branch new_vector_jira1910
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 2fa0c3d27465006f503753725a373bcfdfc4d519
Author: samperson1997 <sz...@mails.tsinghua.edu.cn>
AuthorDate: Tue Nov 2 15:51:45 2021 +0800
Support aligned timeseries in TsFile metadata index
---
.../tsfile/file/metadata/AlignedChunkMetadata.java | 10 ----
.../iotdb/tsfile/file/metadata/ChunkMetadata.java | 11 ----
.../iotdb/tsfile/file/metadata/IChunkMetadata.java | 4 --
.../file/metadata/MetadataIndexConstructor.java | 6 +--
.../tsfile/file/metadata/TimeseriesMetadata.java | 9 ----
.../iotdb/tsfile/read/TsFileSequenceReader.java | 35 -------------
.../iotdb/tsfile/write/writer/TsFileIOWriter.java | 61 +++-------------------
7 files changed, 8 insertions(+), 128 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/AlignedChunkMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/AlignedChunkMetadata.java
index f42bb3c..ee4f570 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/AlignedChunkMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/AlignedChunkMetadata.java
@@ -176,16 +176,6 @@ public class AlignedChunkMetadata implements IChunkMetadata {
return 0;
}
- @Override
- public boolean isTimeColumn() {
- return false;
- }
-
- @Override
- public boolean isValueColumn() {
- return false;
- }
-
public Chunk getTimeChunk() throws IOException {
return timeChunkMetadata.getChunkLoader().loadChunk((ChunkMetadata) timeChunkMetadata);
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
index 33908b6..7fe70d2 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
@@ -19,7 +19,6 @@
package org.apache.iotdb.tsfile.file.metadata;
import org.apache.iotdb.tsfile.common.cache.Accountable;
-import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.common.TimeRange;
@@ -335,16 +334,6 @@ public class ChunkMetadata implements Accountable, IChunkMetadata {
return mask;
}
- @Override
- public boolean isTimeColumn() {
- return mask == TsFileConstant.TIME_COLUMN_MASK;
- }
-
- @Override
- public boolean isValueColumn() {
- return mask == TsFileConstant.VALUE_COLUMN_MASK;
- }
-
public void setMask(byte mask) {
this.mask = mask;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IChunkMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IChunkMetadata.java
index 3893d15..1cc819f 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IChunkMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IChunkMetadata.java
@@ -74,8 +74,4 @@ public interface IChunkMetadata {
int serializeTo(OutputStream outputStream, boolean serializeStatistic) throws IOException;
byte getMask();
-
- boolean isTimeColumn();
-
- boolean isValueColumn();
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexConstructor.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexConstructor.java
index de20f40..062ffd6 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexConstructor.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexConstructor.java
@@ -62,21 +62,17 @@ public class MetadataIndexConstructor {
TimeseriesMetadata timeseriesMetadata;
MetadataIndexNode currentIndexNode =
new MetadataIndexNode(MetadataIndexNodeType.LEAF_MEASUREMENT);
- int serializedTimeseriesMetadataNum = 0;
for (int i = 0; i < entry.getValue().size(); i++) {
timeseriesMetadata = entry.getValue().get(i);
- if (serializedTimeseriesMetadataNum == 0
- || serializedTimeseriesMetadataNum >= config.getMaxDegreeOfIndexNode()) {
+ if (i % config.getMaxDegreeOfIndexNode() == 0) {
if (currentIndexNode.isFull()) {
addCurrentIndexNodeToQueue(currentIndexNode, measurementMetadataIndexQueue, out);
currentIndexNode = new MetadataIndexNode(MetadataIndexNodeType.LEAF_MEASUREMENT);
}
currentIndexNode.addEntry(
new MetadataIndexEntry(timeseriesMetadata.getMeasurementId(), out.getPosition()));
- serializedTimeseriesMetadataNum = 0;
}
timeseriesMetadata.serializeTo(out.wrapAsStream());
- serializedTimeseriesMetadataNum++;
}
addCurrentIndexNodeToQueue(currentIndexNode, measurementMetadataIndexQueue, out);
deviceMetadataIndexMap.put(
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
index 2881711..156d220 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
@@ -20,7 +20,6 @@
package org.apache.iotdb.tsfile.file.metadata;
import org.apache.iotdb.tsfile.common.cache.Accountable;
-import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.controller.IChunkMetadataLoader;
@@ -148,14 +147,6 @@ public class TimeseriesMetadata implements Accountable, ITimeSeriesMetadata {
return timeSeriesMetadataType;
}
- public boolean isTimeColumn() {
- return timeSeriesMetadataType == TsFileConstant.TIME_COLUMN_MASK;
- }
-
- public boolean isValueColumn() {
- return timeSeriesMetadataType == TsFileConstant.VALUE_COLUMN_MASK;
- }
-
public void setTimeSeriesMetadataType(byte timeSeriesMetadataType) {
this.timeSeriesMetadataType = timeSeriesMetadataType;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index 7588b84..c546068 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -370,41 +370,6 @@ public class TsFileSequenceReader implements AutoCloseable {
return searchResult >= 0 ? timeseriesMetadataList.get(searchResult) : null;
}
- /**
- * Find the leaf node that contains this vector, return all the needed subSensor and time column
- *
- * @param path path with time column
- * @param subSensorList value columns that needed
- * @return TimeseriesMetadata for the time column and all the needed subSensor, the order of the
- * element in this list should be the same as subSensorList
- */
- public List<TimeseriesMetadata> readTimeseriesMetadata(Path path, List<String> subSensorList)
- throws IOException {
- Pair<MetadataIndexEntry, Long> metadataIndexPair = getLeafMetadataIndexPair(path);
- if (metadataIndexPair == null) {
- return Collections.emptyList();
- }
- Map<String, TimeseriesMetadata> timeseriesMetadataMap = new HashMap<>();
- ByteBuffer buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right);
- while (buffer.hasRemaining()) {
- TimeseriesMetadata timeseriesMetadata;
- try {
- timeseriesMetadata = TimeseriesMetadata.deserializeFrom(buffer, true);
- } catch (BufferOverflowException e) {
- logger.error(
- "Something error happened while deserializing TimeseriesMetadata of file {}", file);
- throw e;
- }
- timeseriesMetadataMap.put(timeseriesMetadata.getMeasurementId(), timeseriesMetadata);
- }
-
- List<TimeseriesMetadata> timeseriesMetadataList = new ArrayList<>();
- for (String subSensor : subSensorList) {
- timeseriesMetadataList.add(timeseriesMetadataMap.get(subSensor));
- }
- return timeseriesMetadataList;
- }
-
/* Find the leaf node that contains path, return all the sensors in that leaf node which are also in allSensors set */
public List<TimeseriesMetadata> readTimeseriesMetadata(Path path, Set<String> allSensors)
throws IOException {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
index 3759780..524cb4f5 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
@@ -253,44 +253,17 @@ public class TsFileIOWriter {
ReadWriteIOUtils.write(MetaMarker.SEPARATOR, out.wrapAsStream());
// group ChunkMetadata by series
- // only contains ordinary path and time column of vector series
Map<Path, List<IChunkMetadata>> chunkMetadataListMap = new TreeMap<>();
- // time column -> ChunkMetadataList TreeMap of value columns in vector
- Map<Path, Map<Path, List<IChunkMetadata>>> vectorToPathsMap = new HashMap<>();
-
for (ChunkGroupMetadata chunkGroupMetadata : chunkGroupMetadataList) {
List<ChunkMetadata> chunkMetadatas = chunkGroupMetadata.getChunkMetadataList();
- int idx = 0;
- while (idx < chunkMetadatas.size()) {
- IChunkMetadata chunkMetadata = chunkMetadatas.get(idx);
- if (chunkMetadata.getMask() == 0) {
- Path series = new Path(chunkGroupMetadata.getDevice(), chunkMetadata.getMeasurementUid());
- chunkMetadataListMap.computeIfAbsent(series, k -> new ArrayList<>()).add(chunkMetadata);
- idx++;
- } else if (chunkMetadata.isTimeColumn()) {
- // time column of a vector series
- Path series = new Path(chunkGroupMetadata.getDevice(), chunkMetadata.getMeasurementUid());
- chunkMetadataListMap.computeIfAbsent(series, k -> new ArrayList<>()).add(chunkMetadata);
- idx++;
- Map<Path, List<IChunkMetadata>> chunkMetadataListMapInVector =
- vectorToPathsMap.computeIfAbsent(series, key -> new TreeMap<>());
-
- // value columns of a vector series
- while (idx < chunkMetadatas.size() && chunkMetadatas.get(idx).isValueColumn()) {
- chunkMetadata = chunkMetadatas.get(idx);
- Path vectorSeries =
- new Path(chunkGroupMetadata.getDevice(), chunkMetadata.getMeasurementUid());
- chunkMetadataListMapInVector
- .computeIfAbsent(vectorSeries, k -> new ArrayList<>())
- .add(chunkMetadata);
- idx++;
- }
- }
+ for (IChunkMetadata chunkMetadata : chunkMetadatas) {
+ Path series = new Path(chunkGroupMetadata.getDevice(), chunkMetadata.getMeasurementUid());
+ chunkMetadataListMap.computeIfAbsent(series, k -> new ArrayList<>()).add(chunkMetadata);
}
}
- MetadataIndexNode metadataIndex = flushMetadataIndex(chunkMetadataListMap, vectorToPathsMap);
+ MetadataIndexNode metadataIndex = flushMetadataIndex(chunkMetadataListMap);
TsFileMetadata tsFileMetaData = new TsFileMetadata();
tsFileMetaData.setMetadataIndex(metadataIndex);
tsFileMetaData.setMetaOffset(metaOffset);
@@ -330,13 +303,9 @@ public class TsFileIOWriter {
* Flush TsFileMetadata, including ChunkMetadataList and TimeseriesMetaData
*
* @param chunkMetadataListMap chunkMetadata that Path.mask == 0
- * @param vectorToPathsMap Map Path to chunkMataList, Key is Path(timeColumn) and Value is it's
- * sub chunkMetadataListMap
* @return MetadataIndexEntry list in TsFileMetadata
*/
- private MetadataIndexNode flushMetadataIndex(
- Map<Path, List<IChunkMetadata>> chunkMetadataListMap,
- Map<Path, Map<Path, List<IChunkMetadata>>> vectorToPathsMap)
+ private MetadataIndexNode flushMetadataIndex(Map<Path, List<IChunkMetadata>> chunkMetadataListMap)
throws IOException {
// convert ChunkMetadataList to this field
@@ -344,7 +313,7 @@ public class TsFileIOWriter {
// create device -> TimeseriesMetaDataList Map
for (Map.Entry<Path, List<IChunkMetadata>> entry : chunkMetadataListMap.entrySet()) {
// for ordinary path
- flushOneChunkMetadata(entry.getKey(), entry.getValue(), vectorToPathsMap);
+ flushOneChunkMetadata(entry.getKey(), entry.getValue());
}
// construct TsFileMetadata and return
@@ -356,12 +325,8 @@ public class TsFileIOWriter {
*
* @param path Path of chunk
* @param chunkMetadataList List of chunkMetadata about path(previous param)
- * @param vectorToPathsMap Key is Path(timeColumn) and Value is it's sub chunkMetadataListMap
*/
- private void flushOneChunkMetadata(
- Path path,
- List<IChunkMetadata> chunkMetadataList,
- Map<Path, Map<Path, List<IChunkMetadata>>> vectorToPathsMap)
+ private void flushOneChunkMetadata(Path path, List<IChunkMetadata> chunkMetadataList)
throws IOException {
// create TimeseriesMetaData
PublicBAOS publicBAOS = new PublicBAOS();
@@ -391,18 +356,6 @@ public class TsFileIOWriter {
deviceTimeseriesMetadataMap
.computeIfAbsent(path.getDevice(), k -> new ArrayList<>())
.add(timeseriesMetadata);
-
- // for VECTOR
- for (IChunkMetadata chunkMetadata : chunkMetadataList) {
- // chunkMetadata is time column of a vector series
- if (chunkMetadata.isTimeColumn()) {
- Map<Path, List<IChunkMetadata>> vectorMap = vectorToPathsMap.get(path);
- for (Map.Entry<Path, List<IChunkMetadata>> entry : vectorMap.entrySet()) {
- flushOneChunkMetadata(entry.getKey(), entry.getValue(), vectorToPathsMap);
- }
- }
- break;
- }
}
/**