You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2025/03/20 04:09:07 UTC
(iotdb) branch dev/1.3 updated: [To dev/1.3] perf: various types of iterator for multiple tvlists in memchunk (#15115)
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch dev/1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/dev/1.3 by this push:
new 043f0c74312 [To dev/1.3] perf: various types of iterator for multiple tvlists in memchunk (#15115)
043f0c74312 is described below
commit 043f0c74312bddd33d52879c4e2cb36bfbfa28d2
Author: shizy <sh...@gmail.com>
AuthorDate: Thu Mar 20 12:09:01 2025 +0800
[To dev/1.3] perf: various types of iterator for multiple tvlists in memchunk (#15115)
* perf: batch iterator for multiple TVLists/alignedTVList
* fix bug
* current time
* single TVList/alignedTVList iterator
* fix: queryAlignChuckWithDeletionTest
* fix: pr review
* fix: test cases
* remove clone in sortTvListForFlush
---
.../dataregion/memtable/AbstractMemTable.java | 4 +-
.../memtable/AlignedReadOnlyMemChunk.java | 244 ++++++--------
.../memtable/AlignedWritableMemChunk.java | 158 +++++----
.../dataregion/memtable/ReadOnlyMemChunk.java | 160 ++++-----
.../dataregion/memtable/WritableMemChunk.java | 164 ++++-----
.../read/reader/chunk/MemAlignedChunkReader.java | 147 +-------
.../read/reader/chunk/MemAlignedPageReader.java | 22 +-
.../read/reader/chunk/MemChunkReader.java | 111 +------
.../read/reader/chunk/MemPageReader.java | 23 +-
.../db/utils/datastructure/AlignedTVList.java | 370 +++++++++++++++------
.../db/utils/datastructure/MemPointIterator.java | 31 ++
.../datastructure/MemPointIteratorFactory.java | 304 +++++++++++++++++
.../MergeSortAlignedTVListIterator.java | 192 -----------
.../MergeSortMultiAlignedTVListIterator.java | 171 ++++++++++
.../MergeSortMultiTVListIterator.java | 83 +++++
.../datastructure/MergeSortTVListIterator.java | 180 ----------
.../datastructure/MultiAlignedTVListIterator.java | 231 +++++++++++++
.../utils/datastructure/MultiTVListIterator.java | 179 ++++++++++
.../OrderedMultiAlignedTVListIterator.java | 111 +++++++
.../datastructure/OrderedMultiTVListIterator.java | 58 ++++
.../iotdb/db/utils/datastructure/TVList.java | 215 +++++++++---
.../reader/chunk/AlignedMemPageReaderTest.java | 8 +-
.../reader/chunk/MemAlignedChunkLoaderTest.java | 12 +-
.../read/reader/chunk/MemChunkLoaderTest.java | 58 ++--
.../read/reader/chunk/MemPageReaderTest.java | 4 +-
25 files changed, 1990 insertions(+), 1250 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
index 1d39224641c..6df59ed1220 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
@@ -577,7 +577,7 @@ public abstract class AbstractMemTable implements IMemTable {
}
List<BitMap> bitMaps = new ArrayList<>();
- long[] timestamps = alignedMemChunk.getFilteredTimestamp(deletionList, bitMaps, true);
+ long[] timestamps = alignedMemChunk.getFilteredTimestamp(deletionList, bitMaps);
buildAlignedMemChunkHandle(
deviceID,
@@ -614,7 +614,7 @@ public abstract class AbstractMemTable implements IMemTable {
}
List<BitMap> bitMaps = new ArrayList<>();
- long[] timestamps = memChunk.getFilteredTimestamp(deletionList, bitMaps, true);
+ long[] timestamps = memChunk.getFilteredTimestamp(deletionList, bitMaps);
buildAlignedMemChunkHandle(
deviceID,
timestamps,
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java
index 7c6264f1097..d178c574b67 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java
@@ -22,7 +22,8 @@ package org.apache.iotdb.db.storageengine.dataregion.memtable;
import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext;
import org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk.MemAlignedChunkLoader;
import org.apache.iotdb.db.utils.datastructure.AlignedTVList;
-import org.apache.iotdb.db.utils.datastructure.MergeSortAlignedTVListIterator;
+import org.apache.iotdb.db.utils.datastructure.MemPointIterator;
+import org.apache.iotdb.db.utils.datastructure.MemPointIteratorFactory;
import org.apache.iotdb.db.utils.datastructure.TVList;
import org.apache.tsfile.block.column.ColumnBuilder;
@@ -38,7 +39,6 @@ import org.apache.tsfile.read.common.TimeRange;
import org.apache.tsfile.read.common.block.TsBlock;
import org.apache.tsfile.read.common.block.TsBlockBuilder;
import org.apache.tsfile.read.reader.IPointReader;
-import org.apache.tsfile.utils.BitMap;
import org.apache.tsfile.utils.TsPrimitiveType;
import org.apache.tsfile.write.UnSupportedDataTypeException;
import org.apache.tsfile.write.schema.IMeasurementSchema;
@@ -46,13 +46,10 @@ import org.apache.tsfile.write.schema.IMeasurementSchema;
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
-import static org.apache.iotdb.db.utils.ModificationUtils.isPointDeleted;
-
public class AlignedReadOnlyMemChunk extends ReadOnlyMemChunk {
private final String timeChunkName;
@@ -76,7 +73,7 @@ public class AlignedReadOnlyMemChunk extends ReadOnlyMemChunk {
// When we select two of time series [s1, s3], the column index list should be [0, 2]
private final List<Integer> columnIndexList;
- private MergeSortAlignedTVListIterator timeValuePairIterator;
+ private MemPointIterator timeValuePairIterator;
/**
* The constructor for Aligned type.
@@ -94,7 +91,6 @@ public class AlignedReadOnlyMemChunk extends ReadOnlyMemChunk {
Map<TVList, Integer> alignedTvListQueryMap,
List<List<TimeRange>> valueColumnsDeletionList) {
super(context);
- this.pageOffsetsList = new ArrayList<>();
this.timeChunkName = schema.getMeasurementId();
this.valueChunkNames = schema.getSubMeasurementsList();
this.dataTypes = schema.getSubMeasurementsTSDataTypeList();
@@ -126,8 +122,10 @@ public class AlignedReadOnlyMemChunk extends ReadOnlyMemChunk {
Statistics.getStatsByType(TSDataType.VECTOR);
IChunkMetadata timeChunkMetadata =
new ChunkMetadata(timeChunkName, TSDataType.VECTOR, 0, chunkTimeStatistics);
- Statistics<? extends Serializable>[] chunkValueStatistics =
- new Statistics[valueChunkNames.size()];
+ Statistics<? extends Serializable>[] chunkValueStatistics = new Statistics[dataTypes.size()];
+ for (int column = 0; column < dataTypes.size(); column++) {
+ chunkValueStatistics[column] = Statistics.getStatsByType(dataTypes.get(column));
+ }
// create MergeSortAlignedTVListIterator
List<AlignedTVList> alignedTvLists =
@@ -136,134 +134,119 @@ public class AlignedReadOnlyMemChunk extends ReadOnlyMemChunk {
.collect(Collectors.toList());
timeValuePairIterator =
- new MergeSortAlignedTVListIterator(
- alignedTvLists, dataTypes, columnIndexList, floatPrecision, encodingList);
- int[] alignedTvListOffsets = timeValuePairIterator.getAlignedTVListOffsets();
-
- int pointsInChunk = 0;
- int[] timeDeleteCursor = new int[] {0};
- List<int[]> valueColumnDeleteCursor = new ArrayList<>();
- if (valueColumnsDeletionList != null) {
- valueColumnsDeletionList.forEach(x -> valueColumnDeleteCursor.add(new int[] {0}));
- }
-
- Statistics<? extends Serializable> pageTimeStats = null;
- while (timeValuePairIterator.hasNextTimeValuePair()) {
- TimeValuePair tvPair = timeValuePairIterator.currentTimeValuePair();
- TsPrimitiveType[] values = tvPair.getValue().getVector();
-
- // ignore all-null row
- BitMap bitMap = new BitMap(dataTypes.size());
- for (int columnIndex = 0; columnIndex < dataTypes.size(); columnIndex++) {
- if (values[columnIndex] == null) {
- bitMap.mark(columnIndex);
- } else if (valueColumnsDeletionList != null
- && isPointDeleted(
- tvPair.getTimestamp(),
- valueColumnsDeletionList.get(columnIndex),
- valueColumnDeleteCursor.get(columnIndex))) {
- values[columnIndex] = null;
- bitMap.mark(columnIndex);
- }
- }
- if (bitMap.isAllMarked()) {
- timeValuePairIterator.step();
- continue;
- }
-
+ MemPointIteratorFactory.create(
+ dataTypes,
+ columnIndexList,
+ alignedTvLists,
+ valueColumnsDeletionList,
+ floatPrecision,
+ encodingList);
+
+ while (timeValuePairIterator.hasNextBatch()) {
// create pageTimeStatistics and pageValueStatistics for new page
- if (pointsInChunk % MAX_NUMBER_OF_POINTS_IN_PAGE == 0) {
- Statistics<? extends Serializable> pageTimeStatistics =
- Statistics.getStatsByType(TSDataType.VECTOR);
- pageTimeStatistics.setEmpty(false);
- timeStatisticsList.add(pageTimeStatistics);
- pageTimeStats = timeStatisticsList.get(timeStatisticsList.size() - 1);
-
- Statistics<? extends Serializable>[] pageValueStatistics =
- new Statistics[valueChunkNames.size()];
- Arrays.fill(pageValueStatistics, null);
- valueStatisticsList.add(pageValueStatistics);
- pageOffsetsList.add(Arrays.copyOf(alignedTvListOffsets, alignedTvListOffsets.length));
+ Statistics<? extends Serializable> pageTimeStatistics =
+ Statistics.getStatsByType(TSDataType.VECTOR);
+ pageTimeStatistics.setEmpty(false);
+ timeStatisticsList.add(pageTimeStatistics);
+
+ Statistics<? extends Serializable>[] pageValueStatistics = new Statistics[dataTypes.size()];
+ for (int column = 0; column < dataTypes.size(); column++) {
+ pageValueStatistics[column] = Statistics.getStatsByType(dataTypes.get(column));
}
+ valueStatisticsList.add(pageValueStatistics);
- pageTimeStats.update(tvPair.getTimestamp());
- chunkTimeStatistics.update(tvPair.getTimestamp());
-
- Statistics<? extends Serializable>[] pageValuesStats =
- valueStatisticsList.get(valueStatisticsList.size() - 1);
- TsPrimitiveType[] primitiveValues = tvPair.getValue().getVector();
- for (int column = 0; column < primitiveValues.length; column++) {
- if (primitiveValues[column] == null) {
- continue;
- }
-
- if (pageValuesStats[column] == null) {
- Statistics<? extends Serializable> valueStatistics =
- Statistics.getStatsByType(dataTypes.get(column));
- pageValuesStats[column] = valueStatistics;
- }
- if (chunkValueStatistics[column] == null) {
- Statistics<? extends Serializable> chunkValueStats =
- Statistics.getStatsByType(dataTypes.get(column));
- chunkValueStatistics[column] = chunkValueStats;
- }
-
+ TsBlock tsBlock = timeValuePairIterator.nextBatch();
+ // time column
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ pageTimeStatistics.update(tsBlock.getTimeByIndex(i));
+ chunkTimeStatistics.update(tsBlock.getTimeByIndex(i));
+ }
+ // value columns
+ for (int column = 0; column < tsBlock.getValueColumnCount(); column++) {
+ Statistics<? extends Serializable> pageValueStats =
+ Statistics.getStatsByType(dataTypes.get(column));
switch (dataTypes.get(column)) {
case BOOLEAN:
- pageValuesStats[column].update(
- tvPair.getTimestamp(), primitiveValues[column].getBoolean());
- chunkValueStatistics[column].update(
- tvPair.getTimestamp(), primitiveValues[column].getBoolean());
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ if (tsBlock.getColumn(column).isNull(i)) {
+ continue;
+ }
+ pageValueStats.update(
+ tsBlock.getTimeByIndex(i), tsBlock.getColumn(column).getBoolean(i));
+ chunkValueStatistics[column].update(
+ tsBlock.getTimeByIndex(i), tsBlock.getColumn(column).getBoolean(i));
+ }
break;
case INT32:
case DATE:
- pageValuesStats[column].update(tvPair.getTimestamp(), primitiveValues[column].getInt());
- chunkValueStatistics[column].update(
- tvPair.getTimestamp(), primitiveValues[column].getInt());
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ if (tsBlock.getColumn(column).isNull(i)) {
+ continue;
+ }
+ pageValueStats.update(tsBlock.getTimeByIndex(i), tsBlock.getColumn(column).getInt(i));
+ chunkValueStatistics[column].update(
+ tsBlock.getTimeByIndex(i), tsBlock.getColumn(column).getInt(i));
+ }
break;
case INT64:
case TIMESTAMP:
- pageValuesStats[column].update(
- tvPair.getTimestamp(), primitiveValues[column].getLong());
- chunkValueStatistics[column].update(
- tvPair.getTimestamp(), primitiveValues[column].getLong());
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ if (tsBlock.getColumn(column).isNull(i)) {
+ continue;
+ }
+ pageValueStats.update(
+ tsBlock.getTimeByIndex(i), tsBlock.getColumn(column).getLong(i));
+ chunkValueStatistics[column].update(
+ tsBlock.getTimeByIndex(i), tsBlock.getColumn(column).getLong(i));
+ }
break;
case FLOAT:
- pageValuesStats[column].update(
- tvPair.getTimestamp(), primitiveValues[column].getFloat());
- chunkValueStatistics[column].update(
- tvPair.getTimestamp(), primitiveValues[column].getFloat());
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ if (tsBlock.getColumn(column).isNull(i)) {
+ continue;
+ }
+ pageValueStats.update(
+ tsBlock.getTimeByIndex(i), tsBlock.getColumn(column).getFloat(i));
+ chunkValueStatistics[column].update(
+ tsBlock.getTimeByIndex(i), tsBlock.getColumn(column).getFloat(i));
+ }
break;
case DOUBLE:
- pageValuesStats[column].update(
- tvPair.getTimestamp(), primitiveValues[column].getDouble());
- chunkValueStatistics[column].update(
- tvPair.getTimestamp(), primitiveValues[column].getDouble());
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ if (tsBlock.getColumn(column).isNull(i)) {
+ continue;
+ }
+ pageValueStats.update(
+ tsBlock.getTimeByIndex(i), tsBlock.getColumn(column).getDouble(i));
+ chunkValueStatistics[column].update(
+ tsBlock.getTimeByIndex(i), tsBlock.getColumn(column).getDouble(i));
+ }
break;
case TEXT:
case BLOB:
case STRING:
- pageValuesStats[column].update(
- tvPair.getTimestamp(), primitiveValues[column].getBinary());
- chunkValueStatistics[column].update(
- tvPair.getTimestamp(), primitiveValues[column].getBinary());
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ if (tsBlock.getColumn(column).isNull(i)) {
+ continue;
+ }
+ pageValueStats.update(
+ tsBlock.getTimeByIndex(i), tsBlock.getColumn(column).getBinary(i));
+ chunkValueStatistics[column].update(
+ tsBlock.getTimeByIndex(i), tsBlock.getColumn(column).getBinary(i));
+ }
break;
default:
throw new UnSupportedDataTypeException(
String.format("Data type %s is not supported.", dataTypes.get(column)));
}
+ pageValueStatistics[column] = pageValueStats.isEmpty() ? null : pageValueStats;
}
- timeValuePairIterator.step();
- pointsInChunk++;
}
- pageOffsetsList.add(Arrays.copyOf(alignedTvListOffsets, alignedTvListOffsets.length));
- chunkTimeStatistics.setEmpty(pointsInChunk == 0);
-
// aligned chunk meta
List<IChunkMetadata> valueChunkMetadataList = new ArrayList<>();
- for (int column = 0; column < valueChunkNames.size(); column++) {
- if (chunkValueStatistics[column] != null) {
+ for (int column = 0; column < dataTypes.size(); column++) {
+ if (!chunkValueStatistics[column].isEmpty()) {
IChunkMetadata valueChunkMetadata =
new ChunkMetadata(
valueChunkNames.get(column),
@@ -324,37 +307,20 @@ public class AlignedReadOnlyMemChunk extends ReadOnlyMemChunk {
alignedTvListQueryMap.keySet().stream()
.map(x -> (AlignedTVList) x)
.collect(Collectors.toList());
- MergeSortAlignedTVListIterator timeValuePairIterator =
- new MergeSortAlignedTVListIterator(
- alignedTvLists, dataTypes, columnIndexList, floatPrecision, encodingList);
-
- int[] timeDeleteCursor = new int[] {0};
- List<int[]> valueColumnDeleteCursor = new ArrayList<>();
- if (valueColumnsDeletionList != null) {
- valueColumnsDeletionList.forEach(x -> valueColumnDeleteCursor.add(new int[] {0}));
- }
+ MemPointIterator timeValuePairIterator =
+ MemPointIteratorFactory.create(
+ dataTypes,
+ columnIndexList,
+ alignedTvLists,
+ valueColumnsDeletionList,
+ floatPrecision,
+ encodingList);
while (timeValuePairIterator.hasNextTimeValuePair()) {
TimeValuePair tvPair = timeValuePairIterator.nextTimeValuePair();
TsPrimitiveType[] values = tvPair.getValue().getVector();
- BitMap bitMap = new BitMap(values.length);
- for (int columnIndex = 0; columnIndex < values.length; columnIndex++) {
- if (values[columnIndex] == null) {
- bitMap.mark(columnIndex);
- } else if (valueColumnsDeletionList != null
- && isPointDeleted(
- tvPair.getTimestamp(),
- valueColumnsDeletionList.get(columnIndex),
- valueColumnDeleteCursor.get(columnIndex))) {
- values[columnIndex] = null;
- bitMap.mark(columnIndex);
- }
- }
- if (bitMap.isAllMarked()) {
- continue;
- }
-
+ // time column
builder.getTimeColumnBuilder().writeLong(tvPair.getTimestamp());
// value columns
for (int columnIndex = 0; columnIndex < values.length; columnIndex++) {
@@ -403,18 +369,10 @@ public class AlignedReadOnlyMemChunk extends ReadOnlyMemChunk {
return floatPrecision;
}
- public List<Integer> getColumnIndexList() {
- return columnIndexList;
- }
-
public List<List<TimeRange>> getValueColumnsDeletionList() {
return valueColumnsDeletionList;
}
- public List<TSEncoding> getEncodingList() {
- return encodingList;
- }
-
public List<TSDataType> getDataTypes() {
return dataTypes;
}
@@ -427,7 +385,7 @@ public class AlignedReadOnlyMemChunk extends ReadOnlyMemChunk {
return valueStatisticsList;
}
- public MergeSortAlignedTVListIterator getMergeSortAlignedTVListIterator() {
+ public MemPointIterator getMemPointIterator() {
return timeValuePairIterator;
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
index 00e6240723c..49be365336f 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
@@ -27,17 +27,17 @@ import org.apache.iotdb.db.queryengine.plan.planner.memory.MemoryReservationMana
import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView;
import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALWriteUtils;
import org.apache.iotdb.db.utils.datastructure.AlignedTVList;
-import org.apache.iotdb.db.utils.datastructure.MergeSortAlignedTVListIterator;
+import org.apache.iotdb.db.utils.datastructure.MemPointIterator;
+import org.apache.iotdb.db.utils.datastructure.MemPointIteratorFactory;
import org.apache.iotdb.db.utils.datastructure.TVList;
import org.apache.tsfile.common.conf.TSFileDescriptor;
import org.apache.tsfile.enums.TSDataType;
-import org.apache.tsfile.read.TimeValuePair;
import org.apache.tsfile.read.common.TimeRange;
+import org.apache.tsfile.read.common.block.TsBlock;
import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.utils.BitMap;
import org.apache.tsfile.utils.Pair;
-import org.apache.tsfile.utils.TsPrimitiveType;
import org.apache.tsfile.write.UnSupportedDataTypeException;
import org.apache.tsfile.write.chunk.AlignedChunkWriterImpl;
import org.apache.tsfile.write.chunk.IChunkWriter;
@@ -297,7 +297,6 @@ public class AlignedWritableMemChunk implements IWritableMemChunk {
private void filterDeletedTimeStamp(
AlignedTVList alignedTVList,
List<List<TimeRange>> valueColumnsDeletionList,
- boolean ignoreAllNullRows,
Map<Long, BitMap> timestampWithBitmap) {
BitMap allValueColDeletedMap = alignedTVList.getAllValueColDeletedMap();
@@ -331,7 +330,7 @@ public class AlignedWritableMemChunk implements IWritableMemChunk {
}
// skip all-null row
- if (ignoreAllNullRows && bitMap.isAllMarked()) {
+ if (bitMap.isAllMarked()) {
continue;
}
timestampWithBitmap.put(timestamp, bitMap);
@@ -339,13 +338,12 @@ public class AlignedWritableMemChunk implements IWritableMemChunk {
}
}
- public long[] getFilteredTimestamp(
- List<List<TimeRange>> deletionList, List<BitMap> bitMaps, boolean ignoreAllNullRows) {
+ public long[] getFilteredTimestamp(List<List<TimeRange>> deletionList, List<BitMap> bitMaps) {
Map<Long, BitMap> timestampWithBitmap = new TreeMap<>();
- filterDeletedTimeStamp(list, deletionList, ignoreAllNullRows, timestampWithBitmap);
+ filterDeletedTimeStamp(list, deletionList, timestampWithBitmap);
for (AlignedTVList alignedTVList : sortedList) {
- filterDeletedTimeStamp(alignedTVList, deletionList, ignoreAllNullRows, timestampWithBitmap);
+ filterDeletedTimeStamp(alignedTVList, deletionList, timestampWithBitmap);
}
List<Long> filteredTimestamps = new ArrayList<>();
@@ -446,27 +444,6 @@ public class AlignedWritableMemChunk implements IWritableMemChunk {
@Override
public synchronized void sortTvListForFlush() {
- AlignedTVList cloneList = null;
- list.lockQueryList();
- try {
- if (!list.isSorted() && !list.getQueryContextList().isEmpty()) {
- QueryContext firstQuery = list.getQueryContextList().get(0);
- // reserve query memory
- if (firstQuery instanceof FragmentInstanceContext) {
- MemoryReservationManager memoryReservationManager =
- ((FragmentInstanceContext) firstQuery).getMemoryReservationContext();
- memoryReservationManager.reserveMemoryCumulatively(list.calculateRamSize());
- }
- list.setOwnerQuery(firstQuery);
- cloneList = list.clone();
- }
- } finally {
- list.unlockQueryList();
- }
- if (cloneList != null) {
- setWorkingTVList(cloneList);
- }
-
if (!list.isSorted()) {
list.sort();
}
@@ -713,69 +690,82 @@ public class AlignedWritableMemChunk implements IWritableMemChunk {
// create MergeSortAlignedTVListIterator.
List<AlignedTVList> alignedTvLists = new ArrayList<>(sortedList);
alignedTvLists.add(list);
- MergeSortAlignedTVListIterator timeValuePairIterator =
- new MergeSortAlignedTVListIterator(alignedTvLists, dataTypes, null, null, null);
+ MemPointIterator timeValuePairIterator =
+ MemPointIteratorFactory.create(dataTypes, null, alignedTvLists);
int pointNumInPage = 0;
int pointNumInChunk = 0;
long[] times = new long[MAX_NUMBER_OF_POINTS_IN_PAGE];
- while (timeValuePairIterator.hasNextTimeValuePair()) {
- TimeValuePair tvPair = timeValuePairIterator.nextTimeValuePair();
- times[pointNumInPage] = tvPair.getTimestamp();
- TsPrimitiveType[] values = tvPair.getValue().getVector();
-
- for (int columnIndex = 0; columnIndex < values.length; columnIndex++) {
- ValueChunkWriter valueChunkWriter =
- alignedChunkWriter.getValueChunkWriterByIndex(columnIndex);
- boolean isNull = values[columnIndex] == null;
- switch (schemaList.get(columnIndex).getType()) {
- case BOOLEAN:
- valueChunkWriter.write(tvPair.getTimestamp(), values[columnIndex].getBoolean(), isNull);
- break;
- case INT32:
- case DATE:
- valueChunkWriter.write(tvPair.getTimestamp(), values[columnIndex].getInt(), isNull);
- break;
- case INT64:
- case TIMESTAMP:
- valueChunkWriter.write(tvPair.getTimestamp(), values[columnIndex].getLong(), isNull);
- break;
- case FLOAT:
- valueChunkWriter.write(tvPair.getTimestamp(), values[columnIndex].getFloat(), isNull);
- break;
- case DOUBLE:
- valueChunkWriter.write(tvPair.getTimestamp(), values[columnIndex].getDouble(), isNull);
- break;
- case TEXT:
- case BLOB:
- case STRING:
- valueChunkWriter.write(tvPair.getTimestamp(), values[columnIndex].getBinary(), isNull);
- break;
- default:
- break;
- }
+ while (timeValuePairIterator.hasNextBatch()) {
+ TsBlock tsBlock = timeValuePairIterator.nextBatch();
+ if (tsBlock == null) {
+ continue;
}
+ for (int rowIndex = 0; rowIndex < tsBlock.getPositionCount(); rowIndex++) {
+ long time = tsBlock.getTimeByIndex(rowIndex);
+ times[pointNumInPage] = time;
- pointNumInPage++;
- pointNumInChunk++;
+ for (int columnIndex = 0; columnIndex < dataTypes.size(); columnIndex++) {
+ ValueChunkWriter valueChunkWriter =
+ alignedChunkWriter.getValueChunkWriterByIndex(columnIndex);
+ if (tsBlock.getColumn(columnIndex).isNull(rowIndex)) {
+ valueChunkWriter.write(time, null, true);
+ continue;
+ }
+ switch (schemaList.get(columnIndex).getType()) {
+ case BOOLEAN:
+ valueChunkWriter.write(
+ time, tsBlock.getColumn(columnIndex).getBoolean(rowIndex), false);
+ break;
+ case INT32:
+ case DATE:
+ valueChunkWriter.write(time, tsBlock.getColumn(columnIndex).getInt(rowIndex), false);
+ break;
+ case INT64:
+ case TIMESTAMP:
+ valueChunkWriter.write(time, tsBlock.getColumn(columnIndex).getLong(rowIndex), false);
+ break;
+ case FLOAT:
+ valueChunkWriter.write(
+ time, tsBlock.getColumn(columnIndex).getFloat(rowIndex), false);
+ break;
+ case DOUBLE:
+ valueChunkWriter.write(
+ time, tsBlock.getColumn(columnIndex).getDouble(rowIndex), false);
+ break;
+ case TEXT:
+ case BLOB:
+ case STRING:
+ valueChunkWriter.write(
+ time, tsBlock.getColumn(columnIndex).getBinary(rowIndex), false);
+ break;
+ default:
+ break;
+ }
+ }
+ pointNumInPage++;
+ pointNumInChunk++;
- if (pointNumInPage == MAX_NUMBER_OF_POINTS_IN_PAGE
- || pointNumInChunk >= maxNumberOfPointsInChunk) {
- alignedChunkWriter.write(times, pointNumInPage, 0);
- pointNumInPage = 0;
- }
+ // new page
+ if (pointNumInPage == MAX_NUMBER_OF_POINTS_IN_PAGE
+ || pointNumInChunk >= maxNumberOfPointsInChunk) {
+ alignedChunkWriter.write(times, pointNumInPage, 0);
+ pointNumInPage = 0;
+ }
- if (pointNumInChunk >= maxNumberOfPointsInChunk) {
- alignedChunkWriter.sealCurrentPage();
- alignedChunkWriter.clearPageWriter();
- try {
- ioTaskQueue.put(alignedChunkWriter);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
+ // new chunk
+ if (pointNumInChunk >= maxNumberOfPointsInChunk) {
+ alignedChunkWriter.sealCurrentPage();
+ alignedChunkWriter.clearPageWriter();
+ try {
+ ioTaskQueue.put(alignedChunkWriter);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ alignedChunkWriter = new AlignedChunkWriterImpl(schemaList);
+ pointNumInChunk = 0;
}
- alignedChunkWriter = new AlignedChunkWriterImpl(schemaList);
- pointNumInChunk = 0;
}
}
@@ -842,7 +832,7 @@ public class AlignedWritableMemChunk implements IWritableMemChunk {
@Override
public boolean isEmpty() {
- return list.rowCount() == 0;
+ return rowCount() == 0;
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java
index a22f1da890a..22fac1604ca 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java
@@ -23,8 +23,8 @@ import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext;
import org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk.MemChunkLoader;
-import org.apache.iotdb.db.utils.MathUtils;
-import org.apache.iotdb.db.utils.datastructure.MergeSortTVListIterator;
+import org.apache.iotdb.db.utils.datastructure.MemPointIterator;
+import org.apache.iotdb.db.utils.datastructure.MemPointIteratorFactory;
import org.apache.iotdb.db.utils.datastructure.TVList;
import org.apache.tsfile.common.conf.TSFileDescriptor;
@@ -46,13 +46,10 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
-import static org.apache.iotdb.db.utils.ModificationUtils.isPointDeleted;
-
/**
* ReadOnlyMemChunk is a snapshot of the working MemTable and flushing memtable in the memory used
* for querying.
@@ -77,15 +74,10 @@ public class ReadOnlyMemChunk {
// we need to collect page statistic.
private List<Statistics<? extends Serializable>> pageStatisticsList;
- // page offsets in the MemChunk. For example, there are two TVList/AlignedTVList. TVList1: 1,
- // 2, 3, 4, 5, 6, 7 TVList2: 1, 3, 5, 7, 8, 9 If each page has 3 points in it. The page offset
- // lists would be [[0, 0], [3, 2], [6, 3], [7, 6]]. [7, 6] means the end.
- protected List<int[]> pageOffsetsList;
-
// TVList and its rowCount during query
private Map<TVList, Integer> tvListQueryMap;
- private MergeSortTVListIterator timeValuePairIterator;
+ private MemPointIterator timeValuePairIterator;
protected final int MAX_NUMBER_OF_POINTS_IN_PAGE =
TSFileDescriptor.getInstance().getConfig().getMaxNumberOfPointsInPage();
@@ -128,7 +120,6 @@ public class ReadOnlyMemChunk {
this.deletionList = deletionList;
this.tvListQueryMap = tvListQueryMap;
this.pageStatisticsList = new ArrayList<>();
- this.pageOffsetsList = new ArrayList<>();
this.context.addTVListToSet(tvListQueryMap);
}
@@ -145,58 +136,69 @@ public class ReadOnlyMemChunk {
public void initChunkMetaFromTvLists() {
// create chunk statistics
Statistics<? extends Serializable> chunkStatistics = Statistics.getStatsByType(dataType);
- int pointsInChunk = 0;
- int[] deleteCursor = {0};
List<TVList> tvLists = new ArrayList<>(tvListQueryMap.keySet());
- timeValuePairIterator = new MergeSortTVListIterator(tvLists, floatPrecision, encoding);
- int[] tvListOffsets = timeValuePairIterator.getTVListOffsets();
- while (timeValuePairIterator.hasNextTimeValuePair()) {
- if (pointsInChunk % MAX_NUMBER_OF_POINTS_IN_PAGE == 0) {
- Statistics<? extends Serializable> stats = Statistics.getStatsByType(dataType);
- pageStatisticsList.add(stats);
- pageOffsetsList.add(Arrays.copyOf(tvListOffsets, tvListOffsets.length));
- }
- TimeValuePair tvPair = timeValuePairIterator.nextTimeValuePair();
- if (!isPointDeleted(tvPair.getTimestamp(), deletionList, deleteCursor)) {
- Statistics<? extends Serializable> pageStatistics =
- pageStatisticsList.get(pageStatisticsList.size() - 1);
+ timeValuePairIterator =
+ MemPointIteratorFactory.create(dataType, tvLists, deletionList, floatPrecision, encoding);
+ while (timeValuePairIterator.hasNextBatch()) {
+ // statistics for current batch
+ Statistics<? extends Serializable> pageStatistics = Statistics.getStatsByType(dataType);
+ pageStatisticsList.add(pageStatistics);
+
+ TsBlock tsBlock = timeValuePairIterator.nextBatch();
+ if (!tsBlock.isEmpty()) {
switch (dataType) {
case BOOLEAN:
- chunkStatistics.update(tvPair.getTimestamp(), tvPair.getValue().getBoolean());
- pageStatistics.update(tvPair.getTimestamp(), tvPair.getValue().getBoolean());
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ long time = tsBlock.getTimeByIndex(i);
+ chunkStatistics.update(time, tsBlock.getColumn(0).getBoolean(i));
+ pageStatistics.update(time, tsBlock.getColumn(0).getBoolean(i));
+ }
break;
case INT32:
case DATE:
- chunkStatistics.update(tvPair.getTimestamp(), tvPair.getValue().getInt());
- pageStatistics.update(tvPair.getTimestamp(), tvPair.getValue().getInt());
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ long time = tsBlock.getTimeByIndex(i);
+ chunkStatistics.update(time, tsBlock.getColumn(0).getInt(i));
+ pageStatistics.update(time, tsBlock.getColumn(0).getInt(i));
+ }
break;
case INT64:
case TIMESTAMP:
- chunkStatistics.update(tvPair.getTimestamp(), tvPair.getValue().getLong());
- pageStatistics.update(tvPair.getTimestamp(), tvPair.getValue().getLong());
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ long time = tsBlock.getTimeByIndex(i);
+ chunkStatistics.update(time, tsBlock.getColumn(0).getLong(i));
+ pageStatistics.update(time, tsBlock.getColumn(0).getLong(i));
+ }
break;
case FLOAT:
- chunkStatistics.update(tvPair.getTimestamp(), tvPair.getValue().getFloat());
- pageStatistics.update(tvPair.getTimestamp(), tvPair.getValue().getFloat());
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ long time = tsBlock.getTimeByIndex(i);
+ chunkStatistics.update(time, tsBlock.getColumn(0).getFloat(i));
+ pageStatistics.update(time, tsBlock.getColumn(0).getFloat(i));
+ }
break;
case DOUBLE:
- chunkStatistics.update(tvPair.getTimestamp(), tvPair.getValue().getDouble());
- pageStatistics.update(tvPair.getTimestamp(), tvPair.getValue().getDouble());
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ long time = tsBlock.getTimeByIndex(i);
+ chunkStatistics.update(time, tsBlock.getColumn(0).getDouble(i));
+ pageStatistics.update(time, tsBlock.getColumn(0).getDouble(i));
+ }
break;
case TEXT:
case BLOB:
case STRING:
- chunkStatistics.update(tvPair.getTimestamp(), tvPair.getValue().getBinary());
- pageStatistics.update(tvPair.getTimestamp(), tvPair.getValue().getBinary());
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ long time = tsBlock.getTimeByIndex(i);
+ chunkStatistics.update(time, tsBlock.getColumn(0).getBinary(i));
+ pageStatistics.update(time, tsBlock.getColumn(0).getBinary(i));
+ }
break;
default:
throw new UnSupportedDataTypeException(
String.format("Data type %s is not supported.", dataType));
}
}
- pointsInChunk++;
}
- pageOffsetsList.add(Arrays.copyOf(tvListOffsets, tvListOffsets.length));
// chunk meta
IChunkMetadata metaData = new ChunkMetadata(measurementUid, dataType, 0, chunkStatistics);
@@ -242,53 +244,41 @@ public class ReadOnlyMemChunk {
// read all data in memory chunk and write to tsblock
private void writeValidValuesIntoTsBlock(TsBlockBuilder builder) throws IOException {
- int[] deleteCursor = {0};
List<TVList> tvLists = new ArrayList<>(tvListQueryMap.keySet());
- IPointReader timeValuePairIterator =
- new MergeSortTVListIterator(tvLists, floatPrecision, encoding);
+ MemPointIterator timeValuePairIterator =
+ MemPointIteratorFactory.create(
+ getDataType(), tvLists, deletionList, floatPrecision, encoding);
while (timeValuePairIterator.hasNextTimeValuePair()) {
TimeValuePair tvPair = timeValuePairIterator.nextTimeValuePair();
- if (!isPointDeleted(tvPair.getTimestamp(), deletionList, deleteCursor)) {
- builder.getTimeColumnBuilder().writeLong(tvPair.getTimestamp());
- switch (dataType) {
- case BOOLEAN:
- builder.getColumnBuilder(0).writeBoolean(tvPair.getValue().getBoolean());
- break;
- case INT32:
- case DATE:
- builder.getColumnBuilder(0).writeInt(tvPair.getValue().getInt());
- break;
- case INT64:
- case TIMESTAMP:
- builder.getColumnBuilder(0).writeLong(tvPair.getValue().getLong());
- break;
- case FLOAT:
- float fv = tvPair.getValue().getFloat();
- if (!Float.isNaN(fv)
- && (encoding == TSEncoding.RLE || encoding == TSEncoding.TS_2DIFF)) {
- fv = MathUtils.roundWithGivenPrecision(fv, floatPrecision);
- }
- builder.getColumnBuilder(0).writeFloat(fv);
- break;
- case DOUBLE:
- double dv = tvPair.getValue().getDouble();
- if (!Double.isNaN(dv)
- && (encoding == TSEncoding.RLE || encoding == TSEncoding.TS_2DIFF)) {
- dv = MathUtils.roundWithGivenPrecision(dv, floatPrecision);
- }
- builder.getColumnBuilder(0).writeDouble(dv);
- break;
- case TEXT:
- case STRING:
- case BLOB:
- builder.getColumnBuilder(0).writeBinary(tvPair.getValue().getBinary());
- break;
- default:
- break;
- }
- builder.declarePosition();
+ builder.getTimeColumnBuilder().writeLong(tvPair.getTimestamp());
+ switch (dataType) {
+ case BOOLEAN:
+ builder.getColumnBuilder(0).writeBoolean(tvPair.getValue().getBoolean());
+ break;
+ case INT32:
+ case DATE:
+ builder.getColumnBuilder(0).writeInt(tvPair.getValue().getInt());
+ break;
+ case INT64:
+ case TIMESTAMP:
+ builder.getColumnBuilder(0).writeLong(tvPair.getValue().getLong());
+ break;
+ case FLOAT:
+ builder.getColumnBuilder(0).writeFloat(tvPair.getValue().getFloat());
+ break;
+ case DOUBLE:
+ builder.getColumnBuilder(0).writeDouble(tvPair.getValue().getDouble());
+ break;
+ case TEXT:
+ case STRING:
+ case BLOB:
+ builder.getColumnBuilder(0).writeBinary(tvPair.getValue().getBinary());
+ break;
+ default:
+ break;
}
+ builder.declarePosition();
}
}
@@ -320,10 +310,6 @@ public class ReadOnlyMemChunk {
return pageStatisticsList;
}
- public List<int[]> getPageOffsetsList() {
- return pageOffsetsList;
- }
-
public QueryContext getContext() {
return context;
}
@@ -333,7 +319,7 @@ public class ReadOnlyMemChunk {
return null;
}
- public MergeSortTVListIterator getMergeSortTVListIterator() {
+ public MemPointIterator getMemPointIterator() {
return timeValuePairIterator;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java
index bc26cf72b00..3169c878486 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java
@@ -25,15 +25,16 @@ import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext;
import org.apache.iotdb.db.queryengine.plan.planner.memory.MemoryReservationManager;
import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView;
import org.apache.iotdb.db.utils.ModificationUtils;
-import org.apache.iotdb.db.utils.datastructure.MergeSortTVListIterator;
+import org.apache.iotdb.db.utils.datastructure.MemPointIterator;
+import org.apache.iotdb.db.utils.datastructure.MemPointIteratorFactory;
import org.apache.iotdb.db.utils.datastructure.TVList;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.read.TimeValuePair;
import org.apache.tsfile.read.common.TimeRange;
+import org.apache.tsfile.read.common.block.TsBlock;
import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.utils.BitMap;
-import org.apache.tsfile.utils.Pair;
import org.apache.tsfile.write.UnSupportedDataTypeException;
import org.apache.tsfile.write.chunk.ChunkWriterImpl;
import org.apache.tsfile.write.schema.IMeasurementSchema;
@@ -304,29 +305,6 @@ public class WritableMemChunk implements IWritableMemChunk {
@Override
public synchronized void sortTvListForFlush() {
- TVList cloneList = null;
- list.lockQueryList();
- try {
- // During flush, if the working TVList is not sorted and referenced by some query, we need to
- // clone it. The query still refer to original unsorted TVList.
- if (!list.isSorted() && !list.getQueryContextList().isEmpty()) {
- QueryContext firstQuery = list.getQueryContextList().get(0);
- // reserve query memory
- if (firstQuery instanceof FragmentInstanceContext) {
- MemoryReservationManager memoryReservationManager =
- ((FragmentInstanceContext) firstQuery).getMemoryReservationContext();
- memoryReservationManager.reserveMemoryCumulatively(list.calculateRamSize());
- }
- list.setOwnerQuery(firstQuery);
- cloneList = list.clone();
- }
- } finally {
- list.unlockQueryList();
- }
- if (cloneList != null) {
- setWorkingTVList(cloneList);
- }
-
if (!list.isSorted()) {
list.sort();
}
@@ -575,49 +553,6 @@ public class WritableMemChunk implements IWritableMemChunk {
}
}
- private Pair<Long, Integer> writeData(
- ChunkWriterImpl chunkWriterImpl,
- TimeValuePair tvPair,
- long dataSizeInCurrentChunk,
- int pointNumInCurrentChunk) {
- switch (schema.getType()) {
- case BOOLEAN:
- chunkWriterImpl.write(tvPair.getTimestamp(), tvPair.getValue().getBoolean());
- dataSizeInCurrentChunk += 8L + 1L;
- break;
- case INT32:
- case DATE:
- chunkWriterImpl.write(tvPair.getTimestamp(), tvPair.getValue().getInt());
- dataSizeInCurrentChunk += 8L + 4L;
- break;
- case INT64:
- case TIMESTAMP:
- chunkWriterImpl.write(tvPair.getTimestamp(), tvPair.getValue().getLong());
- dataSizeInCurrentChunk += 8L + 8L;
- break;
- case FLOAT:
- chunkWriterImpl.write(tvPair.getTimestamp(), tvPair.getValue().getFloat());
- dataSizeInCurrentChunk += 8L + 4L;
- break;
- case DOUBLE:
- chunkWriterImpl.write(tvPair.getTimestamp(), tvPair.getValue().getDouble());
- dataSizeInCurrentChunk += 8L + 8L;
- break;
- case TEXT:
- case BLOB:
- case STRING:
- Binary value = tvPair.getValue().getBinary();
- chunkWriterImpl.write(tvPair.getTimestamp(), value);
- dataSizeInCurrentChunk += 8L + getBinarySize(value);
- break;
- default:
- LOGGER.error("WritableMemChunk does not support data type: {}", schema.getType());
- break;
- }
- pointNumInCurrentChunk++;
- return new Pair<>(dataSizeInCurrentChunk, pointNumInCurrentChunk);
- }
-
@Override
public synchronized void encode(BlockingQueue<Object> ioTaskQueue) {
if (TVLIST_SORT_THRESHOLD == 0) {
@@ -625,50 +560,81 @@ public class WritableMemChunk implements IWritableMemChunk {
return;
}
+ TSDataType tsDataType = schema.getType();
ChunkWriterImpl chunkWriterImpl = createIChunkWriter();
long dataSizeInCurrentChunk = 0;
int pointNumInCurrentChunk = 0;
- // create MergeSortTvListIterator. It need not handle float/double precision here.
+ // create MultiTvListIterator. It need not handle float/double precision here.
List<TVList> tvLists = new ArrayList<>(sortedList);
tvLists.add(list);
- MergeSortTVListIterator timeValuePairIterator = new MergeSortTVListIterator(tvLists);
+ MemPointIterator timeValuePairIterator =
+ MemPointIteratorFactory.create(schema.getType(), tvLists);
- TimeValuePair prevTvPair = null;
- while (timeValuePairIterator.hasNextTimeValuePair()) {
- TimeValuePair currTvPair = timeValuePairIterator.nextTimeValuePair();
- if (prevTvPair == null) {
- prevTvPair = currTvPair;
+ while (timeValuePairIterator.hasNextBatch()) {
+ TsBlock tsBlock = timeValuePairIterator.nextBatch();
+ if (tsBlock == null) {
continue;
}
- Pair<Long, Integer> updatedStats =
- writeData(chunkWriterImpl, prevTvPair, dataSizeInCurrentChunk, pointNumInCurrentChunk);
- dataSizeInCurrentChunk = updatedStats.left;
- pointNumInCurrentChunk = updatedStats.right;
- prevTvPair = currTvPair;
- if (pointNumInCurrentChunk > MAX_NUMBER_OF_POINTS_IN_CHUNK
- || dataSizeInCurrentChunk > TARGET_CHUNK_SIZE) {
- chunkWriterImpl.sealCurrentPage();
- chunkWriterImpl.clearPageWriter();
- try {
- ioTaskQueue.put(chunkWriterImpl);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
+ for (int rowIndex = 0; rowIndex < tsBlock.getPositionCount(); rowIndex++) {
+ long time = tsBlock.getTimeByIndex(rowIndex);
+ // store last point for SDT
+ if (rowIndex + 1 == tsBlock.getPositionCount() && !timeValuePairIterator.hasNextBatch()) {
+ chunkWriterImpl.setLastPoint(true);
+ }
+
+ switch (tsDataType) {
+ case BOOLEAN:
+ chunkWriterImpl.write(time, tsBlock.getColumn(0).getBoolean(rowIndex));
+ dataSizeInCurrentChunk += 8L + 1L;
+ break;
+ case INT32:
+ case DATE:
+ chunkWriterImpl.write(time, tsBlock.getColumn(0).getInt(rowIndex));
+ dataSizeInCurrentChunk += 8L + 4L;
+ break;
+ case INT64:
+ case TIMESTAMP:
+ chunkWriterImpl.write(time, tsBlock.getColumn(0).getLong(rowIndex));
+ dataSizeInCurrentChunk += 8L + 8L;
+ break;
+ case FLOAT:
+ chunkWriterImpl.write(time, tsBlock.getColumn(0).getFloat(rowIndex));
+ dataSizeInCurrentChunk += 8L + 4L;
+ break;
+ case DOUBLE:
+ chunkWriterImpl.write(time, tsBlock.getColumn(0).getDouble(rowIndex));
+ dataSizeInCurrentChunk += 8L + 8L;
+ break;
+ case TEXT:
+ case BLOB:
+ case STRING:
+ Binary value = tsBlock.getColumn(0).getBinary(rowIndex);
+ chunkWriterImpl.write(time, value);
+ dataSizeInCurrentChunk += 8L + getBinarySize(value);
+ break;
+ default:
+ LOGGER.error("WritableMemChunk does not support data type: {}", tsDataType);
+ break;
+ }
+
+ pointNumInCurrentChunk++;
+ if (pointNumInCurrentChunk > MAX_NUMBER_OF_POINTS_IN_CHUNK
+ || dataSizeInCurrentChunk > TARGET_CHUNK_SIZE) {
+ chunkWriterImpl.sealCurrentPage();
+ chunkWriterImpl.clearPageWriter();
+ try {
+ ioTaskQueue.put(chunkWriterImpl);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ chunkWriterImpl = createIChunkWriter();
+ dataSizeInCurrentChunk = 0;
+ pointNumInCurrentChunk = 0;
}
- chunkWriterImpl = createIChunkWriter();
- dataSizeInCurrentChunk = 0;
- pointNumInCurrentChunk = 0;
}
}
- // last point for SDT
- if (prevTvPair != null) {
- chunkWriterImpl.setLastPoint(true);
- Pair<Long, Integer> updatedStats =
- writeData(chunkWriterImpl, prevTvPair, dataSizeInCurrentChunk, pointNumInCurrentChunk);
- pointNumInCurrentChunk = updatedStats.right;
- }
-
if (pointNumInCurrentChunk != 0) {
chunkWriterImpl.sealCurrentPage();
chunkWriterImpl.clearPageWriter();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkReader.java
index ae27c462df0..093ae9cc327 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkReader.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkReader.java
@@ -20,21 +20,15 @@
package org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk;
import org.apache.iotdb.db.storageengine.dataregion.memtable.AlignedReadOnlyMemChunk;
-import org.apache.iotdb.db.utils.datastructure.MergeSortAlignedTVListIterator;
+import org.apache.iotdb.db.utils.datastructure.MemPointIterator;
-import org.apache.tsfile.block.column.ColumnBuilder;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.statistics.Statistics;
-import org.apache.tsfile.read.TimeValuePair;
import org.apache.tsfile.read.common.BatchData;
-import org.apache.tsfile.read.common.TimeRange;
import org.apache.tsfile.read.common.block.TsBlock;
-import org.apache.tsfile.read.common.block.TsBlockBuilder;
import org.apache.tsfile.read.filter.basic.Filter;
import org.apache.tsfile.read.reader.IChunkReader;
import org.apache.tsfile.read.reader.IPageReader;
-import org.apache.tsfile.utils.BitMap;
-import org.apache.tsfile.utils.TsPrimitiveType;
import java.io.IOException;
import java.io.Serializable;
@@ -42,43 +36,35 @@ import java.util.ArrayList;
import java.util.List;
import java.util.function.Supplier;
-import static org.apache.iotdb.db.utils.ModificationUtils.isPointDeleted;
-
/** To read aligned chunk data in memory. */
public class MemAlignedChunkReader implements IChunkReader {
- private final AlignedReadOnlyMemChunk readableChunk;
- private final MergeSortAlignedTVListIterator timeValuePairIterator;
+ private final MemPointIterator timeValuePairIterator;
private final Filter globalTimeFilter;
private final List<IPageReader> pageReaderList;
public MemAlignedChunkReader(AlignedReadOnlyMemChunk readableChunk, Filter globalTimeFilter) {
- this.readableChunk = readableChunk;
- timeValuePairIterator = readableChunk.getMergeSortAlignedTVListIterator().clone();
+ timeValuePairIterator = readableChunk.getMemPointIterator();
this.globalTimeFilter = globalTimeFilter;
this.pageReaderList = new ArrayList<>();
initAllPageReaders(
readableChunk.getDataTypes(),
readableChunk.getTimeStatisticsList(),
- readableChunk.getValuesStatisticsList(),
- readableChunk.getPageOffsetsList());
+ readableChunk.getValuesStatisticsList());
}
private void initAllPageReaders(
List<TSDataType> tsDataTypes,
List<Statistics<? extends Serializable>> timeStatistics,
- List<Statistics<? extends Serializable>[]> valuesStatistics,
- List<int[]> pageOffsetsList) {
+ List<Statistics<? extends Serializable>[]> valuesStatistics) {
Supplier<TsBlock> tsBlockSupplier = new MemAlignedChunkReader.TsBlockSupplier();
- for (int i = 0; i < timeStatistics.size(); i++) {
+ for (int pageIndex = 0; pageIndex < timeStatistics.size(); pageIndex++) {
MemAlignedPageReader pageReader =
new MemAlignedPageReader(
tsBlockSupplier,
- timeValuePairIterator,
- pageOffsetsList.get(i),
- pageOffsetsList.get(i + 1),
+ pageIndex,
tsDataTypes,
- timeStatistics.get(i),
- valuesStatistics.get(i),
+ timeStatistics.get(pageIndex),
+ valuesStatistics.get(pageIndex),
globalTimeFilter);
this.pageReaderList.add(pageReader);
}
@@ -105,124 +91,17 @@ public class MemAlignedChunkReader implements IChunkReader {
}
class TsBlockSupplier implements Supplier<TsBlock> {
- private int[] pageEndOffsets;
+ private int tsBlockIndex;
public TsBlockSupplier() {}
- public void setPageEndOffsets(int[] pageEndOffsets) {
- this.pageEndOffsets = pageEndOffsets;
+ public void setTsBlockIndex(int tsBlockIndex) {
+ this.tsBlockIndex = tsBlockIndex;
}
@Override
public TsBlock get() {
- return buildTsBlock();
- }
-
- private TsBlock buildTsBlock() {
- try {
- List<TSDataType> tsDataTypes = readableChunk.getDataTypes();
- TsBlockBuilder builder = new TsBlockBuilder(tsDataTypes);
- writeValidValuesIntoTsBlock(builder);
- return builder.build();
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
- private boolean isOutOfMemPageBounds() {
- if (pageEndOffsets == null) {
- return false;
- }
- int[] currTvListOffsets = timeValuePairIterator.getAlignedTVListOffsets();
- for (int i = 0; i < pageEndOffsets.length; i++) {
- if (currTvListOffsets[i] < pageEndOffsets[i]) {
- return false;
- }
- }
- return true;
- }
-
- // read one page and write to tsblock
- private synchronized void writeValidValuesIntoTsBlock(TsBlockBuilder builder) {
- List<TSDataType> tsDataTypes = readableChunk.getDataTypes();
- List<List<TimeRange>> valueColumnsDeletionList = readableChunk.getValueColumnsDeletionList();
-
- int[] timeDeleteCursor = new int[] {0};
- List<int[]> valueColumnDeleteCursor = new ArrayList<>();
- if (valueColumnsDeletionList != null) {
- valueColumnsDeletionList.forEach(x -> valueColumnDeleteCursor.add(new int[] {0}));
- }
-
- while (timeValuePairIterator.hasNextTimeValuePair()) {
- if (isOutOfMemPageBounds()) {
- break;
- }
-
- TimeValuePair tvPair = timeValuePairIterator.nextTimeValuePair();
- TsPrimitiveType[] values = tvPair.getValue().getVector();
-
- BitMap bitMap = new BitMap(tsDataTypes.size());
- for (int columnIndex = 0; columnIndex < tsDataTypes.size(); columnIndex++) {
- if (values[columnIndex] == null) {
- bitMap.mark(columnIndex);
- } else if (valueColumnsDeletionList != null
- && isPointDeleted(
- tvPair.getTimestamp(),
- valueColumnsDeletionList.get(columnIndex),
- valueColumnDeleteCursor.get(columnIndex))) {
- values[columnIndex] = null;
- bitMap.mark(columnIndex);
- }
- }
- if (bitMap.isAllMarked()) {
- timeValuePairIterator.step();
- continue;
- }
-
- // time column
- builder.getTimeColumnBuilder().writeLong(tvPair.getTimestamp());
- // value columns
- for (int columnIndex = 0; columnIndex < values.length; columnIndex++) {
- if (values[columnIndex] == null) {
- builder.getColumnBuilder(columnIndex).appendNull();
- continue;
- }
- ColumnBuilder valueBuilder = builder.getColumnBuilder(columnIndex);
- switch (tsDataTypes.get(columnIndex)) {
- case BOOLEAN:
- valueBuilder.writeBoolean(values[columnIndex].getBoolean());
- break;
- case INT32:
- case DATE:
- valueBuilder.writeInt(values[columnIndex].getInt());
- break;
- case INT64:
- case TIMESTAMP:
- valueBuilder.writeLong(values[columnIndex].getLong());
- break;
- case FLOAT:
- valueBuilder.writeFloat(values[columnIndex].getFloat());
- break;
- case DOUBLE:
- valueBuilder.writeDouble(values[columnIndex].getDouble());
- break;
- case TEXT:
- case BLOB:
- case STRING:
- valueBuilder.writeBinary(values[columnIndex].getBinary());
- break;
- default:
- break;
- }
- }
- builder.declarePosition();
- }
- if (builder.getPositionCount() > readableChunk.getMaxNumberOfPointsInPage()) {
- throw new RuntimeException(
- String.format(
- "Points in current page %d is larger than %d",
- builder.getPositionCount(), readableChunk.getMaxNumberOfPointsInPage()));
- }
+ return timeValuePairIterator.getBatch(tsBlockIndex);
}
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java
index 97108d06f5d..0224f6f7d5c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java
@@ -20,7 +20,6 @@
package org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk;
import org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk.metadata.AlignedPageMetadata;
-import org.apache.iotdb.db.utils.datastructure.MergeSortAlignedTVListIterator;
import org.apache.tsfile.block.column.Column;
import org.apache.tsfile.block.column.ColumnBuilder;
@@ -51,9 +50,7 @@ public class MemAlignedPageReader implements IPageReader {
private TsBlock tsBlock;
private final AlignedPageMetadata pageMetadata;
- private final MergeSortAlignedTVListIterator mergeSortAlignedTVListIterator;
- private final int[] pageStartOffsets;
- private final int[] pageEndOffsets;
+ private final int pageIndex;
private final Supplier<TsBlock> tsBlockSupplier;
private final List<TSDataType> tsDataTypes;
@@ -64,17 +61,13 @@ public class MemAlignedPageReader implements IPageReader {
public MemAlignedPageReader(
Supplier<TsBlock> tsBlockSupplier,
- MergeSortAlignedTVListIterator mergeSortAlignedTVListIterator,
- int[] pageStartOffsets,
- int[] pageEndOffSets,
+ int pageIndex,
List<TSDataType> tsDataTypes,
Statistics<? extends Serializable> timeStatistics,
Statistics<? extends Serializable>[] valueStatistics,
Filter recordFilter) {
this.tsBlockSupplier = tsBlockSupplier;
- this.mergeSortAlignedTVListIterator = mergeSortAlignedTVListIterator;
- this.pageStartOffsets = pageStartOffsets;
- this.pageEndOffsets = pageEndOffSets;
+ this.pageIndex = pageIndex;
this.recordFilter = recordFilter;
this.tsDataTypes = tsDataTypes;
this.pageMetadata = new AlignedPageMetadata(timeStatistics, valueStatistics);
@@ -238,7 +231,7 @@ public class MemAlignedPageReader implements IPageReader {
private void getTsBlock() {
if (tsBlock == null) {
- initializeOffsets();
+ initializeTsBlockIndex();
tsBlock = tsBlockSupplier.get();
if (pageMetadata.getStatistics() == null) {
initPageStatistics();
@@ -246,12 +239,9 @@ public class MemAlignedPageReader implements IPageReader {
}
}
- private void initializeOffsets() {
- if (pageStartOffsets != null) {
- mergeSortAlignedTVListIterator.setAlignedTVListOffsets(pageStartOffsets);
- }
+ private void initializeTsBlockIndex() {
if (tsBlockSupplier instanceof MemAlignedChunkReader.TsBlockSupplier) {
- ((MemAlignedChunkReader.TsBlockSupplier) tsBlockSupplier).setPageEndOffsets(pageEndOffsets);
+ ((MemAlignedChunkReader.TsBlockSupplier) tsBlockSupplier).setTsBlockIndex(pageIndex);
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemChunkReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemChunkReader.java
index 6bc1c3ba5c4..4b21d87d114 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemChunkReader.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemChunkReader.java
@@ -20,15 +20,13 @@
package org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk;
import org.apache.iotdb.db.storageengine.dataregion.memtable.ReadOnlyMemChunk;
-import org.apache.iotdb.db.utils.datastructure.MergeSortTVListIterator;
+import org.apache.iotdb.db.utils.datastructure.MemPointIterator;
-import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.IChunkMetadata;
import org.apache.tsfile.file.metadata.statistics.Statistics;
import org.apache.tsfile.read.TimeValuePair;
import org.apache.tsfile.read.common.BatchData;
import org.apache.tsfile.read.common.block.TsBlock;
-import org.apache.tsfile.read.common.block.TsBlockBuilder;
import org.apache.tsfile.read.filter.basic.Filter;
import org.apache.tsfile.read.reader.IChunkReader;
import org.apache.tsfile.read.reader.IPageReader;
@@ -37,17 +35,13 @@ import org.apache.tsfile.read.reader.IPointReader;
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.List;
import java.util.function.Supplier;
-import static org.apache.iotdb.db.utils.ModificationUtils.isPointDeleted;
-
/** To read chunk data in memory. */
public class MemChunkReader implements IChunkReader, IPointReader {
- private final ReadOnlyMemChunk readableChunk;
- private final MergeSortTVListIterator timeValuePairIterator;
+ private final MemPointIterator timeValuePairIterator;
private final Filter globalTimeFilter;
private final List<IPageReader> pageReaderList;
@@ -55,31 +49,23 @@ public class MemChunkReader implements IChunkReader, IPointReader {
private TimeValuePair cachedTimeValuePair;
public MemChunkReader(ReadOnlyMemChunk readableChunk, Filter globalTimeFilter) {
- this.readableChunk = readableChunk;
- timeValuePairIterator = readableChunk.getMergeSortTVListIterator().clone();
+ this.timeValuePairIterator = readableChunk.getMemPointIterator();
this.globalTimeFilter = globalTimeFilter;
this.pageReaderList = new ArrayList<>();
- initAllPageReaders(
- readableChunk.getChunkMetaData(),
- readableChunk.getPageStatisticsList(),
- readableChunk.getPageOffsetsList());
+ initAllPageReaders(readableChunk.getChunkMetaData(), readableChunk.getPageStatisticsList());
}
private void initAllPageReaders(
- IChunkMetadata metadata,
- List<Statistics<? extends Serializable>> pageStats,
- List<int[]> pageOffsetsList) {
+ IChunkMetadata metadata, List<Statistics<? extends Serializable>> pageStats) {
Supplier<TsBlock> tsBlockSupplier = new TsBlockSupplier();
- for (int i = 0; i < pageStats.size(); i++) {
+ for (int pageIndex = 0; pageIndex < pageStats.size(); pageIndex++) {
MemPageReader pageReader =
new MemPageReader(
tsBlockSupplier,
- timeValuePairIterator,
- pageOffsetsList.get(i),
- pageOffsetsList.get(i + 1),
+ pageIndex,
metadata.getDataType(),
metadata.getMeasurementUid(),
- pageStats.get(i),
+ pageStats.get(pageIndex),
globalTimeFilter);
this.pageReaderList.add(pageReader);
}
@@ -153,90 +139,17 @@ public class MemChunkReader implements IChunkReader, IPointReader {
* TsBlockSupplier object.
*/
class TsBlockSupplier implements Supplier<TsBlock> {
- private int[] pageEndOffsets;
+ private int tsBlockIndex;
public TsBlockSupplier() {}
- public void setPageEndOffsets(int[] pageEndOffsets) {
- this.pageEndOffsets = pageEndOffsets;
+ public void setTsBlockIndex(int tsBlockIndex) {
+ this.tsBlockIndex = tsBlockIndex;
}
@Override
public TsBlock get() {
- return buildTsBlock();
- }
-
- private TsBlock buildTsBlock() {
- try {
- TSDataType tsDataType = readableChunk.getDataType();
- TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(tsDataType));
- writeValidValuesIntoTsBlock(builder);
- return builder.build();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- private boolean isOutOfMemPageBounds() {
- if (pageEndOffsets == null) {
- return false;
- }
- int[] currTvListOffsets = timeValuePairIterator.getTVListOffsets();
- for (int i = 0; i < pageEndOffsets.length; i++) {
- if (currTvListOffsets[i] < pageEndOffsets[i]) {
- return false;
- }
- }
- return true;
- }
-
- // read one page and write to tsblock
- private synchronized void writeValidValuesIntoTsBlock(TsBlockBuilder builder)
- throws IOException {
- TSDataType tsDataType = readableChunk.getDataType();
- int[] deleteCursor = {0};
- while (timeValuePairIterator.hasNextTimeValuePair()) {
- if (isOutOfMemPageBounds()) {
- break;
- }
- TimeValuePair tvPair = timeValuePairIterator.nextTimeValuePair();
- if (!isPointDeleted(tvPair.getTimestamp(), readableChunk.getDeletionList(), deleteCursor)) {
- builder.getTimeColumnBuilder().writeLong(tvPair.getTimestamp());
- switch (tsDataType) {
- case BOOLEAN:
- builder.getColumnBuilder(0).writeBoolean(tvPair.getValue().getBoolean());
- break;
- case INT32:
- case DATE:
- builder.getColumnBuilder(0).writeInt(tvPair.getValue().getInt());
- break;
- case INT64:
- case TIMESTAMP:
- builder.getColumnBuilder(0).writeLong(tvPair.getValue().getLong());
- break;
- case FLOAT:
- builder.getColumnBuilder(0).writeFloat(tvPair.getValue().getFloat());
- break;
- case DOUBLE:
- builder.getColumnBuilder(0).writeDouble(tvPair.getValue().getDouble());
- break;
- case TEXT:
- case STRING:
- case BLOB:
- builder.getColumnBuilder(0).writeBinary(tvPair.getValue().getBinary());
- break;
- default:
- break;
- }
- builder.declarePosition();
- }
- }
- if (builder.getPositionCount() > readableChunk.getMaxNumberOfPointsInPage()) {
- throw new RuntimeException(
- String.format(
- "Points in current page %d is larger than %d",
- builder.getPositionCount(), readableChunk.getMaxNumberOfPointsInPage()));
- }
+ return timeValuePairIterator.getBatch(tsBlockIndex);
}
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemPageReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemPageReader.java
index 26e95ecbda6..90231d1bedf 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemPageReader.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemPageReader.java
@@ -20,7 +20,6 @@
package org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk;
import org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk.metadata.PageMetadata;
-import org.apache.iotdb.db.utils.datastructure.MergeSortTVListIterator;
import org.apache.tsfile.block.column.Column;
import org.apache.tsfile.block.column.ColumnBuilder;
@@ -51,10 +50,7 @@ public class MemPageReader implements IPageReader {
private TsBlock tsBlock;
private Filter recordFilter;
- private final MergeSortTVListIterator mergeSortTvListIterator;
- // MemPage range - [pageStartOffsets, pageEndOffsets)
- private final int[] pageStartOffsets;
- private final int[] pageEndOffsets;
+ private final int pageIndex;
private final Supplier<TsBlock> tsBlockSupplier;
private final TSDataType tsDataType;
private final PageMetadata pageMetadata;
@@ -63,17 +59,13 @@ public class MemPageReader implements IPageReader {
public MemPageReader(
Supplier<TsBlock> tsBlockSupplier,
- MergeSortTVListIterator mergeSortTvListIterator,
- int[] pageStartOffsets,
- int[] pageEndOffSets,
+ int pageIndex,
TSDataType tsDataType,
String measurementUid,
Statistics statistics,
Filter recordFilter) {
this.tsBlockSupplier = tsBlockSupplier;
- this.mergeSortTvListIterator = mergeSortTvListIterator;
- this.pageStartOffsets = pageStartOffsets;
- this.pageEndOffsets = pageEndOffSets;
+ this.pageIndex = pageIndex;
this.recordFilter = recordFilter;
this.tsDataType = tsDataType;
this.pageMetadata = new PageMetadata(measurementUid, tsDataType, statistics);
@@ -240,7 +232,7 @@ public class MemPageReader implements IPageReader {
private void getTsBlock() {
if (tsBlock == null) {
- initializeOffsets();
+ initializeTsBlockIndex();
tsBlock = tsBlockSupplier.get();
if (pageMetadata.getStatistics() == null) {
initPageStatistics();
@@ -248,12 +240,9 @@ public class MemPageReader implements IPageReader {
}
}
- private void initializeOffsets() {
- if (pageStartOffsets != null) {
- mergeSortTvListIterator.setTVListOffsets(pageStartOffsets);
- }
+ private void initializeTsBlockIndex() {
if (tsBlockSupplier instanceof MemChunkReader.TsBlockSupplier) {
- ((MemChunkReader.TsBlockSupplier) tsBlockSupplier).setPageEndOffsets(pageEndOffsets);
+ ((MemChunkReader.TsBlockSupplier) tsBlockSupplier).setTsBlockIndex(pageIndex);
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
index 679ffe2d669..0b9f4d951b0 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.db.storageengine.rescon.memory.PrimitiveArrayManager;
import org.apache.iotdb.db.utils.MathUtils;
import org.apache.tsfile.block.column.ColumnBuilder;
+import org.apache.tsfile.common.conf.TSFileDescriptor;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
import org.apache.tsfile.read.TimeValuePair;
@@ -71,9 +72,12 @@ public abstract class AlignedTVList extends TVList {
// Value is null
// Index relation: columnIndex(dataTypeIndex) -> arrayIndex -> elementIndex
protected List<List<BitMap>> bitMaps;
+
// not null when constructed by queries for tree model
BitMap allValueColDeletedMap;
+ private final AlignedTVList outer = this;
+
AlignedTVList(List<TSDataType> types) {
super();
dataTypes = types;
@@ -1316,55 +1320,62 @@ public abstract class AlignedTVList extends TVList {
public AlignedTVListIterator iterator(
List<TSDataType> dataTypeList,
List<Integer> columnIndexList,
+ List<List<TimeRange>> valueColumnsDeletionList,
Integer floatPrecision,
List<TSEncoding> encodingList) {
- return new AlignedTVListIterator(dataTypeList, columnIndexList, floatPrecision, encodingList);
+ return new AlignedTVListIterator(
+ dataTypeList, columnIndexList, valueColumnsDeletionList, floatPrecision, encodingList);
}
/* AlignedTVList Iterator */
- public class AlignedTVListIterator extends TVListIterator {
- private BitMap allValueColDeletedMap;
+ public class AlignedTVListIterator extends TVListIterator implements MemPointIterator {
+ private final BitMap allValueColDeletedMap;
+ private final List<TSDataType> dataTypeList;
+ private final List<Integer> columnIndexList;
+ List<List<TimeRange>> valueColumnsDeletionList;
+ private final Integer floatPrecision;
+ private final List<TSEncoding> encodingList;
- private List<TSDataType> dataTypeList;
- private List<Integer> columnIndexList;
- private Integer floatPrecision;
- private List<TSEncoding> encodingList;
+ // remember the selected index of last not-null value for each column during prepareNext phase
+ private final int[] selectedIndices;
+ private boolean findValidRow;
- // remember the selected index of last not-null value for each column during prepareNext phase.
- // It is already converted by getValueIndex method, so it can be directly used in
- // getPrimitiveObject method.
- private int[] selectedIndex;
+ private final List<int[]> valueColumnDeleteCursor = new ArrayList<>();
- public AlignedTVListIterator() {
- super();
- }
+ private final int MAX_NUMBER_OF_POINTS_IN_PAGE =
+ TSFileDescriptor.getInstance().getConfig().getMaxNumberOfPointsInPage();
public AlignedTVListIterator(
List<TSDataType> dataTypeList,
List<Integer> columnIndexList,
+ List<List<TimeRange>> valueColumnsDeletionList,
Integer floatPrecision,
List<TSEncoding> encodingList) {
- super(null, null);
+ super(null, null, null);
this.dataTypeList = dataTypeList;
this.columnIndexList =
(columnIndexList == null)
? IntStream.range(0, dataTypes.size()).boxed().collect(Collectors.toList())
: columnIndexList;
this.allValueColDeletedMap = getAllValueColDeletedMap();
+ this.valueColumnsDeletionList = valueColumnsDeletionList;
this.floatPrecision = floatPrecision;
this.encodingList = encodingList;
- this.selectedIndex = new int[dataTypeList.size()];
+ this.selectedIndices = new int[dataTypeList.size()];
+ for (int i = 0; i < dataTypeList.size(); i++) {
+ valueColumnDeleteCursor.add(new int[] {0});
+ }
}
- private void prepareNext() {
+ @Override
+ protected void prepareNext() {
// find the first row that is neither deleted nor empty (all NULL values)
- boolean findValidRow = false;
+ findValidRow = false;
while (index < rows && !findValidRow) {
- int rowIndex = getValueIndex(index);
// all columns values are deleted
- if ((allValueColDeletedMap != null && allValueColDeletedMap.isMarked(rowIndex))) {
+ if ((allValueColDeletedMap != null
+ && allValueColDeletedMap.isMarked(getValueIndex(index)))) {
index++;
- currentTime = index < rows ? getTime(index) : Long.MIN_VALUE;
continue;
}
@@ -1373,147 +1384,306 @@ public abstract class AlignedTVList extends TVList {
probeNext = true;
return;
}
- Arrays.fill(selectedIndex, rowIndex);
+ Arrays.fill(selectedIndices, index);
findValidRow = true;
- }
- // handle duplicated timestamp
- while (index + 1 < rows && getTime(index + 1) == currentTime) {
- index++;
- // skip all-Null rows if allValueColDeletedMap exists
- int rowIndex = getValueIndex(index);
- if (allValueColDeletedMap == null || !allValueColDeletedMap.isMarked(rowIndex)) {
+ // handle duplicated timestamp
+ while (index + 1 < rows && getTime(index + 1) == getTime(index)) {
+ index++;
+ // skip all-Null rows if allValueColDeletedMap exists
+ if (allValueColDeletedMap == null
+ || !allValueColDeletedMap.isMarked(getValueIndex(index))) {
+ for (int columnIndex = 0; columnIndex < dataTypeList.size(); columnIndex++) {
+ // update selected index if the column is not null
+ if (!isNullValue(index, columnIndex)) {
+ selectedIndices[columnIndex] = index;
+ }
+ }
+ }
+ }
+
+ // valueColumnsDeletionList is set when AlignedTVList iterator is created by
+ // MemPointIterator.single method. Otherwise, all-null rows is checked by
+ // MergeSortMultiAlignedTVListIterator or OrderedMultiAlignedTVListIterator.
+ if (valueColumnsDeletionList != null) {
+ BitMap bitMap = new BitMap(dataTypeList.size());
+ long time = getTime(index);
for (int columnIndex = 0; columnIndex < dataTypeList.size(); columnIndex++) {
- // update currTvPair if the column is not null
- if (!isNull(rowIndex, columnIndex)) {
- selectedIndex[columnIndex] = rowIndex;
+ if (isNullValue(index, columnIndex)
+ || isPointDeleted(
+ time,
+ valueColumnsDeletionList.get(columnIndex),
+ valueColumnDeleteCursor.get(columnIndex))) {
+ bitMap.mark(columnIndex);
}
}
+ if (bitMap.isAllMarked()) {
+ findValidRow = false;
+ index++;
+ }
}
}
probeNext = true;
}
@Override
- public boolean hasNext() {
- if (!probeNext) {
- prepareNext();
- }
- return index < rows;
- }
-
- @Override
- public boolean hasCurrent() {
- return index < rows;
- }
-
- @Override
- public TimeValuePair next() {
- if (!hasNext()) {
+ public TimeValuePair nextTimeValuePair() {
+ if (!hasNextTimeValuePair()) {
return null;
}
-
TsPrimitiveType[] vector = new TsPrimitiveType[dataTypeList.size()];
for (int columnIndex = 0; columnIndex < dataTypeList.size(); columnIndex++) {
- vector[columnIndex] = getPrimitiveObject(selectedIndex[columnIndex], columnIndex);
+ vector[columnIndex] = getPrimitiveTypeObject(selectedIndices[columnIndex], columnIndex);
}
TimeValuePair tvPair =
- new TimeValuePair(currentTime, TsPrimitiveType.getByType(TSDataType.VECTOR, vector));
+ new TimeValuePair(getTime(index), TsPrimitiveType.getByType(TSDataType.VECTOR, vector));
- step();
+ next();
return tvPair;
}
@Override
- public TimeValuePair current() {
+ public TimeValuePair currentTimeValuePair() {
if (!hasCurrent()) {
return null;
}
TsPrimitiveType[] vector = new TsPrimitiveType[dataTypeList.size()];
for (int columnIndex = 0; columnIndex < dataTypeList.size(); columnIndex++) {
- vector[columnIndex] = getPrimitiveObject(selectedIndex[columnIndex], columnIndex);
+ vector[columnIndex] = getPrimitiveTypeObject(selectedIndices[columnIndex], columnIndex);
}
- return new TimeValuePair(currentTime, TsPrimitiveType.getByType(TSDataType.VECTOR, vector));
+ return new TimeValuePair(
+ getTime(index), TsPrimitiveType.getByType(TSDataType.VECTOR, vector));
}
- public boolean isNull(int rowIndex, int columnIndex) {
- int validColumnIndex = columnIndexList.get(columnIndex);
- if (validColumnIndex < 0 || validColumnIndex >= dataTypes.size()) {
- return true;
- }
- return isNullValue(rowIndex, validColumnIndex);
- }
-
- public TsPrimitiveType getPrimitiveObject(int rowIndex, int columnIndex) {
- if (rowIndex < 0 || rowIndex >= rows) {
+ public TsPrimitiveType getPrimitiveTypeObject(int rowIndex, int columnIndex) {
+ int valueIndex = getValueIndex(index);
+ if (valueIndex < 0 || valueIndex >= rows) {
return null;
}
int validColumnIndex = columnIndexList.get(columnIndex);
if (validColumnIndex < 0 || validColumnIndex >= dataTypes.size()) {
return null;
}
- if (isNullValue(rowIndex, validColumnIndex)) {
+ if (outer.isNullValue(valueIndex, validColumnIndex)) {
return null;
}
switch (dataTypeList.get(columnIndex)) {
case BOOLEAN:
return TsPrimitiveType.getByType(
- TSDataType.BOOLEAN, getBooleanByValueIndex(rowIndex, validColumnIndex));
+ TSDataType.BOOLEAN, getBooleanByValueIndex(valueIndex, validColumnIndex));
case INT32:
case DATE:
return TsPrimitiveType.getByType(
- TSDataType.INT32, getIntByValueIndex(rowIndex, validColumnIndex));
+ TSDataType.INT32, getIntByValueIndex(valueIndex, validColumnIndex));
case INT64:
case TIMESTAMP:
return TsPrimitiveType.getByType(
- TSDataType.INT64, getLongByValueIndex(rowIndex, validColumnIndex));
+ TSDataType.INT64, getLongByValueIndex(valueIndex, validColumnIndex));
case FLOAT:
- float valueF = getFloatByValueIndex(rowIndex, validColumnIndex);
- if (floatPrecision != null
- && encodingList != null
- && !Float.isNaN(valueF)
- && (encodingList.get(columnIndex) == TSEncoding.RLE
- || encodingList.get(columnIndex) == TSEncoding.TS_2DIFF)) {
- valueF = MathUtils.roundWithGivenPrecision(valueF, floatPrecision);
- }
- return TsPrimitiveType.getByType(TSDataType.FLOAT, valueF);
+ return TsPrimitiveType.getByType(
+ TSDataType.FLOAT,
+ roundValueWithGivenPrecision(
+ getFloatByValueIndex(valueIndex, validColumnIndex),
+ floatPrecision,
+ encodingList.get(columnIndex)));
case DOUBLE:
- double valueD = getDoubleByValueIndex(rowIndex, validColumnIndex);
- if (floatPrecision != null
- && encodingList != null
- && !Double.isNaN(valueD)
- && (encodingList.get(columnIndex) == TSEncoding.RLE
- || encodingList.get(columnIndex) == TSEncoding.TS_2DIFF)) {
- valueD = MathUtils.roundWithGivenPrecision(valueD, floatPrecision);
- }
- return TsPrimitiveType.getByType(TSDataType.DOUBLE, valueD);
+ return TsPrimitiveType.getByType(
+ TSDataType.DOUBLE,
+ roundValueWithGivenPrecision(
+ getDoubleByValueIndex(valueIndex, validColumnIndex),
+ floatPrecision,
+ encodingList.get(columnIndex)));
case TEXT:
case BLOB:
case STRING:
return TsPrimitiveType.getByType(
- TSDataType.TEXT, getBinaryByValueIndex(rowIndex, validColumnIndex));
+ TSDataType.TEXT, getBinaryByValueIndex(valueIndex, validColumnIndex));
default:
throw new UnSupportedDataTypeException(
String.format("Data type %s is not supported.", dataTypeList.get(columnIndex)));
}
}
- public int getSelectedIndex(int columnIndex) {
- return selectedIndex[columnIndex];
+ @Override
+ public boolean hasNextBatch() {
+ if (!probeNext) {
+ prepareNext();
+ }
+ if (findValidRow && selectedIndices.length > 0) {
+ index = Arrays.stream(selectedIndices).min().getAsInt();
+ }
+ return index < rows;
}
@Override
- public AlignedTVListIterator clone() {
- AlignedTVListIterator iterator = new AlignedTVListIterator();
- iterator.rows = rows;
- iterator.dataTypeList = dataTypeList;
- iterator.columnIndexList = columnIndexList;
- iterator.allValueColDeletedMap = allValueColDeletedMap;
- iterator.floatPrecision = floatPrecision;
- iterator.encodingList = this.encodingList;
- iterator.selectedIndex = new int[dataTypeList.size()];
- iterator.reset();
- return iterator;
+ public TsBlock nextBatch() {
+ TsBlockBuilder builder = new TsBlockBuilder(dataTypeList);
+ // Time column
+ TimeColumnBuilder timeBuilder = builder.getTimeColumnBuilder();
+
+ int validRowCount = 0;
+ // duplicated time or deleted time are all invalid, true if we don't need this row
+ BitMap timeDuplicateInfo = null;
+
+ int startIndex = index;
+ // time column
+ for (; index < rows; index++) {
+ if (validRowCount > MAX_NUMBER_OF_POINTS_IN_PAGE) {
+ break;
+ }
+ // skip empty row
+ if (allValueColDeletedMap != null && allValueColDeletedMap.isMarked(getValueIndex(index))) {
+ continue;
+ }
+ int nextRowIndex = index + 1;
+ while (nextRowIndex < rows
+ && allValueColDeletedMap != null
+ && allValueColDeletedMap.isMarked(getValueIndex(nextRowIndex))) {
+ nextRowIndex++;
+ }
+ long timestamp = getTime(index);
+ if (nextRowIndex == rows || timestamp != getTime(nextRowIndex)) {
+ timeBuilder.writeLong(getTime(index));
+ validRowCount++;
+ } else {
+ if (Objects.isNull(timeDuplicateInfo)) {
+ timeDuplicateInfo = new BitMap(rows);
+ }
+ timeDuplicateInfo.mark(index);
+ }
+ index = nextRowIndex - 1;
+ }
+
+ int columnCount = dataTypeList.size();
+ // value columns
+ for (int columnIndex = 0; columnIndex < columnCount; columnIndex++) {
+ int validColumnIndex = columnIndexList.get(columnIndex);
+
+ int[] deleteCursor = {0};
+ // Pair of Time and Index
+ Pair<Long, Integer> lastValidPointIndexForTimeDupCheck = null;
+ if (Objects.nonNull(timeDuplicateInfo)) {
+ lastValidPointIndexForTimeDupCheck = new Pair<>(Long.MIN_VALUE, null);
+ }
+ ColumnBuilder valueBuilder = builder.getColumnBuilder(columnIndex);
+ for (int sortedRowIndex = startIndex; sortedRowIndex < index; sortedRowIndex++) {
+ // skip empty row
+ if (allValueColDeletedMap != null
+ && allValueColDeletedMap.isMarked(getValueIndex(sortedRowIndex))) {
+ continue;
+ }
+ // skip time duplicated or totally deleted rows
+ if (Objects.nonNull(timeDuplicateInfo)) {
+ if (!outer.isNullValue(getValueIndex(sortedRowIndex), validColumnIndex)) {
+ lastValidPointIndexForTimeDupCheck.left = getTime(sortedRowIndex);
+ lastValidPointIndexForTimeDupCheck.right = getValueIndex(sortedRowIndex);
+ }
+ if (timeDuplicateInfo.isMarked(sortedRowIndex)) {
+ continue;
+ }
+ }
+
+ // append null value when query column does not exist in current aligned TVList
+ if (validColumnIndex < 0 || validColumnIndex >= dataTypes.size()) {
+ valueBuilder.appendNull();
+ continue;
+ }
+
+ // The part of code solves the following problem:
+ // Time: 1,2,2,3
+ // Value: 1,2,null,null
+ // When rowIndex:1, pair(min,null), timeDuplicateInfo:false, write(T:1,V:1)
+ // When rowIndex:2, pair(2,2), timeDuplicateInfo:true, skip writing value
+ // When rowIndex:3, pair(2,2), timeDuplicateInfo:false, T:2==pair.left:2, write(T:2,V:2)
+ // When rowIndex:4, pair(2,2), timeDuplicateInfo:false, T:3!=pair.left:2,
+ // write(T:3,V:null)
+ int originRowIndex;
+ if (Objects.nonNull(lastValidPointIndexForTimeDupCheck)
+ && (getTime(sortedRowIndex) == lastValidPointIndexForTimeDupCheck.left)) {
+ originRowIndex = lastValidPointIndexForTimeDupCheck.right;
+ } else {
+ originRowIndex = getValueIndex(sortedRowIndex);
+ }
+ if (outer.isNullValue(originRowIndex, validColumnIndex)
+ || isPointDeleted(
+ getTime(sortedRowIndex),
+ Objects.isNull(valueColumnsDeletionList)
+ ? null
+ : valueColumnsDeletionList.get(columnIndex),
+ deleteCursor)) {
+ valueBuilder.appendNull();
+ continue;
+ }
+ switch (dataTypes.get(validColumnIndex)) {
+ case BOOLEAN:
+ valueBuilder.writeBoolean(getBooleanByValueIndex(originRowIndex, validColumnIndex));
+ break;
+ case INT32:
+ case DATE:
+ valueBuilder.writeInt(getIntByValueIndex(originRowIndex, validColumnIndex));
+ break;
+ case INT64:
+ case TIMESTAMP:
+ valueBuilder.writeLong(getLongByValueIndex(originRowIndex, validColumnIndex));
+ break;
+ case FLOAT:
+ valueBuilder.writeFloat(
+ roundValueWithGivenPrecision(
+ getFloatByValueIndex(originRowIndex, validColumnIndex),
+ floatPrecision,
+ encodingList.get(columnIndex)));
+ break;
+ case DOUBLE:
+ valueBuilder.writeDouble(
+ roundValueWithGivenPrecision(
+ getDoubleByValueIndex(originRowIndex, validColumnIndex),
+ floatPrecision,
+ encodingList.get(columnIndex)));
+ break;
+ case TEXT:
+ case BLOB:
+ case STRING:
+ valueBuilder.writeBinary(getBinaryByValueIndex(originRowIndex, validColumnIndex));
+ break;
+ default:
+ break;
+ }
+ }
+ }
+ builder.declarePositions(validRowCount);
+ TsBlock tsBlock = builder.build();
+ tsBlocks.add(tsBlock);
+
+ probeNext = false;
+ return tsBlock;
+ }
+
+ public int[] getSelectedIndices() {
+ return selectedIndices;
+ }
+
+ public int getSelectedIndex(int column) {
+ return selectedIndices[column];
+ }
+
+ public AlignedTVList getAlignedTVList() {
+ return outer;
+ }
+
+ /**
+ * @param rowIndex index of sorted rows in the Aligned TVList.
+ * @param columnIndex index of columnIndexList for the query.
+ * @return boolean
+ */
+ public boolean isNullValue(int rowIndex, int columnIndex) {
+ // valueIndex is converted index of values
+ int valueIndex = getValueIndex(rowIndex);
+ // validColumnIndex is converted index of columns in the Aligned TVList.
+ int validColumnIndex = columnIndexList.get(columnIndex);
+ if (validColumnIndex < 0 || validColumnIndex >= dataTypes.size()) {
+ return true;
+ }
+ return outer.isNullValue(valueIndex, validColumnIndex);
}
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MemPointIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MemPointIterator.java
new file mode 100644
index 00000000000..b6864a27cb4
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MemPointIterator.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.utils.datastructure;
+
+import org.apache.tsfile.read.common.block.TsBlock;
+import org.apache.tsfile.read.reader.IPointReader;
+
+public interface MemPointIterator extends IPointReader {
+ TsBlock getBatch(int tsBlockIndex);
+
+ boolean hasNextBatch();
+
+ TsBlock nextBatch();
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MemPointIteratorFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MemPointIteratorFactory.java
new file mode 100644
index 00000000000..25600e05d07
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MemPointIteratorFactory.java
@@ -0,0 +1,304 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.utils.datastructure;
+
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.tsfile.read.common.TimeRange;
+
+import java.util.List;
+
+public class MemPointIteratorFactory {
+ private MemPointIteratorFactory() {
+ // forbidden construction
+ }
+
+ // TVListIterator
+ private static MemPointIterator single(List<TVList> tvLists) {
+ return tvLists.get(0).iterator(null, null, null);
+ }
+
+ private static MemPointIterator single(List<TVList> tvLists, List<TimeRange> deletionList) {
+ return tvLists.get(0).iterator(deletionList, null, null);
+ }
+
+ private static MemPointIterator single(
+ List<TVList> tvLists,
+ List<TimeRange> deletionList,
+ Integer floatPrecision,
+ TSEncoding encoding) {
+ return tvLists.get(0).iterator(deletionList, floatPrecision, encoding);
+ }
+
+ // MergeSortMultiTVListIterator
+ private static MemPointIterator mergeSort(TSDataType tsDataType, List<TVList> tvLists) {
+ return new MergeSortMultiTVListIterator(tsDataType, tvLists, null, null, null);
+ }
+
+ private static MemPointIterator mergeSort(
+ TSDataType tsDataType, List<TVList> tvLists, List<TimeRange> deletionList) {
+ return new MergeSortMultiTVListIterator(tsDataType, tvLists, deletionList, null, null);
+ }
+
+ private static MemPointIterator mergeSort(
+ TSDataType tsDataType,
+ List<TVList> tvLists,
+ List<TimeRange> deletionList,
+ Integer floatPrecision,
+ TSEncoding encoding) {
+ return new MergeSortMultiTVListIterator(
+ tsDataType, tvLists, deletionList, floatPrecision, encoding);
+ }
+
+ // OrderedMultiTVListIterator
+ private static MemPointIterator ordered(TSDataType tsDataType, List<TVList> tvLists) {
+ return new OrderedMultiTVListIterator(tsDataType, tvLists, null, null, null);
+ }
+
+ private static MemPointIterator ordered(
+ TSDataType tsDataType, List<TVList> tvLists, List<TimeRange> deletionList) {
+ return new OrderedMultiTVListIterator(tsDataType, tvLists, deletionList, null, null);
+ }
+
+ private static MemPointIterator ordered(
+ TSDataType tsDataType,
+ List<TVList> tvLists,
+ List<TimeRange> deletionList,
+ Integer floatPrecision,
+ TSEncoding encoding) {
+ return new OrderedMultiTVListIterator(
+ tsDataType, tvLists, deletionList, floatPrecision, encoding);
+ }
+
+ // AlignedTVListIterator
+ private static MemPointIterator single(
+ List<TSDataType> tsDataTypes,
+ List<Integer> columnIndexList,
+ List<AlignedTVList> alignedTvLists) {
+ return alignedTvLists.get(0).iterator(tsDataTypes, columnIndexList, null, null, null);
+ }
+
+ private static MemPointIterator single(
+ List<TSDataType> tsDataTypes,
+ List<Integer> columnIndexList,
+ List<AlignedTVList> alignedTvLists,
+ List<List<TimeRange>> valueColumnsDeletionList) {
+ return alignedTvLists
+ .get(0)
+ .iterator(tsDataTypes, columnIndexList, valueColumnsDeletionList, null, null);
+ }
+
+ private static MemPointIterator single(
+ List<TSDataType> tsDataTypes,
+ List<Integer> columnIndexList,
+ List<AlignedTVList> alignedTvLists,
+ List<List<TimeRange>> valueColumnsDeletionList,
+ Integer floatPrecision,
+ List<TSEncoding> encodingList) {
+ return alignedTvLists
+ .get(0)
+ .iterator(
+ tsDataTypes, columnIndexList, valueColumnsDeletionList, floatPrecision, encodingList);
+ }
+
+ // MergeSortMultiAlignedTVListIterator
+ private static MemPointIterator mergeSort(
+ List<TSDataType> tsDataTypes,
+ List<Integer> columnIndexList,
+ List<AlignedTVList> alignedTvLists) {
+ return new MergeSortMultiAlignedTVListIterator(
+ tsDataTypes, columnIndexList, alignedTvLists, null, null, null);
+ }
+
+ private static MemPointIterator mergeSort(
+ List<TSDataType> tsDataTypes,
+ List<Integer> columnIndexList,
+ List<AlignedTVList> alignedTvLists,
+ List<List<TimeRange>> valueColumnsDeletionList) {
+ return new MergeSortMultiAlignedTVListIterator(
+ tsDataTypes, columnIndexList, alignedTvLists, valueColumnsDeletionList, null, null);
+ }
+
+ private static MemPointIterator mergeSort(
+ List<TSDataType> tsDataTypes,
+ List<Integer> columnIndexList,
+ List<AlignedTVList> alignedTvLists,
+ List<List<TimeRange>> valueColumnsDeletionList,
+ Integer floatPrecision,
+ List<TSEncoding> encodingList) {
+ return new MergeSortMultiAlignedTVListIterator(
+ tsDataTypes,
+ columnIndexList,
+ alignedTvLists,
+ valueColumnsDeletionList,
+ floatPrecision,
+ encodingList);
+ }
+
+ // OrderedMultiAlignedTVListIterator
+ private static MemPointIterator ordered(
+ List<TSDataType> tsDataTypes,
+ List<Integer> columnIndexList,
+ List<AlignedTVList> alignedTvLists) {
+ return new OrderedMultiAlignedTVListIterator(
+ tsDataTypes, columnIndexList, alignedTvLists, null, null, null);
+ }
+
+ private static MemPointIterator ordered(
+ List<TSDataType> tsDataTypes,
+ List<Integer> columnIndexList,
+ List<AlignedTVList> alignedTvLists,
+ List<List<TimeRange>> valueColumnsDeletionList) {
+ return new OrderedMultiAlignedTVListIterator(
+ tsDataTypes, columnIndexList, alignedTvLists, valueColumnsDeletionList, null, null);
+ }
+
+ private static MemPointIterator ordered(
+ List<TSDataType> tsDataTypes,
+ List<Integer> columnIndexList,
+ List<AlignedTVList> alignedTvLists,
+ List<List<TimeRange>> valueColumnsDeletionList,
+ Integer floatPrecision,
+ List<TSEncoding> encodingList) {
+ return new OrderedMultiAlignedTVListIterator(
+ tsDataTypes,
+ columnIndexList,
+ alignedTvLists,
+ valueColumnsDeletionList,
+ floatPrecision,
+ encodingList);
+ }
+
+ public static MemPointIterator create(TSDataType tsDataType, List<TVList> tvLists) {
+ if (tvLists.size() == 1) {
+ return single(tvLists);
+ } else if (isCompleteOrdered(tvLists)) {
+ return ordered(tsDataType, tvLists);
+ } else {
+ return mergeSort(tsDataType, tvLists);
+ }
+ }
+
+ public static MemPointIterator create(
+ TSDataType tsDataType, List<TVList> tvLists, List<TimeRange> deletionList) {
+ if (tvLists.size() == 1) {
+ return single(tvLists, deletionList);
+ } else if (isCompleteOrdered(tvLists)) {
+ return ordered(tsDataType, tvLists, deletionList);
+ } else {
+ return mergeSort(tsDataType, tvLists, deletionList);
+ }
+ }
+
+ public static MemPointIterator create(
+ TSDataType tsDataType,
+ List<TVList> tvLists,
+ List<TimeRange> deletionList,
+ Integer floatPrecision,
+ TSEncoding encoding) {
+ if (tvLists.size() == 1) {
+ return single(tvLists, deletionList, floatPrecision, encoding);
+ } else if (isCompleteOrdered(tvLists)) {
+ return ordered(tsDataType, tvLists, deletionList, floatPrecision, encoding);
+ } else {
+ return mergeSort(tsDataType, tvLists, deletionList, floatPrecision, encoding);
+ }
+ }
+
+ public static MemPointIterator create(
+ List<TSDataType> tsDataTypes,
+ List<Integer> columnIndexList,
+ List<AlignedTVList> alignedTvLists) {
+ if (alignedTvLists.size() == 1) {
+ return single(tsDataTypes, columnIndexList, alignedTvLists);
+ } else if (isCompleteOrdered(alignedTvLists)) {
+ return ordered(tsDataTypes, columnIndexList, alignedTvLists);
+ } else {
+ return mergeSort(tsDataTypes, columnIndexList, alignedTvLists);
+ }
+ }
+
+ public static MemPointIterator create(
+ List<TSDataType> tsDataTypes,
+ List<Integer> columnIndexList,
+ List<AlignedTVList> alignedTvLists,
+ List<List<TimeRange>> valueColumnsDeletionList) {
+ if (alignedTvLists.size() == 1) {
+ return single(tsDataTypes, columnIndexList, alignedTvLists, valueColumnsDeletionList);
+ } else if (isCompleteOrdered(alignedTvLists)) {
+ return ordered(tsDataTypes, columnIndexList, alignedTvLists, valueColumnsDeletionList);
+ } else {
+ return mergeSort(tsDataTypes, columnIndexList, alignedTvLists, valueColumnsDeletionList);
+ }
+ }
+
+ public static MemPointIterator create(
+ List<TSDataType> tsDataTypes,
+ List<Integer> columnIndexList,
+ List<AlignedTVList> alignedTvLists,
+ List<List<TimeRange>> valueColumnsDeletionList,
+ Integer floatPrecision,
+ List<TSEncoding> encodingList) {
+ if (alignedTvLists.size() == 1) {
+ return single(
+ tsDataTypes,
+ columnIndexList,
+ alignedTvLists,
+ valueColumnsDeletionList,
+ floatPrecision,
+ encodingList);
+ } else if (isCompleteOrdered(alignedTvLists)) {
+ return ordered(
+ tsDataTypes,
+ columnIndexList,
+ alignedTvLists,
+ valueColumnsDeletionList,
+ floatPrecision,
+ encodingList);
+ } else {
+ return mergeSort(
+ tsDataTypes,
+ columnIndexList,
+ alignedTvLists,
+ valueColumnsDeletionList,
+ floatPrecision,
+ encodingList);
+ }
+ }
+
+ private static boolean isCompleteOrdered(List<? extends TVList> tvLists) {
+ long time = Long.MIN_VALUE;
+ for (int i = 0; i < tvLists.size(); i++) {
+ TVList list = tvLists.get(i);
+ if (!list.isSorted()) {
+ return false;
+ }
+
+ if (tvLists.get(i).rowCount() == 0) {
+ continue;
+ }
+ if (i > 0 && list.getTime(0) <= time) {
+ return false;
+ }
+ time = list.getTime(list.rowCount() - 1);
+ }
+ return true;
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortAlignedTVListIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortAlignedTVListIterator.java
deleted file mode 100644
index 98f444abeff..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortAlignedTVListIterator.java
+++ /dev/null
@@ -1,192 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.utils.datastructure;
-
-import org.apache.tsfile.enums.TSDataType;
-import org.apache.tsfile.file.metadata.enums.TSEncoding;
-import org.apache.tsfile.read.TimeValuePair;
-import org.apache.tsfile.read.reader.IPointReader;
-import org.apache.tsfile.utils.Pair;
-import org.apache.tsfile.utils.TsPrimitiveType;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.PriorityQueue;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-
-public class MergeSortAlignedTVListIterator implements IPointReader {
- private List<AlignedTVList.AlignedTVListIterator> alignedTvListIterators;
-
- private boolean probeNext = false;
- private TimeValuePair currentTvPair;
-
- private List<Integer> probeIterators;
-
- // Min-Heap: minimal timestamp; if same timestamp, maximum TVList index
- private final PriorityQueue<Pair<Long, Integer>> minHeap =
- new PriorityQueue<>(
- (a, b) -> a.left.equals(b.left) ? b.right.compareTo(a.right) : a.left.compareTo(b.left));
-
- private int[] alignedTvListOffsets;
-
- public MergeSortAlignedTVListIterator(
- List<AlignedTVList> alignedTvLists,
- List<TSDataType> tsDataTypes,
- List<Integer> columnIndexList,
- Integer floatPrecision,
- List<TSEncoding> encodingList) {
- this.alignedTvListIterators = new ArrayList<>(alignedTvLists.size());
- for (AlignedTVList alignedTvList : alignedTvLists) {
- alignedTvListIterators.add(
- alignedTvList.iterator(tsDataTypes, columnIndexList, floatPrecision, encodingList));
- }
- this.alignedTvListOffsets = new int[alignedTvLists.size()];
- this.probeIterators =
- IntStream.range(0, alignedTvListIterators.size()).boxed().collect(Collectors.toList());
- }
-
- private MergeSortAlignedTVListIterator() {}
-
- private void prepareNextRow() {
- currentTvPair = null;
- if (alignedTvListIterators.size() == 1) {
- AlignedTVList.AlignedTVListIterator iterator = alignedTvListIterators.get(0);
- if (iterator.hasNext()) {
- currentTvPair = iterator.current();
- }
- probeNext = true;
- return;
- }
-
- for (int i : probeIterators) {
- AlignedTVList.AlignedTVListIterator iterator = alignedTvListIterators.get(i);
- if (iterator.hasNext()) {
- minHeap.add(new Pair<>(iterator.currentTime(), i));
- }
- }
- probeIterators.clear();
-
- if (!minHeap.isEmpty()) {
- Pair<Long, Integer> top = minHeap.poll();
- long time = top.left;
- probeIterators.add(top.right);
- currentTvPair = alignedTvListIterators.get(top.right).current();
- TsPrimitiveType[] currentValues = currentTvPair.getValue().getVector();
-
- while (!minHeap.isEmpty() && minHeap.peek().left == time) {
- Pair<Long, Integer> element = minHeap.poll();
- probeIterators.add(element.right);
-
- TimeValuePair tvPair = alignedTvListIterators.get(element.right).current();
- TsPrimitiveType[] values = tvPair.getValue().getVector();
- for (int columnIndex = 0; columnIndex < values.length; columnIndex++) {
- // if the column is currently not null, it needs not update
- if (currentValues[columnIndex] == null) {
- currentValues[columnIndex] = values[columnIndex];
- }
- }
- }
- }
- probeNext = true;
- }
-
- @Override
- public boolean hasNextTimeValuePair() {
- if (!probeNext) {
- prepareNextRow();
- }
- return currentTvPair != null;
- }
-
- @Override
- public TimeValuePair nextTimeValuePair() {
- if (!hasNextTimeValuePair()) {
- return null;
- }
- step();
- return currentTvPair;
- }
-
- @Override
- public TimeValuePair currentTimeValuePair() {
- if (!hasNextTimeValuePair()) {
- return null;
- }
- return currentTvPair;
- }
-
- public void step() {
- if (alignedTvListIterators.size() == 1) {
- AlignedTVList.AlignedTVListIterator iterator = alignedTvListIterators.get(0);
- iterator.step();
- alignedTvListOffsets[0] = iterator.getIndex();
- } else {
- for (int index : probeIterators) {
- AlignedTVList.AlignedTVListIterator iterator = alignedTvListIterators.get(index);
- iterator.step();
- alignedTvListOffsets[index] = iterator.getIndex();
- }
- }
- probeNext = false;
- }
-
- @Override
- public long getUsedMemorySize() {
- // not used
- return 0;
- }
-
- @Override
- public void close() throws IOException {}
-
- public int[] getAlignedTVListOffsets() {
- return alignedTvListOffsets;
- }
-
- public void setAlignedTVListOffsets(int[] alignedTvListOffsets) {
- for (int i = 0; i < alignedTvListIterators.size(); i++) {
- alignedTvListIterators.get(i).setIndex(alignedTvListOffsets[i]);
- this.alignedTvListOffsets[i] = alignedTvListOffsets[i];
- }
- if (alignedTvListIterators.size() > 1) {
- minHeap.clear();
- probeIterators.clear();
- for (int i = 0; i < alignedTvListIterators.size(); i++) {
- probeIterators.add(i);
- }
- }
- probeNext = false;
- }
-
- @Override
- public MergeSortAlignedTVListIterator clone() {
- MergeSortAlignedTVListIterator cloneIterator = new MergeSortAlignedTVListIterator();
- cloneIterator.alignedTvListIterators = new ArrayList<>(alignedTvListIterators.size());
- for (int i = 0; i < alignedTvListIterators.size(); i++) {
- cloneIterator.alignedTvListIterators.add(alignedTvListIterators.get(i).clone());
- }
- cloneIterator.alignedTvListOffsets = new int[alignedTvListIterators.size()];
- cloneIterator.probeIterators =
- IntStream.range(0, alignedTvListIterators.size()).boxed().collect(Collectors.toList());
- return cloneIterator;
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiAlignedTVListIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiAlignedTVListIterator.java
new file mode 100644
index 00000000000..d72b013051d
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiAlignedTVListIterator.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.utils.datastructure;
+
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.tsfile.read.common.TimeRange;
+import org.apache.tsfile.utils.BitMap;
+import org.apache.tsfile.utils.Pair;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.PriorityQueue;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static org.apache.iotdb.db.utils.ModificationUtils.isPointDeleted;
+
+public class MergeSortMultiAlignedTVListIterator extends MultiAlignedTVListIterator {
+ private final Set<Integer> probeIterators;
+ private final int[] iteratorIndices;
+ private final int[] rowIndices;
+
+ private final BitMap bitMap;
+ private final List<int[]> valueColumnDeleteCursor;
+ // Min-Heap: minimal timestamp; if same timestamp, maximum TVList index
+ private final PriorityQueue<Pair<Long, Integer>> minHeap =
+ new PriorityQueue<>(
+ (a, b) -> a.left.equals(b.left) ? b.right.compareTo(a.right) : a.left.compareTo(b.left));
+
+ public MergeSortMultiAlignedTVListIterator(
+ List<TSDataType> tsDataTypes,
+ List<Integer> columnIndexList,
+ List<AlignedTVList> alignedTvLists,
+ List<List<TimeRange>> valueColumnsDeletionList,
+ Integer floatPrecision,
+ List<TSEncoding> encodingList) {
+ super(
+ tsDataTypes,
+ columnIndexList,
+ alignedTvLists,
+ valueColumnsDeletionList,
+ floatPrecision,
+ encodingList);
+ this.probeIterators =
+ IntStream.range(0, alignedTvListIterators.size()).boxed().collect(Collectors.toSet());
+ this.bitMap = new BitMap(tsDataTypeList.size());
+ this.iteratorIndices = new int[tsDataTypeList.size()];
+ this.rowIndices = new int[tsDataTypeList.size()];
+ this.valueColumnDeleteCursor = new ArrayList<>();
+ for (int i = 0; i < tsDataTypeList.size(); i++) {
+ valueColumnDeleteCursor.add(new int[] {0});
+ }
+ }
+
+ @Override
+ protected void prepareNext() {
+ hasNext = false;
+ for (int i : probeIterators) {
+ AlignedTVList.AlignedTVListIterator iterator = alignedTvListIterators.get(i);
+ if (iterator.hasNextTimeValuePair()) {
+ minHeap.add(new Pair<>(iterator.currentTime(), i));
+ }
+ }
+ probeIterators.clear();
+
+ while (!minHeap.isEmpty() && !hasNext) {
+ bitMap.reset();
+ Pair<Long, Integer> top = minHeap.poll();
+ currentTime = top.left;
+ probeIterators.add(top.right);
+
+ for (int columnIndex = 0; columnIndex < tsDataTypeList.size(); columnIndex++) {
+ iteratorIndices[columnIndex] = top.right;
+ rowIndices[columnIndex] =
+ alignedTvListIterators.get(top.right).getSelectedIndex(columnIndex);
+ if (alignedTvListIterators
+ .get(top.right)
+ .isNullValue(rowIndices[columnIndex], columnIndex)) {
+ bitMap.mark(columnIndex);
+ }
+ }
+ hasNext = true;
+
+ // duplicated timestamps
+ while (!minHeap.isEmpty() && minHeap.peek().left == currentTime) {
+ Pair<Long, Integer> element = minHeap.poll();
+ probeIterators.add(element.right);
+
+ for (int columnIndex = 0; columnIndex < tsDataTypeList.size(); columnIndex++) {
+ // if current column null, it needs update
+ if (alignedTvListIterators
+ .get(iteratorIndices[columnIndex])
+ .isNullValue(rowIndices[columnIndex], columnIndex)) {
+ iteratorIndices[columnIndex] = element.right;
+ rowIndices[columnIndex] =
+ alignedTvListIterators.get(element.right).getSelectedIndex(columnIndex);
+ if (!alignedTvListIterators
+ .get(element.right)
+ .isNullValue(rowIndices[columnIndex], columnIndex)) {
+ bitMap.unmark(columnIndex);
+ }
+ }
+
+ // check valueColumnsDeletionList
+ if (valueColumnsDeletionList != null
+ && isPointDeleted(
+ currentTime,
+ valueColumnsDeletionList.get(columnIndex),
+ valueColumnDeleteCursor.get(columnIndex))) {
+ iteratorIndices[columnIndex] = -1;
+ bitMap.mark(columnIndex);
+ }
+ }
+ }
+
+ if (bitMap.isAllMarked()) {
+ Iterator<Integer> it = probeIterators.iterator();
+ while (it.hasNext()) {
+ int idx = it.next();
+ AlignedTVList.AlignedTVListIterator iterator = alignedTvListIterators.get(idx);
+ iterator.next();
+ if (iterator.hasNextTimeValuePair()) {
+ minHeap.add(new Pair<>(iterator.currentTime(), idx));
+ } else {
+ it.remove();
+ }
+ }
+ hasNext = false;
+ }
+ }
+ probeNext = true;
+ }
+
+ @Override
+ protected void next() {
+ for (int index : probeIterators) {
+ alignedTvListIterators.get(index).next();
+ }
+ probeNext = false;
+ }
+
+ @Override
+ protected int currentIteratorIndex(int columnIndex) {
+ return iteratorIndices[columnIndex];
+ }
+
+ @Override
+ protected int currentRowIndex(int columnIndex) {
+ return rowIndices[columnIndex];
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiTVListIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiTVListIterator.java
new file mode 100644
index 00000000000..1436d519cb5
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiTVListIterator.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.utils.datastructure;
+
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.tsfile.read.common.TimeRange;
+import org.apache.tsfile.utils.Pair;
+
+import java.util.List;
+import java.util.PriorityQueue;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+public class MergeSortMultiTVListIterator extends MultiTVListIterator {
+ private final List<Integer> probeIterators;
+ private final PriorityQueue<Pair<Long, Integer>> minHeap =
+ new PriorityQueue<>(
+ (a, b) -> a.left.equals(b.left) ? b.right.compareTo(a.right) : a.left.compareTo(b.left));
+
+ public MergeSortMultiTVListIterator(
+ TSDataType tsDataType,
+ List<TVList> tvLists,
+ List<TimeRange> deletionList,
+ Integer floatPrecision,
+ TSEncoding encoding) {
+ super(tsDataType, tvLists, deletionList, floatPrecision, encoding);
+ this.probeIterators =
+ IntStream.range(0, tvListIterators.size()).boxed().collect(Collectors.toList());
+ }
+
+ @Override
+ protected void prepareNext() {
+ hasNext = false;
+ for (int i : probeIterators) {
+ TVList.TVListIterator iterator = tvListIterators.get(i);
+ if (iterator.hasNextTimeValuePair()) {
+ minHeap.add(new Pair<>(iterator.currentTime(), i));
+ }
+ }
+ probeIterators.clear();
+
+ if (!minHeap.isEmpty()) {
+ Pair<Long, Integer> top = minHeap.poll();
+ iteratorIndex = top.right;
+ probeIterators.add(iteratorIndex);
+ rowIndex = tvListIterators.get(iteratorIndex).getIndex();
+ hasNext = true;
+
+ // duplicated timestamps
+ while (!minHeap.isEmpty() && minHeap.peek().left.longValue() == top.left.longValue()) {
+ Pair<Long, Integer> element = minHeap.poll();
+ probeIterators.add(element.right);
+ }
+ }
+ probeNext = true;
+ }
+
+ @Override
+ protected void next() {
+ for (int index : probeIterators) {
+ tvListIterators.get(index).next();
+ }
+ probeNext = false;
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortTVListIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortTVListIterator.java
deleted file mode 100644
index 7c4834efe9c..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortTVListIterator.java
+++ /dev/null
@@ -1,180 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.utils.datastructure;
-
-import org.apache.tsfile.file.metadata.enums.TSEncoding;
-import org.apache.tsfile.read.TimeValuePair;
-import org.apache.tsfile.read.reader.IPointReader;
-import org.apache.tsfile.utils.Pair;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.PriorityQueue;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-
-public class MergeSortTVListIterator implements IPointReader {
- private List<TVList.TVListIterator> tvListIterators;
- private int[] tvListOffsets;
-
- private boolean probeNext = false;
- private TimeValuePair currentTvPair;
-
- private List<Integer> probeIterators;
- private final PriorityQueue<Pair<Long, Integer>> minHeap =
- new PriorityQueue<>(
- (a, b) -> a.left.equals(b.left) ? b.right.compareTo(a.right) : a.left.compareTo(b.left));
-
- public MergeSortTVListIterator(List<TVList> tvLists) {
- tvListIterators = new ArrayList<>(tvLists.size());
- for (TVList tvList : tvLists) {
- tvListIterators.add(tvList.iterator(null, null));
- }
- this.tvListOffsets = new int[tvLists.size()];
- this.probeIterators =
- IntStream.range(0, tvListIterators.size()).boxed().collect(Collectors.toList());
- }
-
- public MergeSortTVListIterator(
- List<TVList> tvLists, Integer floatPrecision, TSEncoding encoding) {
- tvListIterators = new ArrayList<>(tvLists.size());
- for (TVList tvList : tvLists) {
- tvListIterators.add(tvList.iterator(floatPrecision, encoding));
- }
- this.tvListOffsets = new int[tvLists.size()];
- this.probeIterators =
- IntStream.range(0, tvListIterators.size()).boxed().collect(Collectors.toList());
- }
-
- private MergeSortTVListIterator() {}
-
- private void prepareNext() {
- currentTvPair = null;
- if (tvListIterators.size() == 1) {
- TVList.TVListIterator iterator = tvListIterators.get(0);
- if (iterator.hasNext()) {
- currentTvPair = iterator.current();
- }
- probeNext = true;
- return;
- }
-
- for (int i : probeIterators) {
- TVList.TVListIterator iterator = tvListIterators.get(i);
- if (iterator.hasNext()) {
- minHeap.add(new Pair<>(iterator.currentTime(), i));
- }
- }
- probeIterators.clear();
-
- if (!minHeap.isEmpty()) {
- Pair<Long, Integer> top = minHeap.poll();
- probeIterators.add(top.right);
- currentTvPair = tvListIterators.get(top.right).current();
- while (!minHeap.isEmpty() && minHeap.peek().left.longValue() == top.left.longValue()) {
- Pair<Long, Integer> element = minHeap.poll();
- probeIterators.add(element.right);
- }
- }
- probeNext = true;
- }
-
- @Override
- public boolean hasNextTimeValuePair() {
- if (!probeNext) {
- prepareNext();
- }
- return currentTvPair != null;
- }
-
- @Override
- public TimeValuePair nextTimeValuePair() {
- if (!hasNextTimeValuePair()) {
- return null;
- }
- step();
- return currentTvPair;
- }
-
- @Override
- public TimeValuePair currentTimeValuePair() {
- if (!hasNextTimeValuePair()) {
- return null;
- }
- return currentTvPair;
- }
-
- public void step() {
- if (tvListIterators.size() == 1) {
- TVList.TVListIterator iterator = tvListIterators.get(0);
- iterator.step();
- tvListOffsets[0] = iterator.getIndex();
- } else {
- for (int index : probeIterators) {
- TVList.TVListIterator iterator = tvListIterators.get(index);
- iterator.step();
- tvListOffsets[index] = iterator.getIndex();
- }
- }
- probeNext = false;
- }
-
- @Override
- public long getUsedMemorySize() {
- // not used
- return 0;
- }
-
- @Override
- public void close() throws IOException {}
-
- public int[] getTVListOffsets() {
- return tvListOffsets;
- }
-
- public void setTVListOffsets(int[] tvListOffsets) {
- for (int i = 0; i < tvListIterators.size(); i++) {
- tvListIterators.get(i).setIndex(tvListOffsets[i]);
- this.tvListOffsets[i] = tvListOffsets[i];
- }
- if (tvListIterators.size() > 1) {
- minHeap.clear();
- probeIterators.clear();
- for (int i = 0; i < tvListIterators.size(); i++) {
- probeIterators.add(i);
- }
- }
- probeNext = false;
- }
-
- @Override
- public MergeSortTVListIterator clone() {
- MergeSortTVListIterator cloneIterator = new MergeSortTVListIterator();
- cloneIterator.tvListIterators = new ArrayList<>(tvListIterators.size());
- for (int i = 0; i < tvListIterators.size(); i++) {
- cloneIterator.tvListIterators.add(tvListIterators.get(i).clone());
- }
- cloneIterator.tvListOffsets = new int[tvListIterators.size()];
- cloneIterator.probeIterators =
- IntStream.range(0, tvListIterators.size()).boxed().collect(Collectors.toList());
- return cloneIterator;
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MultiAlignedTVListIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MultiAlignedTVListIterator.java
new file mode 100644
index 00000000000..e2ee1af4aca
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MultiAlignedTVListIterator.java
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.utils.datastructure;
+
+import org.apache.tsfile.block.column.ColumnBuilder;
+import org.apache.tsfile.common.conf.TSFileDescriptor;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.tsfile.read.TimeValuePair;
+import org.apache.tsfile.read.common.TimeRange;
+import org.apache.tsfile.read.common.block.TsBlock;
+import org.apache.tsfile.read.common.block.TsBlockBuilder;
+import org.apache.tsfile.read.common.block.column.TimeColumnBuilder;
+import org.apache.tsfile.utils.TsPrimitiveType;
+import org.apache.tsfile.write.UnSupportedDataTypeException;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public abstract class MultiAlignedTVListIterator implements MemPointIterator {
+ protected List<TSDataType> tsDataTypeList;
+ protected List<Integer> columnIndexList;
+ protected List<AlignedTVList.AlignedTVListIterator> alignedTvListIterators;
+ protected int floatPrecision;
+ protected List<TSEncoding> encodingList;
+ protected List<List<TimeRange>> valueColumnsDeletionList;
+
+ protected boolean probeNext = false;
+ protected boolean hasNext = false;
+
+ protected List<TsBlock> tsBlocks;
+ protected long currentTime;
+
+ protected final int MAX_NUMBER_OF_POINTS_IN_PAGE =
+ TSFileDescriptor.getInstance().getConfig().getMaxNumberOfPointsInPage();
+
+ protected MultiAlignedTVListIterator(
+ List<TSDataType> tsDataTypeList,
+ List<Integer> columnIndexList,
+ List<AlignedTVList> alignedTvLists,
+ List<List<TimeRange>> valueColumnsDeletionList,
+ Integer floatPrecision,
+ List<TSEncoding> encodingList) {
+ this.tsDataTypeList = tsDataTypeList;
+ this.columnIndexList = columnIndexList;
+ this.alignedTvListIterators = new ArrayList<>(alignedTvLists.size());
+ for (AlignedTVList alignedTvList : alignedTvLists) {
+ alignedTvListIterators.add(
+ alignedTvList.iterator(
+ tsDataTypeList, columnIndexList, null, floatPrecision, encodingList));
+ }
+ this.valueColumnsDeletionList = valueColumnsDeletionList;
+ this.floatPrecision = floatPrecision != null ? floatPrecision : 0;
+ this.encodingList = encodingList;
+ this.tsBlocks = new ArrayList<>();
+ }
+
+ @Override
+ public boolean hasNextTimeValuePair() {
+ if (!probeNext) {
+ prepareNext();
+ }
+ return hasNext;
+ }
+
+ @Override
+ public TimeValuePair nextTimeValuePair() {
+ if (!hasNextTimeValuePair()) {
+ return null;
+ }
+ TsPrimitiveType[] vector = new TsPrimitiveType[tsDataTypeList.size()];
+ for (int columnIndex = 0; columnIndex < tsDataTypeList.size(); columnIndex++) {
+ AlignedTVList.AlignedTVListIterator iterator =
+ alignedTvListIterators.get(currentIteratorIndex(columnIndex));
+ vector[columnIndex] =
+ iterator.getPrimitiveTypeObject(currentRowIndex(columnIndex), columnIndex);
+ }
+ TimeValuePair currentTvPair =
+ new TimeValuePair(currentTime, TsPrimitiveType.getByType(TSDataType.VECTOR, vector));
+ next();
+ return currentTvPair;
+ }
+
+ @Override
+ public TimeValuePair currentTimeValuePair() {
+ if (!hasNextTimeValuePair()) {
+ return null;
+ }
+ TsPrimitiveType[] vector = new TsPrimitiveType[tsDataTypeList.size()];
+ for (int columnIndex = 0; columnIndex < tsDataTypeList.size(); columnIndex++) {
+ AlignedTVList.AlignedTVListIterator iterator =
+ alignedTvListIterators.get(currentIteratorIndex(columnIndex));
+ vector[columnIndex] =
+ iterator.getPrimitiveTypeObject(currentRowIndex(columnIndex), columnIndex);
+ }
+ return new TimeValuePair(currentTime, TsPrimitiveType.getByType(TSDataType.VECTOR, vector));
+ }
+
+ @Override
+ public boolean hasNextBatch() {
+ return hasNextTimeValuePair();
+ }
+
+ @Override
+ public TsBlock nextBatch() {
+ TsBlockBuilder builder = new TsBlockBuilder(tsDataTypeList);
+ // Time column
+ TimeColumnBuilder timeBuilder = builder.getTimeColumnBuilder();
+
+ while (hasNextTimeValuePair() && builder.getPositionCount() < MAX_NUMBER_OF_POINTS_IN_PAGE) {
+ timeBuilder.writeLong(currentTime);
+ for (int columnIndex = 0; columnIndex < tsDataTypeList.size(); columnIndex++) {
+ // Value column
+ ColumnBuilder valueBuilder = builder.getColumnBuilder(columnIndex);
+ AlignedTVList alignedTVList =
+ alignedTvListIterators.get(currentIteratorIndex(columnIndex)).getAlignedTVList();
+
+ // sanity check
+ int validColumnIndex =
+ columnIndexList != null ? columnIndexList.get(columnIndex) : columnIndex;
+ if (validColumnIndex < 0 || validColumnIndex >= alignedTVList.dataTypes.size()) {
+ valueBuilder.appendNull();
+ continue;
+ }
+
+ // null value
+ if (alignedTVList.isNullValue(currentRowIndex(columnIndex), validColumnIndex)) {
+ valueBuilder.appendNull();
+ continue;
+ }
+
+ switch (tsDataTypeList.get(columnIndex)) {
+ case BOOLEAN:
+ valueBuilder.writeBoolean(
+ alignedTVList.getBooleanByValueIndex(
+ currentRowIndex(columnIndex), validColumnIndex));
+ break;
+ case INT32:
+ case DATE:
+ valueBuilder.writeInt(
+ alignedTVList.getIntByValueIndex(currentRowIndex(columnIndex), validColumnIndex));
+ break;
+ case INT64:
+ case TIMESTAMP:
+ valueBuilder.writeLong(
+ alignedTVList.getLongByValueIndex(currentRowIndex(columnIndex), validColumnIndex));
+ break;
+ case FLOAT:
+ float valueF =
+ alignedTVList.getFloatByValueIndex(currentRowIndex(columnIndex), validColumnIndex);
+ if (encodingList != null) {
+ valueF =
+ alignedTVList.roundValueWithGivenPrecision(
+ valueF, floatPrecision, encodingList.get(columnIndex));
+ }
+ valueBuilder.writeFloat(valueF);
+ break;
+ case DOUBLE:
+ double valueD =
+ alignedTVList.getDoubleByValueIndex(currentRowIndex(columnIndex), validColumnIndex);
+ if (encodingList != null) {
+ valueD =
+ alignedTVList.roundValueWithGivenPrecision(
+ valueD, floatPrecision, encodingList.get(columnIndex));
+ }
+ valueBuilder.writeDouble(valueD);
+ break;
+ case TEXT:
+ case BLOB:
+ case STRING:
+ valueBuilder.writeBinary(
+ alignedTVList.getBinaryByValueIndex(
+ currentRowIndex(columnIndex), validColumnIndex));
+ break;
+ default:
+ throw new UnSupportedDataTypeException(
+ String.format("Data type %s is not supported.", tsDataTypeList.get(columnIndex)));
+ }
+ }
+ next();
+
+ builder.declarePosition();
+ }
+ TsBlock tsBlock = builder.build();
+ tsBlocks.add(tsBlock);
+ return tsBlock;
+ }
+
+ @Override
+ public TsBlock getBatch(int tsBlockIndex) {
+ if (tsBlockIndex < 0 || tsBlockIndex >= tsBlocks.size()) {
+ return null;
+ }
+ return tsBlocks.get(tsBlockIndex);
+ }
+
+ @Override
+ public long getUsedMemorySize() {
+ // not used
+ return 0;
+ }
+
+ protected abstract int currentIteratorIndex(int columnIndex);
+
+ protected abstract int currentRowIndex(int columnIndex);
+
+ @Override
+ public void close() throws IOException {}
+
+ protected abstract void prepareNext();
+
+ protected abstract void next();
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MultiTVListIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MultiTVListIterator.java
new file mode 100644
index 00000000000..cb203a5abb9
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MultiTVListIterator.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.utils.datastructure;
+
+import org.apache.tsfile.common.conf.TSFileDescriptor;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.tsfile.read.TimeValuePair;
+import org.apache.tsfile.read.common.TimeRange;
+import org.apache.tsfile.read.common.block.TsBlock;
+import org.apache.tsfile.read.common.block.TsBlockBuilder;
+import org.apache.tsfile.write.UnSupportedDataTypeException;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public abstract class MultiTVListIterator implements MemPointIterator {
+ protected TSDataType tsDataType;
+ protected List<TVList.TVListIterator> tvListIterators;
+ protected List<TsBlock> tsBlocks;
+ protected int floatPrecision;
+ protected TSEncoding encoding;
+
+ protected boolean probeNext = false;
+ protected boolean hasNext = false;
+ protected int iteratorIndex = 0;
+ protected int rowIndex = 0;
+
+ protected final int MAX_NUMBER_OF_POINTS_IN_PAGE =
+ TSFileDescriptor.getInstance().getConfig().getMaxNumberOfPointsInPage();
+
+ protected MultiTVListIterator(
+ TSDataType tsDataType,
+ List<TVList> tvLists,
+ List<TimeRange> deletionList,
+ Integer floatPrecision,
+ TSEncoding encoding) {
+ this.tsDataType = tsDataType;
+ this.tvListIterators = new ArrayList<>(tvLists.size());
+ for (TVList tvList : tvLists) {
+ tvListIterators.add(tvList.iterator(deletionList, null, null));
+ }
+ this.floatPrecision = floatPrecision != null ? floatPrecision : 0;
+ this.encoding = encoding;
+ this.tsBlocks = new ArrayList<>();
+ }
+
+ @Override
+ public boolean hasNextTimeValuePair() {
+ if (!probeNext) {
+ prepareNext();
+ }
+ return hasNext;
+ }
+
+ @Override
+ public TimeValuePair nextTimeValuePair() {
+ if (!hasNextTimeValuePair()) {
+ return null;
+ }
+ TVList.TVListIterator iterator = tvListIterators.get(iteratorIndex);
+ TimeValuePair currentTvPair =
+ iterator
+ .getTVList()
+ .getTimeValuePair(rowIndex, iterator.currentTime(), floatPrecision, encoding);
+ next();
+ return currentTvPair;
+ }
+
+ @Override
+ public TimeValuePair currentTimeValuePair() {
+ if (!hasNextTimeValuePair()) {
+ return null;
+ }
+ TVList.TVListIterator iterator = tvListIterators.get(iteratorIndex);
+ return iterator
+ .getTVList()
+ .getTimeValuePair(rowIndex, iterator.currentTime(), floatPrecision, encoding);
+ }
+
+ @Override
+ public boolean hasNextBatch() {
+ return hasNextTimeValuePair();
+ }
+
+ @Override
+ public TsBlock nextBatch() {
+ TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(tsDataType));
+ while (hasNextTimeValuePair() && builder.getPositionCount() < MAX_NUMBER_OF_POINTS_IN_PAGE) {
+ TVList.TVListIterator iterator = tvListIterators.get(iteratorIndex);
+ builder.getTimeColumnBuilder().writeLong(iterator.currentTime());
+ switch (tsDataType) {
+ case BOOLEAN:
+ builder.getColumnBuilder(0).writeBoolean(iterator.getTVList().getBoolean(rowIndex));
+ break;
+ case INT32:
+ case DATE:
+ builder.getColumnBuilder(0).writeInt(iterator.getTVList().getInt(rowIndex));
+ break;
+ case INT64:
+ case TIMESTAMP:
+ builder.getColumnBuilder(0).writeLong(iterator.getTVList().getLong(rowIndex));
+ break;
+ case FLOAT:
+ TVList floatTvList = iterator.getTVList();
+ builder
+ .getColumnBuilder(0)
+ .writeFloat(
+ floatTvList.roundValueWithGivenPrecision(
+ floatTvList.getFloat(rowIndex), floatPrecision, encoding));
+ break;
+ case DOUBLE:
+ TVList doubleTvList = iterator.getTVList();
+ builder
+ .getColumnBuilder(0)
+ .writeDouble(
+ doubleTvList.roundValueWithGivenPrecision(
+ doubleTvList.getDouble(rowIndex), floatPrecision, encoding));
+ break;
+ case TEXT:
+ case BLOB:
+ case STRING:
+ builder.getColumnBuilder(0).writeBinary(iterator.getTVList().getBinary(rowIndex));
+ break;
+ default:
+ throw new UnSupportedDataTypeException(
+ String.format("Data type %s is not supported.", tsDataType));
+ }
+ next();
+
+ builder.declarePosition();
+ }
+ TsBlock tsBlock = builder.build();
+ tsBlocks.add(tsBlock);
+ return tsBlock;
+ }
+
+ @Override
+ public TsBlock getBatch(int tsBlockIndex) {
+ if (tsBlockIndex < 0 || tsBlockIndex >= tsBlocks.size()) {
+ return null;
+ }
+ return tsBlocks.get(tsBlockIndex);
+ }
+
+ @Override
+ public long getUsedMemorySize() {
+ // not used
+ return 0;
+ }
+
+ @Override
+ public void close() throws IOException {
+ tsBlocks.clear();
+ }
+
+ protected abstract void prepareNext();
+
+ protected abstract void next();
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/OrderedMultiAlignedTVListIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/OrderedMultiAlignedTVListIterator.java
new file mode 100644
index 00000000000..4dae6e840c7
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/OrderedMultiAlignedTVListIterator.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.utils.datastructure;
+
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.tsfile.read.common.TimeRange;
+import org.apache.tsfile.utils.BitMap;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.iotdb.db.utils.ModificationUtils.isPointDeleted;
+
+public class OrderedMultiAlignedTVListIterator extends MultiAlignedTVListIterator {
+ private final BitMap bitMap;
+ private final List<int[]> valueColumnDeleteCursor;
+ private int iteratorIndex = 0;
+ private int[] rowIndices;
+
+ public OrderedMultiAlignedTVListIterator(
+ List<TSDataType> tsDataTypes,
+ List<Integer> columnIndexList,
+ List<AlignedTVList> alignedTvLists,
+ List<List<TimeRange>> valueColumnsDeletionList,
+ Integer floatPrecision,
+ List<TSEncoding> encodingList) {
+ super(
+ tsDataTypes,
+ columnIndexList,
+ alignedTvLists,
+ valueColumnsDeletionList,
+ floatPrecision,
+ encodingList);
+ this.bitMap = new BitMap(tsDataTypeList.size());
+ this.valueColumnDeleteCursor = new ArrayList<>();
+ for (int i = 0; i < tsDataTypeList.size(); i++) {
+ valueColumnDeleteCursor.add(new int[] {0});
+ }
+ }
+
+ @Override
+ protected void prepareNext() {
+ hasNext = false;
+
+ while (iteratorIndex < alignedTvListIterators.size() && !hasNext) {
+ AlignedTVList.AlignedTVListIterator iterator = alignedTvListIterators.get(iteratorIndex);
+ if (!iterator.hasNextTimeValuePair()) {
+ iteratorIndex++;
+ continue;
+ }
+
+ bitMap.reset();
+ rowIndices = iterator.getSelectedIndices();
+ currentTime = iterator.currentTime();
+ hasNext = true;
+
+ // check valueColumnsDeletionList
+ for (int columnIndex = 0; columnIndex < tsDataTypeList.size(); columnIndex++) {
+ if ((valueColumnsDeletionList != null
+ && isPointDeleted(
+ currentTime,
+ valueColumnsDeletionList.get(columnIndex),
+ valueColumnDeleteCursor.get(columnIndex)))
+ || iterator.isNullValue(rowIndices[columnIndex], columnIndex)) {
+ bitMap.mark(columnIndex);
+ }
+ }
+ if (bitMap.isAllMarked()) {
+ iterator.next();
+ hasNext = false;
+ }
+ }
+ probeNext = true;
+ }
+
+ @Override
+ protected void next() {
+ TVList.TVListIterator iterator = alignedTvListIterators.get(iteratorIndex);
+ iterator.next();
+ rowIndices = null;
+ probeNext = false;
+ }
+
+ @Override
+ protected int currentIteratorIndex(int columnIndex) {
+ return iteratorIndex;
+ }
+
+ @Override
+ protected int currentRowIndex(int columnIndex) {
+ return rowIndices[columnIndex];
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/OrderedMultiTVListIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/OrderedMultiTVListIterator.java
new file mode 100644
index 00000000000..48e58aceab1
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/OrderedMultiTVListIterator.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.utils.datastructure;
+
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.tsfile.read.common.TimeRange;
+
+import java.util.List;
+
+public class OrderedMultiTVListIterator extends MultiTVListIterator {
+ public OrderedMultiTVListIterator(
+ TSDataType tsDataType,
+ List<TVList> tvLists,
+ List<TimeRange> deletionList,
+ Integer floatPrecision,
+ TSEncoding encoding) {
+ super(tsDataType, tvLists, deletionList, floatPrecision, encoding);
+ }
+
+ @Override
+ protected void prepareNext() {
+ hasNext = false;
+ while (iteratorIndex < tvListIterators.size() - 1
+ && !tvListIterators.get(iteratorIndex).hasNextTimeValuePair()) {
+ iteratorIndex++;
+ }
+ TVList.TVListIterator iterator = tvListIterators.get(iteratorIndex);
+ if (iterator.hasNextTimeValuePair()) {
+ rowIndex = iterator.getIndex();
+ hasNext = true;
+ }
+ probeNext = true;
+ }
+
+ @Override
+ protected void next() {
+ tvListIterators.get(iteratorIndex).next();
+ probeNext = false;
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
index bdbcaefd238..6c88c1526f1 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue;
import org.apache.iotdb.db.storageengine.rescon.memory.PrimitiveArrayManager;
import org.apache.iotdb.db.utils.MathUtils;
+import org.apache.tsfile.common.conf.TSFileDescriptor;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
import org.apache.tsfile.read.TimeValuePair;
@@ -45,6 +46,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantLock;
import static org.apache.iotdb.db.storageengine.rescon.memory.PrimitiveArrayManager.ARRAY_SIZE;
+import static org.apache.iotdb.db.utils.ModificationUtils.isPointDeleted;
import static org.apache.tsfile.utils.RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
import static org.apache.tsfile.utils.RamUsageEstimator.NUM_BYTES_OBJECT_REF;
@@ -83,6 +85,8 @@ public abstract class TVList implements WALEntryValue {
protected AtomicInteger referenceCount;
private long version;
+ private final TVList outer = this;
+
protected TVList() {
timestamps = new ArrayList<>();
rowCount = 0;
@@ -632,83 +636,208 @@ public abstract class TVList implements WALEntryValue {
queryListLock.unlock();
}
- public TVListIterator iterator(Integer floatPrecision, TSEncoding encoding) {
- return new TVListIterator(floatPrecision, encoding);
+ public TVListIterator iterator(
+ List<TimeRange> deletionList, Integer floatPrecision, TSEncoding encoding) {
+ return new TVListIterator(deletionList, floatPrecision, encoding);
}
/* TVList Iterator */
- public class TVListIterator {
+ public class TVListIterator implements MemPointIterator {
protected int index;
protected int rows;
- protected long currentTime;
protected boolean probeNext;
- private Integer floatPrecision;
- private TSEncoding encoding;
+ protected List<TsBlock> tsBlocks;
+
+ private final List<TimeRange> deletionList;
+ private final int[] deleteCursor = {0};
+ private final int floatPrecision;
+ private final TSEncoding encoding;
- public TVListIterator() {}
+ private final int MAX_NUMBER_OF_POINTS_IN_PAGE =
+ TSFileDescriptor.getInstance().getConfig().getMaxNumberOfPointsInPage();
- public TVListIterator(Integer floatPrecision, TSEncoding encoding) {
+ public TVListIterator(
+ List<TimeRange> deletionList, Integer floatPrecision, TSEncoding encoding) {
+ this.deletionList = deletionList;
+ this.floatPrecision = floatPrecision != null ? floatPrecision : 0;
+ this.encoding = encoding;
this.index = 0;
this.rows = rowCount;
- this.currentTime = index < rows ? getTime(index) : Long.MIN_VALUE;
- this.floatPrecision = floatPrecision;
- this.encoding = encoding;
+ this.probeNext = false;
+ this.tsBlocks = new ArrayList<>();
}
- private void prepareNext() {
+ protected void prepareNext() {
// skip deleted rows
- int prevIndex = index;
- while (index < rows && (bitMap != null && isNullValue(getValueIndex(index)))) {
+ while (index < rows
+ && (isNullValue(getValueIndex(index))
+ || isPointDeleted(getTime(index), deletionList, deleteCursor))) {
index++;
}
- // update current timestamp if needed
- if (index > prevIndex) {
- currentTime = index < rows ? getTime(index) : Long.MIN_VALUE;
- }
// skip duplicated timestamp
- while (index + 1 < rows && getTime(index + 1) == currentTime) {
+ while (index + 1 < rows && getTime(index + 1) == getTime(index)) {
index++;
}
probeNext = true;
}
- public boolean hasNext() {
+ @Override
+ public boolean hasNextTimeValuePair() {
if (!probeNext) {
prepareNext();
}
return index < rows;
}
- public TimeValuePair next() {
- if (!hasNext()) {
+ @Override
+ public TimeValuePair nextTimeValuePair() {
+ if (!hasNextTimeValuePair()) {
return null;
}
- TimeValuePair ret = getTimeValuePair(index++, currentTime, floatPrecision, encoding);
- currentTime = index < rows ? getTime(index) : Long.MIN_VALUE;
- probeNext = false;
- return ret;
+ TimeValuePair tvp = getTimeValuePair(index);
+ next();
+ return tvp;
}
- public TimeValuePair current() {
- if (!hasCurrent()) {
+ @Override
+ public TimeValuePair currentTimeValuePair() {
+ if (!hasNextTimeValuePair()) {
return null;
}
- return getTimeValuePair(index, currentTime, floatPrecision, encoding);
+ return getTimeValuePair(index);
}
- public boolean hasCurrent() {
- if (bitMap == null) {
- return index < rows;
+ @Override
+ public TsBlock getBatch(int tsBlockIndex) {
+ if (tsBlockIndex < 0 || tsBlockIndex >= tsBlocks.size()) {
+ return null;
+ }
+ return tsBlocks.get(tsBlockIndex);
+ }
+
+ @Override
+ public boolean hasNextBatch() {
+ return hasNextTimeValuePair();
+ }
+
+ @Override
+ public TsBlock nextBatch() {
+ TSDataType dataType = getDataType();
+ TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(dataType));
+ switch (dataType) {
+ case BOOLEAN:
+ while (index < rows && builder.getPositionCount() < MAX_NUMBER_OF_POINTS_IN_PAGE) {
+ if (!isNullValue(getValueIndex(index))
+ && !isPointDeleted(getTime(index), deletionList, deleteCursor)
+ && (index == rows - 1 || getTime(index) != getTime(index + 1))) {
+ builder.getTimeColumnBuilder().writeLong(getTime(index));
+ builder.getColumnBuilder(0).writeBoolean(getBoolean(index));
+ builder.declarePosition();
+ }
+ index++;
+ }
+ break;
+ case INT32:
+ case DATE:
+ while (index < rows && builder.getPositionCount() < MAX_NUMBER_OF_POINTS_IN_PAGE) {
+ if (!isNullValue(getValueIndex(index))
+ && !isPointDeleted(getTime(index), deletionList, deleteCursor)
+ && (index == rows - 1 || getTime(index) != getTime(index + 1))) {
+ builder.getTimeColumnBuilder().writeLong(getTime(index));
+ builder.getColumnBuilder(0).writeInt(getInt(index));
+ builder.declarePosition();
+ }
+ index++;
+ }
+ break;
+ case INT64:
+ case TIMESTAMP:
+ while (index < rows && builder.getPositionCount() < MAX_NUMBER_OF_POINTS_IN_PAGE) {
+ if (!isNullValue(getValueIndex(index))
+ && !isPointDeleted(getTime(index), deletionList, deleteCursor)
+ && (index == rows - 1 || getTime(index) != getTime(index + 1))) {
+ builder.getTimeColumnBuilder().writeLong(getTime(index));
+ builder.getColumnBuilder(0).writeLong(getLong(index));
+ builder.declarePosition();
+ }
+ index++;
+ }
+ break;
+ case FLOAT:
+ while (index < rows && builder.getPositionCount() < MAX_NUMBER_OF_POINTS_IN_PAGE) {
+ if (!isNullValue(getValueIndex(index))
+ && !isPointDeleted(getTime(index), deletionList, deleteCursor)
+ && (index == rows - 1 || getTime(index) != getTime(index + 1))) {
+ builder.getTimeColumnBuilder().writeLong(getTime(index));
+ builder
+ .getColumnBuilder(0)
+ .writeFloat(
+ roundValueWithGivenPrecision(getFloat(index), floatPrecision, encoding));
+ builder.declarePosition();
+ }
+ index++;
+ }
+ break;
+ case DOUBLE:
+ while (index < rows && builder.getPositionCount() < MAX_NUMBER_OF_POINTS_IN_PAGE) {
+ if (!isNullValue(getValueIndex(index))
+ && !isPointDeleted(getTime(index), deletionList, deleteCursor)
+ && (index == rows - 1 || getTime(index) != getTime(index + 1))) {
+ builder.getTimeColumnBuilder().writeLong(getTime(index));
+ builder
+ .getColumnBuilder(0)
+ .writeDouble(
+ roundValueWithGivenPrecision(getDouble(index), floatPrecision, encoding));
+ builder.declarePosition();
+ }
+ index++;
+ }
+ break;
+ case TEXT:
+ case BLOB:
+ case STRING:
+ while (index < rows && builder.getPositionCount() < MAX_NUMBER_OF_POINTS_IN_PAGE) {
+ if (!isNullValue(getValueIndex(index))
+ && !isPointDeleted(getTime(index), deletionList, deleteCursor)
+ && (index == rows - 1 || getTime(index) != getTime(index + 1))) {
+ builder.getTimeColumnBuilder().writeLong(getTime(index));
+ builder.getColumnBuilder(0).writeBinary(getBinary(index));
+ builder.declarePosition();
+ }
+ index++;
+ }
+ break;
}
- return index < rows && !isNullValue(getValueIndex(index));
+ TsBlock tsBlock = builder.build();
+ tsBlocks.add(tsBlock);
+ return tsBlock;
+ }
+
+ @Override
+ public long getUsedMemorySize() {
+ return 0;
+ }
+
+ @Override
+ public void close() throws IOException {
+ tsBlocks.clear();
+ }
+
+ public void next() {
+ index++;
+ probeNext = false;
+ }
+
+ public boolean hasCurrent() {
+ return index < rows;
}
public long currentTime() {
if (!hasCurrent()) {
return Long.MIN_VALUE;
}
- return currentTime;
+ return getTime(index);
}
public int getIndex() {
@@ -718,29 +847,15 @@ public abstract class TVList implements WALEntryValue {
public void setIndex(int index) {
this.index = index;
this.probeNext = false;
- this.currentTime = index < rows ? getTime(index) : Long.MIN_VALUE;
- }
-
- protected void step() {
- index++;
- probeNext = false;
- currentTime = index < rows ? getTime(index) : Long.MIN_VALUE;
}
public void reset() {
index = 0;
- currentTime = index < rows ? getTime(index) : Long.MIN_VALUE;
probeNext = false;
}
- @Override
- public TVListIterator clone() {
- TVListIterator iterator = new TVListIterator();
- iterator.rows = rows;
- iterator.floatPrecision = floatPrecision;
- iterator.encoding = encoding;
- iterator.reset();
- return iterator;
+ public TVList getTVList() {
+ return outer;
}
}
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/AlignedMemPageReaderTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/AlignedMemPageReaderTest.java
index ca98504ce97..f0623f82b51 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/AlignedMemPageReaderTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/AlignedMemPageReaderTest.java
@@ -104,9 +104,7 @@ public class AlignedMemPageReaderTest {
MemAlignedPageReader alignedPageReader =
new MemAlignedPageReader(
tsBlockSupplier,
- null,
- null,
- null,
+ 0,
Arrays.asList(TSDataType.INT32, TSDataType.INT32),
chunkMetadata1.getTimeStatistics(),
valueStatistcsList,
@@ -124,9 +122,7 @@ public class AlignedMemPageReaderTest {
MemAlignedPageReader alignedPageReader =
new MemAlignedPageReader(
tsBlockSupplier,
- null,
- null,
- null,
+ 0,
Arrays.asList(TSDataType.INT32),
chunkMetadata2.getTimeStatistics(),
valueStatistcsList,
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkLoaderTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkLoaderTest.java
index 33b249b9f6e..1422ec48e11 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkLoaderTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkLoaderTest.java
@@ -22,7 +22,8 @@ package org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk;
import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext;
import org.apache.iotdb.db.storageengine.dataregion.memtable.AlignedReadOnlyMemChunk;
import org.apache.iotdb.db.utils.datastructure.AlignedTVList;
-import org.apache.iotdb.db.utils.datastructure.MergeSortAlignedTVListIterator;
+import org.apache.iotdb.db.utils.datastructure.MemPointIterator;
+import org.apache.iotdb.db.utils.datastructure.MemPointIteratorFactory;
import org.apache.iotdb.db.utils.datastructure.TVList;
import org.apache.tsfile.common.conf.TSFileConfig;
@@ -103,8 +104,6 @@ public class MemAlignedChunkLoaderTest {
Mockito.when(chunk.getValuesStatisticsList()).thenReturn(valuesStatitsticsList);
// Mock AlignedReadOnlyMemChunk Getter
- List<int[]> pageOffsets = Arrays.asList(new int[] {0, 0}, new int[] {2, 1});
- Mockito.when(chunk.getPageOffsetsList()).thenReturn(pageOffsets);
List<TSDataType> dataTypes = buildTsDataTypes();
Mockito.when(chunk.getDataTypes()).thenReturn(dataTypes);
Mockito.when(chunk.getValueColumnsDeletionList()).thenReturn(null);
@@ -114,9 +113,10 @@ public class MemAlignedChunkLoaderTest {
Mockito.when(chunk.getAligendTvListQueryMap()).thenReturn(alignedTvListMap);
List<AlignedTVList> alignedTvLists =
alignedTvListMap.keySet().stream().map(x -> (AlignedTVList) x).collect(Collectors.toList());
- MergeSortAlignedTVListIterator timeValuePairIterator =
- new MergeSortAlignedTVListIterator(alignedTvLists, dataTypes, null, null, null);
- Mockito.when(chunk.getMergeSortAlignedTVListIterator()).thenReturn(timeValuePairIterator);
+ MemPointIterator timeValuePairIterator =
+ MemPointIteratorFactory.create(dataTypes, null, alignedTvLists);
+ timeValuePairIterator.nextBatch();
+ Mockito.when(chunk.getMemPointIterator()).thenReturn(timeValuePairIterator);
AlignedChunkMetadata chunkMetadata1 = Mockito.mock(AlignedChunkMetadata.class);
Mockito.when(chunk.getChunkMetaData()).thenReturn(chunkMetadata1);
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemChunkLoaderTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemChunkLoaderTest.java
index b56f2720ef0..7ed50f2e009 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemChunkLoaderTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemChunkLoaderTest.java
@@ -21,7 +21,8 @@ package org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk;
import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext;
import org.apache.iotdb.db.storageengine.dataregion.memtable.ReadOnlyMemChunk;
-import org.apache.iotdb.db.utils.datastructure.MergeSortTVListIterator;
+import org.apache.iotdb.db.utils.datastructure.MemPointIterator;
+import org.apache.iotdb.db.utils.datastructure.MemPointIteratorFactory;
import org.apache.iotdb.db.utils.datastructure.TVList;
import org.apache.tsfile.common.conf.TSFileConfig;
@@ -44,7 +45,6 @@ import org.mockito.Mockito;
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
@@ -68,9 +68,10 @@ public class MemChunkLoaderTest {
Map<TVList, Integer> booleanTvListMap = buildBooleanTvListMap();
Mockito.when(chunk.getTvListQueryMap()).thenReturn(booleanTvListMap);
List<TVList> booleanTvLists = new ArrayList<>(booleanTvListMap.keySet());
- MergeSortTVListIterator timeValuePairIterator =
- new MergeSortTVListIterator(booleanTvLists, null, null);
- Mockito.when(chunk.getMergeSortTVListIterator()).thenReturn(timeValuePairIterator);
+ MemPointIterator timeValuePairIterator =
+ MemPointIteratorFactory.create(TSDataType.BOOLEAN, booleanTvLists);
+ timeValuePairIterator.nextBatch();
+ Mockito.when(chunk.getMemPointIterator()).thenReturn(timeValuePairIterator);
ChunkMetadata chunkMetadata = Mockito.mock(ChunkMetadata.class);
MemChunkLoader memChunkLoader = new MemChunkLoader(new QueryContext(), chunk);
@@ -83,9 +84,7 @@ public class MemChunkLoaderTest {
Statistics<? extends Serializable> pageStatistics = Mockito.mock(BooleanStatistics.class);
List<Statistics<? extends Serializable>> pageStats = Collections.singletonList(pageStatistics);
- List<int[]> pageOffsets = Arrays.asList(new int[] {0, 0}, new int[] {2, 1});
Mockito.when(chunk.getPageStatisticsList()).thenReturn(pageStats);
- Mockito.when(chunk.getPageOffsetsList()).thenReturn(pageOffsets);
ChunkMetadata chunkMetadata1 = Mockito.mock(ChunkMetadata.class);
Mockito.when(chunk.getChunkMetaData()).thenReturn(chunkMetadata1);
@@ -148,9 +147,10 @@ public class MemChunkLoaderTest {
Map<TVList, Integer> int32TvListMap = buildInt32TvListMap();
Mockito.when(chunk.getTvListQueryMap()).thenReturn(int32TvListMap);
List<TVList> int32TvLists = new ArrayList<>(int32TvListMap.keySet());
- MergeSortTVListIterator timeValuePairIterator =
- new MergeSortTVListIterator(int32TvLists, null, null);
- Mockito.when(chunk.getMergeSortTVListIterator()).thenReturn(timeValuePairIterator);
+ MemPointIterator timeValuePairIterator =
+ MemPointIteratorFactory.create(TSDataType.INT32, int32TvLists);
+ timeValuePairIterator.nextBatch();
+ Mockito.when(chunk.getMemPointIterator()).thenReturn(timeValuePairIterator);
ChunkMetadata chunkMetadata = Mockito.mock(ChunkMetadata.class);
MemChunkLoader memChunkLoader = new MemChunkLoader(new QueryContext(), chunk);
@@ -163,9 +163,7 @@ public class MemChunkLoaderTest {
Statistics<? extends Serializable> pageStatistics = Mockito.mock(IntegerStatistics.class);
List<Statistics<? extends Serializable>> pageStats = Collections.singletonList(pageStatistics);
- List<int[]> pageOffsets = Arrays.asList(new int[] {0, 0}, new int[] {2, 1});
Mockito.when(chunk.getPageStatisticsList()).thenReturn(pageStats);
- Mockito.when(chunk.getPageOffsetsList()).thenReturn(pageOffsets);
ChunkMetadata chunkMetadata1 = Mockito.mock(ChunkMetadata.class);
Mockito.when(chunk.getChunkMetaData()).thenReturn(chunkMetadata1);
@@ -229,9 +227,10 @@ public class MemChunkLoaderTest {
Map<TVList, Integer> int64TvListMap = buildInt64TvListMap();
Mockito.when(chunk.getTvListQueryMap()).thenReturn(int64TvListMap);
List<TVList> int64TvLists = new ArrayList<>(int64TvListMap.keySet());
- MergeSortTVListIterator timeValuePairIterator =
- new MergeSortTVListIterator(int64TvLists, null, null);
- Mockito.when(chunk.getMergeSortTVListIterator()).thenReturn(timeValuePairIterator);
+ MemPointIterator timeValuePairIterator =
+ MemPointIteratorFactory.create(TSDataType.INT64, int64TvLists);
+ timeValuePairIterator.nextBatch();
+ Mockito.when(chunk.getMemPointIterator()).thenReturn(timeValuePairIterator);
ChunkMetadata chunkMetadata = Mockito.mock(ChunkMetadata.class);
MemChunkLoader memChunkLoader = new MemChunkLoader(new QueryContext(), chunk);
@@ -244,9 +243,7 @@ public class MemChunkLoaderTest {
Statistics<? extends Serializable> pageStatistics = Mockito.mock(LongStatistics.class);
List<Statistics<? extends Serializable>> pageStats = Collections.singletonList(pageStatistics);
- List<int[]> pageOffsets = Arrays.asList(new int[] {0, 0}, new int[] {2, 1});
Mockito.when(chunk.getPageStatisticsList()).thenReturn(pageStats);
- Mockito.when(chunk.getPageOffsetsList()).thenReturn(pageOffsets);
ChunkMetadata chunkMetadata1 = Mockito.mock(ChunkMetadata.class);
Mockito.when(chunk.getChunkMetaData()).thenReturn(chunkMetadata1);
@@ -310,9 +307,10 @@ public class MemChunkLoaderTest {
Map<TVList, Integer> floatTvListMap = buildFloatTvListMap();
Mockito.when(chunk.getTvListQueryMap()).thenReturn(floatTvListMap);
List<TVList> floatTvLists = new ArrayList<>(floatTvListMap.keySet());
- MergeSortTVListIterator timeValuePairIterator =
- new MergeSortTVListIterator(floatTvLists, null, null);
- Mockito.when(chunk.getMergeSortTVListIterator()).thenReturn(timeValuePairIterator);
+ MemPointIterator timeValuePairIterator =
+ MemPointIteratorFactory.create(TSDataType.FLOAT, floatTvLists);
+ timeValuePairIterator.nextBatch();
+ Mockito.when(chunk.getMemPointIterator()).thenReturn(timeValuePairIterator);
ChunkMetadata chunkMetadata = Mockito.mock(ChunkMetadata.class);
MemChunkLoader memChunkLoader = new MemChunkLoader(new QueryContext(), chunk);
@@ -325,9 +323,7 @@ public class MemChunkLoaderTest {
Statistics<? extends Serializable> pageStatistics = Mockito.mock(FloatStatistics.class);
List<Statistics<? extends Serializable>> pageStats = Collections.singletonList(pageStatistics);
- List<int[]> pageOffsets = Arrays.asList(new int[] {0, 0}, new int[] {2, 1});
Mockito.when(chunk.getPageStatisticsList()).thenReturn(pageStats);
- Mockito.when(chunk.getPageOffsetsList()).thenReturn(pageOffsets);
ChunkMetadata chunkMetadata1 = Mockito.mock(ChunkMetadata.class);
Mockito.when(chunk.getChunkMetaData()).thenReturn(chunkMetadata1);
@@ -391,9 +387,10 @@ public class MemChunkLoaderTest {
Map<TVList, Integer> doubleTvListMap = buildDoubleTvListMap();
Mockito.when(chunk.getTvListQueryMap()).thenReturn(doubleTvListMap);
List<TVList> doubleTvLists = new ArrayList<>(doubleTvListMap.keySet());
- MergeSortTVListIterator timeValuePairIterator =
- new MergeSortTVListIterator(doubleTvLists, null, null);
- Mockito.when(chunk.getMergeSortTVListIterator()).thenReturn(timeValuePairIterator);
+ MemPointIterator timeValuePairIterator =
+ MemPointIteratorFactory.create(TSDataType.DOUBLE, doubleTvLists);
+ timeValuePairIterator.nextBatch();
+ Mockito.when(chunk.getMemPointIterator()).thenReturn(timeValuePairIterator);
ChunkMetadata chunkMetadata = Mockito.mock(ChunkMetadata.class);
MemChunkLoader memChunkLoader = new MemChunkLoader(new QueryContext(), chunk);
@@ -406,9 +403,7 @@ public class MemChunkLoaderTest {
Statistics<? extends Serializable> pageStatistics = Mockito.mock(DoubleStatistics.class);
List<Statistics<? extends Serializable>> pageStats = Collections.singletonList(pageStatistics);
- List<int[]> pageOffsets = Arrays.asList(new int[] {0, 0}, new int[] {2, 1});
Mockito.when(chunk.getPageStatisticsList()).thenReturn(pageStats);
- Mockito.when(chunk.getPageOffsetsList()).thenReturn(pageOffsets);
ChunkMetadata chunkMetadata1 = Mockito.mock(ChunkMetadata.class);
Mockito.when(chunk.getChunkMetaData()).thenReturn(chunkMetadata1);
@@ -472,9 +467,10 @@ public class MemChunkLoaderTest {
Map<TVList, Integer> textTvListMap = buildTextTvListMap();
Mockito.when(chunk.getTvListQueryMap()).thenReturn(textTvListMap);
List<TVList> textTvLists = new ArrayList<>(textTvListMap.keySet());
- MergeSortTVListIterator timeValuePairIterator =
- new MergeSortTVListIterator(textTvLists, null, null);
- Mockito.when(chunk.getMergeSortTVListIterator()).thenReturn(timeValuePairIterator);
+ MemPointIterator timeValuePairIterator =
+ MemPointIteratorFactory.create(TSDataType.TEXT, textTvLists);
+ timeValuePairIterator.nextBatch();
+ Mockito.when(chunk.getMemPointIterator()).thenReturn(timeValuePairIterator);
ChunkMetadata chunkMetadata = Mockito.mock(ChunkMetadata.class);
MemChunkLoader memChunkLoader = new MemChunkLoader(new QueryContext(), chunk);
@@ -487,9 +483,7 @@ public class MemChunkLoaderTest {
Statistics<? extends Serializable> pageStatistics = Mockito.mock(BinaryStatistics.class);
List<Statistics<? extends Serializable>> pageStats = Collections.singletonList(pageStatistics);
- List<int[]> pageOffsets = Arrays.asList(new int[] {0, 0}, new int[] {2, 1});
Mockito.when(chunk.getPageStatisticsList()).thenReturn(pageStats);
- Mockito.when(chunk.getPageOffsetsList()).thenReturn(pageOffsets);
ChunkMetadata chunkMetadata1 = Mockito.mock(ChunkMetadata.class);
Mockito.when(chunk.getChunkMetaData()).thenReturn(chunkMetadata1);
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemPageReaderTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemPageReaderTest.java
index 695c0b2f195..689c0671bc3 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemPageReaderTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemPageReaderTest.java
@@ -65,9 +65,7 @@ public class MemPageReaderTest {
Supplier<TsBlock> tsBlockSupplier = () -> tsBlock;
return new MemPageReader(
tsBlockSupplier,
- null,
- null,
- null,
+ 0,
chunkMetadata.getDataType(),
chunkMetadata.getMeasurementUid(),
null,