You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2022/05/02 13:50:04 UTC
[iotdb] 01/02: non-aligned
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch memtsblock
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 33acca88015eb9c4bba3010e0cdee38f7fdc45a6
Author: HTHou <hh...@outlook.com>
AuthorDate: Mon May 2 21:44:01 2022 +0800
non-aligned
---
.../db/engine/querycontext/ReadOnlyMemChunk.java | 59 +--------
.../db/metadata/utils/ResourceByPathUtils.java | 1 -
.../db/utils/datastructure/AlignedTVList.java | 15 ++-
.../iotdb/db/utils/datastructure/BinaryTVList.java | 33 +++--
.../db/utils/datastructure/BooleanTVList.java | 33 +++--
.../iotdb/db/utils/datastructure/DoubleTVList.java | 53 ++++----
.../iotdb/db/utils/datastructure/FloatTVList.java | 52 ++++----
.../iotdb/db/utils/datastructure/IntTVList.java | 33 +++--
.../iotdb/db/utils/datastructure/LongTVList.java | 33 +++--
.../iotdb/db/utils/datastructure/TVList.java | 144 ++++++++-------------
.../db/engine/memtable/PrimitiveMemTableTest.java | 18 ---
11 files changed, 207 insertions(+), 267 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/querycontext/ReadOnlyMemChunk.java b/server/src/main/java/org/apache/iotdb/db/engine/querycontext/ReadOnlyMemChunk.java
index 40ce93b523..512b4565ec 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/querycontext/ReadOnlyMemChunk.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/querycontext/ReadOnlyMemChunk.java
@@ -78,7 +78,6 @@ public class ReadOnlyMemChunk {
TSEncoding encoding,
TVList tvList,
Map<String, String> props,
- int size,
List<TimeRange> deletionList)
throws IOException, QueryProcessException {
this.measurementUid = measurementUid;
@@ -102,56 +101,11 @@ public class ReadOnlyMemChunk {
}
this.chunkData = tvList;
- this.chunkDataSize = size;
+ this.chunkDataSize = tvList.rowCount();
this.deletionList = deletionList;
-
- if (IoTDBDescriptor.getInstance().getConfig().isMppMode()) {
- this.tsblock = tvList.getTsBlock(floatPrecision, encoding, chunkDataSize, deletionList);
- initChunkMetaFromTsBlock();
- this.chunkPointReader = tsblock.getTsBlockSingleColumnIterator();
- } else {
- this.chunkPointReader =
- tvList.getIterator(floatPrecision, encoding, chunkDataSize, deletionList);
- initChunkMeta();
- }
- }
-
- private void initChunkMeta() throws IOException, QueryProcessException {
- Statistics statsByType = Statistics.getStatsByType(dataType);
- IChunkMetadata metaData = new ChunkMetadata(measurementUid, dataType, 0, statsByType);
- if (!isEmpty()) {
- IPointReader iterator =
- chunkData.getIterator(floatPrecision, encoding, chunkDataSize, deletionList);
- while (iterator.hasNextTimeValuePair()) {
- TimeValuePair timeValuePair = iterator.nextTimeValuePair();
- switch (dataType) {
- case BOOLEAN:
- statsByType.update(timeValuePair.getTimestamp(), timeValuePair.getValue().getBoolean());
- break;
- case TEXT:
- statsByType.update(timeValuePair.getTimestamp(), timeValuePair.getValue().getBinary());
- break;
- case FLOAT:
- statsByType.update(timeValuePair.getTimestamp(), timeValuePair.getValue().getFloat());
- break;
- case INT32:
- statsByType.update(timeValuePair.getTimestamp(), timeValuePair.getValue().getInt());
- break;
- case INT64:
- statsByType.update(timeValuePair.getTimestamp(), timeValuePair.getValue().getLong());
- break;
- case DOUBLE:
- statsByType.update(timeValuePair.getTimestamp(), timeValuePair.getValue().getDouble());
- break;
- default:
- throw new QueryProcessException("Unsupported data type:" + dataType);
- }
- }
- }
- statsByType.setEmpty(isEmpty());
- metaData.setChunkLoader(new MemChunkLoader(this));
- metaData.setVersion(Long.MAX_VALUE);
- cachedMetaData = metaData;
+ this.tsblock = tvList.getTsBlock(floatPrecision, encoding, deletionList);
+ initChunkMetaFromTsBlock();
+ this.chunkPointReader = tsblock.getTsBlockSingleColumnIterator();
}
private void initChunkMetaFromTsBlock() throws IOException, QueryProcessException {
@@ -159,7 +113,9 @@ public class ReadOnlyMemChunk {
IChunkMetadata metaData = new ChunkMetadata(measurementUid, dataType, 0, statsByType);
if (!isEmpty()) {
IPointReader iterator =
- chunkData.getTsBlock(floatPrecision, encoding, chunkDataSize, deletionList).getTsBlockSingleColumnIterator();
+ chunkData
+ .getTsBlock(floatPrecision, encoding, deletionList)
+ .getTsBlockSingleColumnIterator();
while (iterator.hasNextTimeValuePair()) {
TimeValuePair timeValuePair = iterator.nextTimeValuePair();
switch (dataType) {
@@ -209,7 +165,6 @@ public class ReadOnlyMemChunk {
}
public IPointReader getPointReader() {
- chunkPointReader = chunkData.getIterator(floatPrecision, encoding, chunkDataSize, deletionList);
return chunkPointReader;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/utils/ResourceByPathUtils.java b/server/src/main/java/org/apache/iotdb/db/metadata/utils/ResourceByPathUtils.java
index c509b7d25e..9e4422ea53 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/utils/ResourceByPathUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/utils/ResourceByPathUtils.java
@@ -550,7 +550,6 @@ class MeasurementResourceByPathUtils extends ResourceByPathUtils {
partialPath.getMeasurementSchema().getEncodingType(),
chunkCopy,
partialPath.getMeasurementSchema().getProps(),
- curSize,
deletionList);
}
/**
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
index 4819d1cd89..03c846c59f 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
@@ -28,7 +28,7 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.read.TimeValuePair;
import org.apache.iotdb.tsfile.read.common.TimeRange;
-import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
import org.apache.iotdb.tsfile.read.reader.IPointReader;
import org.apache.iotdb.tsfile.utils.Binary;
import org.apache.iotdb.tsfile.utils.BitMap;
@@ -917,10 +917,15 @@ public class AlignedTVList extends TVList {
}
@Override
- public TsBlock getTsBlock(int floatPrecision, TSEncoding encoding, int size,
- List<TimeRange> deletionList) {
- return null;
- }
+ protected void writeValuesIntoTsBlock(
+ ColumnBuilder valueBuilder, int floatPrecision, TSEncoding encoding) {}
+
+ @Override
+ protected void writeUnDeletedValuesIntoTsBlock(
+ ColumnBuilder valueBuilder,
+ int floatPrecision,
+ TSEncoding encoding,
+ List<TimeRange> deletionList) {}
public IPointReader getAlignedIterator(
int floatPrecision,
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java
index fc15313619..fa78d82255 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java
@@ -18,7 +18,6 @@
*/
package org.apache.iotdb.db.utils.datastructure;
-import java.util.Collections;
import org.apache.iotdb.db.rescon.PrimitiveArrayManager;
import org.apache.iotdb.db.wal.buffer.IWALByteBufferView;
import org.apache.iotdb.db.wal.utils.WALWriteUtils;
@@ -26,10 +25,7 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.read.TimeValuePair;
import org.apache.iotdb.tsfile.read.common.TimeRange;
-import org.apache.iotdb.tsfile.read.common.block.TsBlock;
-import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
-import org.apache.iotdb.tsfile.read.common.block.column.TimeColumnBuilder;
import org.apache.iotdb.tsfile.utils.Binary;
import org.apache.iotdb.tsfile.utils.BitMap;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
@@ -203,19 +199,28 @@ public class BinaryTVList extends TVList {
}
@Override
- public TsBlock getTsBlock(int floatPrecision, TSEncoding encoding, int size,
- List<TimeRange> deletionList) {
- TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(this.getDataType()));
- TimeColumnBuilder timeBuilder = builder.getTimeColumnBuilder();
- ColumnBuilder valueBuilder = builder.getColumnBuilder(0);
+ protected void writeValuesIntoTsBlock(
+ ColumnBuilder valueBuilder, int floatPrecision, TSEncoding encoding) {
for (int i = 0; i < timestamps.size() - 1; i++) {
- timeBuilder.writeLongs(timestamps.get(i), ARRAY_SIZE);
valueBuilder.writeBinaries(values.get(i), ARRAY_SIZE);
}
- timeBuilder.writeLongs(timestamps.get(timestamps.size() - 1), size % ARRAY_SIZE == 0 ? ARRAY_SIZE : size % ARRAY_SIZE);
- valueBuilder.writeBinaries(values.get(values.size() - 1), size % ARRAY_SIZE == 0 ? ARRAY_SIZE : size % ARRAY_SIZE);
- builder.declarePositions(size);
- return builder.build();
+ valueBuilder.writeBinaries(
+ values.get(values.size() - 1),
+ rowCount % ARRAY_SIZE == 0 ? ARRAY_SIZE : rowCount % ARRAY_SIZE);
+ }
+
+ @Override
+ protected void writeUnDeletedValuesIntoTsBlock(
+ ColumnBuilder valueBuilder,
+ int floatPrecision,
+ TSEncoding encoding,
+ List<TimeRange> deletionList) {
+ Integer deleteCursor = 0;
+ for (int i = 0; i < rowCount; i++) {
+ if (!isPointDeleted(getTime(i), deletionList, deleteCursor)) {
+ valueBuilder.writeBoolean(getBoolean(i));
+ }
+ }
}
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/BooleanTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/BooleanTVList.java
index 93df9b173f..c2a648557a 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/BooleanTVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/BooleanTVList.java
@@ -18,7 +18,6 @@
*/
package org.apache.iotdb.db.utils.datastructure;
-import java.util.Collections;
import org.apache.iotdb.db.rescon.PrimitiveArrayManager;
import org.apache.iotdb.db.wal.buffer.IWALByteBufferView;
import org.apache.iotdb.db.wal.utils.WALWriteUtils;
@@ -26,10 +25,7 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.read.TimeValuePair;
import org.apache.iotdb.tsfile.read.common.TimeRange;
-import org.apache.iotdb.tsfile.read.common.block.TsBlock;
-import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
-import org.apache.iotdb.tsfile.read.common.block.column.TimeColumnBuilder;
import org.apache.iotdb.tsfile.utils.BitMap;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
import org.apache.iotdb.tsfile.utils.TsPrimitiveType;
@@ -203,19 +199,28 @@ public class BooleanTVList extends TVList {
}
@Override
- public TsBlock getTsBlock(int floatPrecision, TSEncoding encoding, int size,
- List<TimeRange> deletionList) {
- TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(this.getDataType()));
- TimeColumnBuilder timeBuilder = builder.getTimeColumnBuilder();
- ColumnBuilder valueBuilder = builder.getColumnBuilder(0);
+ protected void writeValuesIntoTsBlock(
+ ColumnBuilder valueBuilder, int floatPrecision, TSEncoding encoding) {
for (int i = 0; i < timestamps.size() - 1; i++) {
- timeBuilder.writeLongs(timestamps.get(i), ARRAY_SIZE);
valueBuilder.writeBooleans(values.get(i), ARRAY_SIZE);
}
- timeBuilder.writeLongs(timestamps.get(timestamps.size() - 1), size % ARRAY_SIZE == 0 ? ARRAY_SIZE : size % ARRAY_SIZE);
- valueBuilder.writeBooleans(values.get(values.size() - 1), size % ARRAY_SIZE == 0 ? ARRAY_SIZE : size % ARRAY_SIZE);
- builder.declarePositions(size);
- return builder.build();
+ valueBuilder.writeBooleans(
+ values.get(values.size() - 1),
+ rowCount % ARRAY_SIZE == 0 ? ARRAY_SIZE : rowCount % ARRAY_SIZE);
+ }
+
+ @Override
+ protected void writeUnDeletedValuesIntoTsBlock(
+ ColumnBuilder valueBuilder,
+ int floatPrecision,
+ TSEncoding encoding,
+ List<TimeRange> deletionList) {
+ Integer deleteCursor = 0;
+ for (int i = 0; i < rowCount; i++) {
+ if (!isPointDeleted(getTime(i), deletionList, deleteCursor)) {
+ valueBuilder.writeBoolean(getBoolean(i));
+ }
+ }
}
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/DoubleTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/DoubleTVList.java
index 9abb886a70..39a64e1265 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/DoubleTVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/DoubleTVList.java
@@ -18,7 +18,6 @@
*/
package org.apache.iotdb.db.utils.datastructure;
-import java.util.Collections;
import org.apache.iotdb.db.rescon.PrimitiveArrayManager;
import org.apache.iotdb.db.utils.MathUtils;
import org.apache.iotdb.db.wal.buffer.IWALByteBufferView;
@@ -27,10 +26,7 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.read.TimeValuePair;
import org.apache.iotdb.tsfile.read.common.TimeRange;
-import org.apache.iotdb.tsfile.read.common.block.TsBlock;
-import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
-import org.apache.iotdb.tsfile.read.common.block.column.TimeColumnBuilder;
import org.apache.iotdb.tsfile.utils.BitMap;
import org.apache.iotdb.tsfile.utils.TsPrimitiveType;
@@ -206,33 +202,46 @@ public class DoubleTVList extends TVList {
}
@Override
- public TsBlock getTsBlock(int floatPrecision, TSEncoding encoding, int size,
- List<TimeRange> deletionList) {
- TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(this.getDataType()));
- TimeColumnBuilder timeBuilder = builder.getTimeColumnBuilder();
- ColumnBuilder valueBuilder = builder.getColumnBuilder(0);
+ protected void writeValuesIntoTsBlock(
+ ColumnBuilder valueBuilder, int floatPrecision, TSEncoding encoding) {
for (int i = 0; i < timestamps.size() - 1; i++) {
- timeBuilder.writeLongs(timestamps.get(i), ARRAY_SIZE);
- getValuesWithGivenPrecision(values.get(i), floatPrecision, encoding);
+ for (int j = 0; j < values.get(i).length; j++) {
+ values.get(i)[j] = roundValueWithGivenPrecision(values.get(i)[j], floatPrecision, encoding);
+ }
valueBuilder.writeDoubles(values.get(i), ARRAY_SIZE);
}
- timeBuilder.writeLongs(timestamps.get(timestamps.size() - 1), size % ARRAY_SIZE == 0 ? ARRAY_SIZE : size % ARRAY_SIZE);
- getValuesWithGivenPrecision(values.get(values.size() - 1), floatPrecision, encoding);
- valueBuilder.writeDoubles(values.get(values.size() - 1), size % ARRAY_SIZE == 0 ? ARRAY_SIZE : size % ARRAY_SIZE);
- builder.declarePositions(size);
- return builder.build();
+ for (int j = 0; j < values.get(values.size() - 1).length; j++) {
+ values.get(values.size() - 1)[j] =
+ roundValueWithGivenPrecision(values.get(values.size() - 1)[j], floatPrecision, encoding);
+ }
+ valueBuilder.writeDoubles(
+ values.get(values.size() - 1),
+ rowCount % ARRAY_SIZE == 0 ? ARRAY_SIZE : rowCount % ARRAY_SIZE);
}
- private void getValuesWithGivenPrecision(double[] values, int floatPrecision, TSEncoding encoding) {
- if (encoding == TSEncoding.RLE || encoding == TSEncoding.TS_2DIFF) {
- for (int i = 0; i < values.length; i++) {
- if (!Double.isNaN(values[i])) {
- values[i] = MathUtils.roundWithGivenPrecision(values[i], floatPrecision);
- }
+ @Override
+ protected void writeUnDeletedValuesIntoTsBlock(
+ ColumnBuilder valueBuilder,
+ int floatPrecision,
+ TSEncoding encoding,
+ List<TimeRange> deletionList) {
+ Integer deleteCursor = 0;
+ for (int i = 0; i < rowCount; i++) {
+ if (!isPointDeleted(getTime(i), deletionList, deleteCursor)) {
+ valueBuilder.writeDouble(
+ roundValueWithGivenPrecision(getDouble(i), floatPrecision, encoding));
}
}
}
+ private double roundValueWithGivenPrecision(
+ double value, int floatPrecision, TSEncoding encoding) {
+ if (!Double.isNaN(value) && (encoding == TSEncoding.RLE || encoding == TSEncoding.TS_2DIFF)) {
+ return MathUtils.roundWithGivenPrecision(value, floatPrecision);
+ }
+ return value;
+ }
+
@Override
protected void releaseLastValueArray() {
PrimitiveArrayManager.release(values.remove(values.size() - 1));
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/FloatTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/FloatTVList.java
index 31eb193699..f010489059 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/FloatTVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/FloatTVList.java
@@ -18,7 +18,6 @@
*/
package org.apache.iotdb.db.utils.datastructure;
-import java.util.Collections;
import org.apache.iotdb.db.rescon.PrimitiveArrayManager;
import org.apache.iotdb.db.utils.MathUtils;
import org.apache.iotdb.db.wal.buffer.IWALByteBufferView;
@@ -27,10 +26,7 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.read.TimeValuePair;
import org.apache.iotdb.tsfile.read.common.TimeRange;
-import org.apache.iotdb.tsfile.read.common.block.TsBlock;
-import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
-import org.apache.iotdb.tsfile.read.common.block.column.TimeColumnBuilder;
import org.apache.iotdb.tsfile.utils.BitMap;
import org.apache.iotdb.tsfile.utils.TsPrimitiveType;
@@ -206,33 +202,45 @@ public class FloatTVList extends TVList {
}
@Override
- public TsBlock getTsBlock(int floatPrecision, TSEncoding encoding, int size,
- List<TimeRange> deletionList) {
- TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(this.getDataType()));
- TimeColumnBuilder timeBuilder = builder.getTimeColumnBuilder();
- ColumnBuilder valueBuilder = builder.getColumnBuilder(0);
+ protected void writeValuesIntoTsBlock(
+ ColumnBuilder valueBuilder, int floatPrecision, TSEncoding encoding) {
for (int i = 0; i < timestamps.size() - 1; i++) {
- timeBuilder.writeLongs(timestamps.get(i), ARRAY_SIZE);
- getValuesWithGivenPrecision(values.get(i), floatPrecision, encoding);
+ for (int j = 0; j < values.get(i).length; j++) {
+ values.get(i)[j] = roundValueWithGivenPrecision(values.get(i)[j], floatPrecision, encoding);
+ }
valueBuilder.writeFloats(values.get(i), ARRAY_SIZE);
}
- timeBuilder.writeLongs(timestamps.get(timestamps.size() - 1), size % ARRAY_SIZE == 0 ? ARRAY_SIZE : size % ARRAY_SIZE);
- getValuesWithGivenPrecision(values.get(values.size() - 1), floatPrecision, encoding);
- valueBuilder.writeFloats(values.get(values.size() - 1), size % ARRAY_SIZE == 0 ? ARRAY_SIZE : size % ARRAY_SIZE);
- builder.declarePositions(size);
- return builder.build();
+ for (int j = 0; j < values.get(values.size() - 1).length; j++) {
+ values.get(values.size() - 1)[j] =
+ roundValueWithGivenPrecision(values.get(values.size() - 1)[j], floatPrecision, encoding);
+ }
+ valueBuilder.writeFloats(
+ values.get(values.size() - 1),
+ rowCount % ARRAY_SIZE == 0 ? ARRAY_SIZE : rowCount % ARRAY_SIZE);
}
- private void getValuesWithGivenPrecision(float[] values, int floatPrecision, TSEncoding encoding) {
- if (encoding == TSEncoding.RLE || encoding == TSEncoding.TS_2DIFF) {
- for (int i = 0; i < values.length; i++) {
- if (!Float.isNaN(values[i])) {
- values[i] = MathUtils.roundWithGivenPrecision(values[i], floatPrecision);
- }
+ @Override
+ protected void writeUnDeletedValuesIntoTsBlock(
+ ColumnBuilder valueBuilder,
+ int floatPrecision,
+ TSEncoding encoding,
+ List<TimeRange> deletionList) {
+ Integer deleteCursor = 0;
+ for (int i = 0; i < rowCount; i++) {
+ if (!isPointDeleted(getTime(i), deletionList, deleteCursor)) {
+ valueBuilder.writeFloat(
+ roundValueWithGivenPrecision(getFloat(i), floatPrecision, encoding));
}
}
}
+ private float roundValueWithGivenPrecision(float value, int floatPrecision, TSEncoding encoding) {
+ if (!Float.isNaN(value) && (encoding == TSEncoding.RLE || encoding == TSEncoding.TS_2DIFF)) {
+ return MathUtils.roundWithGivenPrecision(value, floatPrecision);
+ }
+ return value;
+ }
+
@Override
protected void releaseLastValueArray() {
PrimitiveArrayManager.release(values.remove(values.size() - 1));
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/IntTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/IntTVList.java
index e785c3ab91..e07d0c3876 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/IntTVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/IntTVList.java
@@ -18,7 +18,6 @@
*/
package org.apache.iotdb.db.utils.datastructure;
-import java.util.Collections;
import org.apache.iotdb.db.rescon.PrimitiveArrayManager;
import org.apache.iotdb.db.wal.buffer.IWALByteBufferView;
import org.apache.iotdb.db.wal.utils.WALWriteUtils;
@@ -26,10 +25,7 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.read.TimeValuePair;
import org.apache.iotdb.tsfile.read.common.TimeRange;
-import org.apache.iotdb.tsfile.read.common.block.TsBlock;
-import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
-import org.apache.iotdb.tsfile.read.common.block.column.TimeColumnBuilder;
import org.apache.iotdb.tsfile.utils.BitMap;
import org.apache.iotdb.tsfile.utils.TsPrimitiveType;
@@ -201,19 +197,28 @@ public class IntTVList extends TVList {
}
@Override
- public TsBlock getTsBlock(int floatPrecision, TSEncoding encoding, int size,
- List<TimeRange> deletionList) {
- TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(this.getDataType()));
- TimeColumnBuilder timeBuilder = builder.getTimeColumnBuilder();
- ColumnBuilder valueBuilder = builder.getColumnBuilder(0);
+ protected void writeValuesIntoTsBlock(
+ ColumnBuilder valueBuilder, int floatPrecision, TSEncoding encoding) {
for (int i = 0; i < timestamps.size() - 1; i++) {
- timeBuilder.writeLongs(timestamps.get(i), ARRAY_SIZE);
valueBuilder.writeInts(values.get(i), ARRAY_SIZE);
}
- timeBuilder.writeLongs(timestamps.get(timestamps.size() - 1), size % ARRAY_SIZE == 0 ? ARRAY_SIZE : size % ARRAY_SIZE);
- valueBuilder.writeInts(values.get(values.size() - 1), size % ARRAY_SIZE == 0 ? ARRAY_SIZE : size % ARRAY_SIZE);
- builder.declarePositions(size);
- return builder.build();
+ valueBuilder.writeInts(
+ values.get(values.size() - 1),
+ rowCount % ARRAY_SIZE == 0 ? ARRAY_SIZE : rowCount % ARRAY_SIZE);
+ }
+
+ @Override
+ protected void writeUnDeletedValuesIntoTsBlock(
+ ColumnBuilder valueBuilder,
+ int floatPrecision,
+ TSEncoding encoding,
+ List<TimeRange> deletionList) {
+ Integer deleteCursor = 0;
+ for (int i = 0; i < rowCount; i++) {
+ if (!isPointDeleted(getTime(i), deletionList, deleteCursor)) {
+ valueBuilder.writeInt(getInt(i));
+ }
+ }
}
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/LongTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/LongTVList.java
index 9d7f6d4af5..65d706a725 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/LongTVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/LongTVList.java
@@ -18,7 +18,6 @@
*/
package org.apache.iotdb.db.utils.datastructure;
-import java.util.Collections;
import org.apache.iotdb.db.rescon.PrimitiveArrayManager;
import org.apache.iotdb.db.wal.buffer.IWALByteBufferView;
import org.apache.iotdb.db.wal.utils.WALWriteUtils;
@@ -26,10 +25,7 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.read.TimeValuePair;
import org.apache.iotdb.tsfile.read.common.TimeRange;
-import org.apache.iotdb.tsfile.read.common.block.TsBlock;
-import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
-import org.apache.iotdb.tsfile.read.common.block.column.TimeColumnBuilder;
import org.apache.iotdb.tsfile.utils.BitMap;
import org.apache.iotdb.tsfile.utils.TsPrimitiveType;
@@ -201,19 +197,28 @@ public class LongTVList extends TVList {
}
@Override
- public TsBlock getTsBlock(int floatPrecision, TSEncoding encoding, int size,
- List<TimeRange> deletionList) {
- TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(this.getDataType()));
- TimeColumnBuilder timeBuilder = builder.getTimeColumnBuilder();
- ColumnBuilder valueBuilder = builder.getColumnBuilder(0);
+ protected void writeValuesIntoTsBlock(
+ ColumnBuilder valueBuilder, int floatPrecision, TSEncoding encoding) {
for (int i = 0; i < timestamps.size() - 1; i++) {
- timeBuilder.writeLongs(timestamps.get(i), ARRAY_SIZE);
valueBuilder.writeLongs(values.get(i), ARRAY_SIZE);
}
- timeBuilder.writeLongs(timestamps.get(timestamps.size() - 1), size % ARRAY_SIZE == 0 ? ARRAY_SIZE : size % ARRAY_SIZE);
- valueBuilder.writeLongs(values.get(values.size() - 1), size % ARRAY_SIZE == 0 ? ARRAY_SIZE : size % ARRAY_SIZE);
- builder.declarePositions(size);
- return builder.build();
+ valueBuilder.writeLongs(
+ values.get(values.size() - 1),
+ rowCount % ARRAY_SIZE == 0 ? ARRAY_SIZE : rowCount % ARRAY_SIZE);
+ }
+
+ @Override
+ protected void writeUnDeletedValuesIntoTsBlock(
+ ColumnBuilder valueBuilder,
+ int floatPrecision,
+ TSEncoding encoding,
+ List<TimeRange> deletionList) {
+ Integer deleteCursor = 0;
+ for (int i = 0; i < rowCount; i++) {
+ if (!isPointDeleted(getTime(i), deletionList, deleteCursor)) {
+ valueBuilder.writeLong(getLong(i));
+ }
+ }
}
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
index fb9f790bff..e9f13ef4fe 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
@@ -19,8 +19,6 @@
package org.apache.iotdb.db.utils.datastructure;
-import java.util.Collections;
-import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.db.rescon.PrimitiveArrayManager;
import org.apache.iotdb.db.wal.buffer.WALEntryValue;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -31,7 +29,6 @@ import org.apache.iotdb.tsfile.read.common.block.TsBlock;
import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
import org.apache.iotdb.tsfile.read.common.block.column.TimeColumnBuilder;
-import org.apache.iotdb.tsfile.read.reader.IPointReader;
import org.apache.iotdb.tsfile.utils.Binary;
import org.apache.iotdb.tsfile.utils.BitMap;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
@@ -39,6 +36,7 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
import java.io.DataInputStream;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
@@ -526,104 +524,68 @@ public abstract class TVList implements WALEntryValue {
throw new UnsupportedOperationException(ERR_DATATYPE_NOT_CONSISTENT);
}
- @TestOnly
- public IPointReader getIterator() {
- return new Ite();
- }
-
- public IPointReader getIterator(
- int floatPrecision, TSEncoding encoding, int size, List<TimeRange> deletionList) {
- return new Ite(floatPrecision, encoding, size, deletionList);
- }
-
- protected class Ite implements IPointReader {
-
- protected TimeValuePair cachedTimeValuePair;
- protected boolean hasCachedPair;
- protected int cur;
- protected Integer floatPrecision;
- private TSEncoding encoding;
- private int deleteCursor = 0;
- /**
- * because TV list may be share with different query, each iterator has to record it's own size
- */
- protected int iteSize = 0;
- /** this field is effective only in the Tvlist in a RealOnlyMemChunk. */
- private List<TimeRange> deletionList;
-
- public Ite() {
- this.iteSize = TVList.this.rowCount;
+ public TsBlock getTsBlock(int floatPrecision, TSEncoding encoding, List<TimeRange> deletionList) {
+ if (deletionList == null) {
+ return this.getTsBlockWithoutDeletionList(floatPrecision, encoding);
}
-
- public Ite(int floatPrecision, TSEncoding encoding, int size, List<TimeRange> deletionList) {
- this.floatPrecision = floatPrecision;
- this.encoding = encoding;
- this.iteSize = size;
- this.deletionList = deletionList;
- }
-
- @Override
- public boolean hasNextTimeValuePair() {
- if (hasCachedPair) {
- return true;
- }
-
- while (cur < iteSize) {
- long time = getTime(cur);
- if (isPointDeleted(time) || (cur + 1 < rowCount() && (time == getTime(cur + 1)))) {
- cur++;
- continue;
- }
- TimeValuePair tvPair;
- tvPair = getTimeValuePair(cur, time, floatPrecision, encoding);
- cur++;
- if (tvPair.getValue() != null) {
- cachedTimeValuePair = tvPair;
- hasCachedPair = true;
- return true;
- }
+ Integer deleteCursor = 0;
+ TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(this.getDataType()));
+ // Time column
+ TimeColumnBuilder timeBuilder = builder.getTimeColumnBuilder();
+ for (int i = 0; i < rowCount; i++) {
+ if (!isPointDeleted(getTime(i), deletionList, deleteCursor)) {
+ timeBuilder.writeLong(this.getTime(i));
}
-
- return false;
}
- protected boolean isPointDeleted(long timestamp) {
- while (deletionList != null && deleteCursor < deletionList.size()) {
- if (deletionList.get(deleteCursor).contains(timestamp)) {
- return true;
- } else if (deletionList.get(deleteCursor).getMax() < timestamp) {
- deleteCursor++;
- } else {
- return false;
- }
- }
- return false;
- }
+ // value column
+ ColumnBuilder valueBuilder = builder.getColumnBuilder(0);
+ writeUnDeletedValuesIntoTsBlock(valueBuilder, floatPrecision, encoding, deletionList);
+ builder.declarePositions(rowCount);
+ return builder.build();
+ }
- @Override
- public TimeValuePair nextTimeValuePair() throws IOException {
- if (hasCachedPair || hasNextTimeValuePair()) {
- hasCachedPair = false;
- return cachedTimeValuePair;
+ private TsBlock getTsBlockWithoutDeletionList(int floatPrecision, TSEncoding encoding) {
+ TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(this.getDataType()));
+ // Time column
+ TimeColumnBuilder timeBuilder = builder.getTimeColumnBuilder();
+ for (int i = 0; i < timestamps.size() - 1; i++) {
+ timeBuilder.writeLongs(timestamps.get(i), ARRAY_SIZE);
+ }
+ timeBuilder.writeLongs(
+ timestamps.get(timestamps.size() - 1),
+ rowCount % ARRAY_SIZE == 0 ? ARRAY_SIZE : rowCount % ARRAY_SIZE);
+
+ // value column
+ ColumnBuilder valueBuilder = builder.getColumnBuilder(0);
+ writeValuesIntoTsBlock(valueBuilder, floatPrecision, encoding);
+ builder.declarePositions(rowCount);
+ return builder.build();
+ }
+
+ protected abstract void writeValuesIntoTsBlock(
+ ColumnBuilder valueBuilder, int floatPrecision, TSEncoding encoding);
+
+ protected abstract void writeUnDeletedValuesIntoTsBlock(
+ ColumnBuilder valueBuilder,
+ int floatPrecision,
+ TSEncoding encoding,
+ List<TimeRange> deletionList);
+
+ protected boolean isPointDeleted(
+ long timestamp, List<TimeRange> deletionList, Integer deleteCursor) {
+ while (deleteCursor < deletionList.size()) {
+ if (deletionList.get(deleteCursor).contains(timestamp)) {
+ return true;
+ } else if (deletionList.get(deleteCursor).getMax() < timestamp) {
+ deleteCursor++;
} else {
- throw new IOException("no next time value pair");
+ return false;
}
}
-
- @Override
- public TimeValuePair currentTimeValuePair() {
- return cachedTimeValuePair;
- }
-
- @Override
- public void close() throws IOException {
- // Do nothing because of this is an in memory object
- }
+ return false;
}
- public abstract TsBlock getTsBlock(
- int floatPrecision, TSEncoding encoding, int size, List<TimeRange> deletionList);
-
public abstract TSDataType getDataType();
public static TVList deserialize(DataInputStream stream) throws IOException {
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTableTest.java b/server/src/test/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTableTest.java
index cc64db7d56..b39381cb9f 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTableTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTableTest.java
@@ -69,24 +69,6 @@ public class PrimitiveMemTableTest {
delta = Math.pow(0.1, TSFileDescriptor.getInstance().getConfig().getFloatPrecision());
}
- @Test
- public void memSeriesSortIteratorTest() throws IOException {
- TSDataType dataType = TSDataType.INT32;
- WritableMemChunk series =
- new WritableMemChunk(new MeasurementSchema("s1", dataType, TSEncoding.PLAIN));
- int count = 1000;
- for (int i = 0; i < count; i++) {
- series.write(i, i);
- }
- IPointReader it = series.getSortedTvListForQuery().getIterator();
- int i = 0;
- while (it.hasNextTimeValuePair()) {
- Assert.assertEquals(i, it.nextTimeValuePair().getTimestamp());
- i++;
- }
- Assert.assertEquals(count, i);
- }
-
@Test
public void memSeriesToStringTest() throws IOException {
TSDataType dataType = TSDataType.INT32;