You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2019/11/30 10:31:18 UTC
[incubator-iotdb] branch master updated: [IOTDB-325] Refactor
Statistics (#587)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new af831ab [IOTDB-325] Refactor Statistics (#587)
af831ab is described below
commit af831ab803bf54193d5893e49ab5cc2f82628481
Author: Jialin Qiao <qj...@mails.tsinghua.edu.cn>
AuthorDate: Sat Nov 30 18:31:11 2019 +0800
[IOTDB-325] Refactor Statistics (#587)
* refactor statistics
---
.../apache/iotdb/hive/TSFHiveInputFormatTest.java | 8 +-
.../db/query/aggregation/impl/AvgAggrFunc.java | 2 +-
.../db/query/aggregation/impl/CountAggrFunc.java | 2 +-
.../db/query/aggregation/impl/FirstAggrFunc.java | 2 +-
.../db/query/aggregation/impl/LastAggrFunc.java | 4 +-
.../db/query/aggregation/impl/MaxTimeAggrFunc.java | 2 +-
.../query/aggregation/impl/MaxValueAggrFunc.java | 2 +-
.../db/query/aggregation/impl/MinTimeAggrFunc.java | 2 +-
.../query/aggregation/impl/MinValueAggrFunc.java | 2 +-
.../groupby/GroupByWithoutValueFilterDataSet.java | 8 +-
.../db/query/executor/AggregateEngineExecutor.java | 8 +-
.../resourceRelated/UnseqResourceMergeReader.java | 19 +-
.../apache/iotdb/db/tools/TsFileSketchTool.java | 65 +--
.../db/engine/memtable/MemTableFlushTaskTest.java | 2 +-
.../merge/MaxSeriesMergeFileSelectorTest.java | 1 -
.../engine/storagegroup/TsFileProcessorTest.java | 6 +-
tsfile/format-changelist.md | 9 +
.../iotdb/tsfile/file/header/ChunkHeader.java | 54 +-
.../iotdb/tsfile/file/header/PageHeader.java | 128 +----
.../iotdb/tsfile/file/header/package-info.java | 2 +-
.../tsfile/file/metadata/ChunkGroupMetaData.java | 81 +--
.../iotdb/tsfile/file/metadata/ChunkMetaData.java | 159 +-----
.../tsfile/file/metadata/TsDeviceMetadata.java | 83 +--
.../file/metadata/statistics/BinaryStatistics.java | 192 +++----
.../metadata/statistics/BooleanStatistics.java | 184 +++----
.../file/metadata/statistics/DoubleStatistics.java | 211 +++----
.../file/metadata/statistics/FloatStatistics.java | 190 ++++---
.../metadata/statistics/IntegerStatistics.java | 191 ++++---
.../file/metadata/statistics/LongStatistics.java | 201 +++----
.../file/metadata/statistics/NoStatistics.java | 181 ------
.../file/metadata/statistics/Statistics.java | 609 ++++++++-------------
.../statistics/StatisticsClassException.java | 5 +
.../iotdb/tsfile/read/TsFileSequenceReader.java | 68 +--
.../iotdb/tsfile/read/filter/DigestForFilter.java | 154 ------
.../iotdb/tsfile/read/filter/basic/Filter.java | 10 +-
.../tsfile/read/filter/operator/AndFilter.java | 6 +-
.../iotdb/tsfile/read/filter/operator/Eq.java | 13 +-
.../iotdb/tsfile/read/filter/operator/Gt.java | 11 +-
.../iotdb/tsfile/read/filter/operator/GtEq.java | 11 +-
.../iotdb/tsfile/read/filter/operator/Lt.java | 11 +-
.../iotdb/tsfile/read/filter/operator/LtEq.java | 11 +-
.../iotdb/tsfile/read/filter/operator/NotEq.java | 13 +-
.../tsfile/read/filter/operator/NotFilter.java | 6 +-
.../tsfile/read/filter/operator/OrFilter.java | 6 +-
.../query/executor/ExecutorWithTimeGenerator.java | 2 +-
.../tsfile/read/query/executor/TsFileExecutor.java | 2 +-
.../tsfile/read/reader/chunk/ChunkReader.java | 2 +-
.../read/reader/chunk/ChunkReaderByTimestamp.java | 2 +-
.../read/reader/chunk/ChunkReaderWithFilter.java | 10 +-
.../reader/chunk/ChunkReaderWithoutFilter.java | 2 +-
.../reader/series/FileSeriesReaderByTimestamp.java | 2 +-
.../reader/series/FileSeriesReaderWithFilter.java | 15 +-
.../tool/upgrade/TsfileUpgradeToolV0_8_0.java | 28 +-
.../iotdb/tsfile/utils/ReadWriteIOUtils.java | 14 +
.../iotdb/tsfile/write/chunk/ChunkWriterImpl.java | 70 +--
.../iotdb/tsfile/write/chunk/IChunkWriter.java | 10 -
.../apache/iotdb/tsfile/write/page/PageWriter.java | 64 +--
.../write/writer/RestorableTsFileIOWriter.java | 2 +-
.../iotdb/tsfile/write/writer/TsFileIOWriter.java | 61 +--
.../iotdb/tsfile/file/header/PageHeaderTest.java | 66 +--
.../file/metadata/ChunkGroupMetaDataTest.java | 52 --
.../tsfile/file/metadata/ChunkMetaDataTest.java | 114 ----
.../tsfile/file/metadata/TsDeviceMetadataTest.java | 64 ---
.../metadata/statistics/BooleanStatisticsTest.java | 21 +-
.../metadata/statistics/DoubleStatisticsTest.java | 30 +-
.../metadata/statistics/FloatStatisticsTest.java | 30 +-
.../metadata/statistics/IntegerStatisticsTest.java | 30 +-
.../metadata/statistics/LongStatisticsTest.java | 40 +-
.../metadata/statistics/StringStatisticsTest.java | 23 +-
.../tsfile/file/metadata/utils/TestHelper.java | 109 +---
.../iotdb/tsfile/file/metadata/utils/Utils.java | 28 +-
.../iotdb/tsfile/read/ReadInPartitionTest.java | 4 +-
.../iotdb/tsfile/read/ReadOnlyTsFileTest.java | 4 +-
.../org/apache/iotdb/tsfile/read/ReadTest.java | 16 +-
.../tsfile/read/TsFileSequenceReaderTest.java | 4 +-
.../tsfile/read/controller/ChunkLoaderTest.java | 3 +-
.../controller/IMetadataQuerierByFileImplTest.java | 4 +-
.../iotdb/tsfile/read/filter/DigestFilterTest.java | 116 ----
.../tsfile/read/filter/StatisticsFilterTest.java | 108 ++++
.../iotdb/tsfile/read/reader/ReaderTest.java | 2 +-
.../apache/iotdb/tsfile/utils/FileGenerator.java | 23 +-
.../iotdb/tsfile/utils/TsFileGeneratorForTest.java | 12 +-
.../iotdb/tsfile/write/TsFileIOWriterTest.java | 6 +-
.../write/writer/RestorableTsFileIOWriterTest.java | 2 +-
84 files changed, 1268 insertions(+), 2864 deletions(-)
diff --git a/hive-connector/src/test/java/org/apache/iotdb/hive/TSFHiveInputFormatTest.java b/hive-connector/src/test/java/org/apache/iotdb/hive/TSFHiveInputFormatTest.java
index 86ac429..efca538 100644
--- a/hive-connector/src/test/java/org/apache/iotdb/hive/TSFHiveInputFormatTest.java
+++ b/hive-connector/src/test/java/org/apache/iotdb/hive/TSFHiveInputFormatTest.java
@@ -52,7 +52,7 @@ public class TSFHiveInputFormatTest {
private String filePath = "test.tsfile";
@Before
- public void setUp() throws IOException {
+ public void setUp() {
TsFileTestHelper.writeTsFile(filePath);
inputFormat = new TSFHiveInputFormat();
job = new JobConf();
@@ -66,7 +66,7 @@ public class TSFHiveInputFormatTest {
measurementIds[i] = "sensor_" + (i + 1);
}
startOffset = 12L;
- endOffset = 3727528L;
+ endOffset = 3728888L;
length = endOffset - startOffset;
TSFInputSplit.ChunkGroupInfo chunkGroupInfo = new TSFInputSplit.ChunkGroupInfo(deviceId, measurementIds, startOffset, endOffset);
chunkGroupInfoList.add(chunkGroupInfo);
@@ -97,12 +97,12 @@ public class TSFHiveInputFormatTest {
assertEquals(1, inputSplits.length);
assertTrue(inputSplits[0] instanceof TSFInputSplit);
TSFInputSplit inputSplit = (TSFInputSplit) inputSplits[0];
- assertEquals(length, inputSplit.getLength());
- assertEquals(1, inputSplit.getChunkGroupInfoList().size());
TSFInputSplit.ChunkGroupInfo chunkGroupInfo = inputSplit.getChunkGroupInfoList().get(0);
assertEquals(deviceId, chunkGroupInfo.getDeviceId());
assertEquals(startOffset, chunkGroupInfo.getStartOffset());
assertEquals(endOffset, chunkGroupInfo.getEndOffset());
+ assertEquals(length, inputSplit.getLength());
+ assertEquals(1, inputSplit.getChunkGroupInfoList().size());
assertEquals(Arrays.stream(measurementIds).collect(toSet()), Arrays.stream(chunkGroupInfo.getMeasurementIds()).collect(toSet()));
} catch (IOException e) {
e.printStackTrace();
diff --git a/server/src/main/java/org/apache/iotdb/db/query/aggregation/impl/AvgAggrFunc.java b/server/src/main/java/org/apache/iotdb/db/query/aggregation/impl/AvgAggrFunc.java
index 7fc4812..7888021 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/aggregation/impl/AvgAggrFunc.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/aggregation/impl/AvgAggrFunc.java
@@ -59,7 +59,7 @@ public class AvgAggrFunc extends AggregateFunction {
@Override
public void calculateValueFromPageHeader(PageHeader pageHeader) {
- sum += pageHeader.getStatistics().getSum();
+ sum += pageHeader.getStatistics().getSumValue();
cnt += pageHeader.getNumOfValues();
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/aggregation/impl/CountAggrFunc.java b/server/src/main/java/org/apache/iotdb/db/query/aggregation/impl/CountAggrFunc.java
index edb1334..cab4e90 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/aggregation/impl/CountAggrFunc.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/aggregation/impl/CountAggrFunc.java
@@ -54,7 +54,7 @@ public class CountAggrFunc extends AggregateFunction {
public void calculateValueFromPageHeader(PageHeader pageHeader) {
if (logger.isDebugEnabled()) {
logger.debug("PageHeader>>>>>>>>>>>>num of rows:{}, minTimeStamp:{}, maxTimeStamp{}",
- pageHeader.getNumOfValues(), pageHeader.getMinTimestamp(), pageHeader.getMaxTimestamp());
+ pageHeader.getNumOfValues(), pageHeader.getStartTime(), pageHeader.getEndTime());
}
long preValue = resultData.getLongRet();
preValue += pageHeader.getNumOfValues();
diff --git a/server/src/main/java/org/apache/iotdb/db/query/aggregation/impl/FirstAggrFunc.java b/server/src/main/java/org/apache/iotdb/db/query/aggregation/impl/FirstAggrFunc.java
index 58b6069..3728d84 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/aggregation/impl/FirstAggrFunc.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/aggregation/impl/FirstAggrFunc.java
@@ -51,7 +51,7 @@ public class FirstAggrFunc extends AggregateFunction {
return;
}
- Object firstVal = pageHeader.getStatistics().getFirst();
+ Object firstVal = pageHeader.getStatistics().getFirstValue();
if (firstVal == null) {
throw new QueryProcessException("PageHeader contains no FIRST value");
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/aggregation/impl/LastAggrFunc.java b/server/src/main/java/org/apache/iotdb/db/query/aggregation/impl/LastAggrFunc.java
index 7a4687b..f98d5ad 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/aggregation/impl/LastAggrFunc.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/aggregation/impl/LastAggrFunc.java
@@ -50,8 +50,8 @@ public class LastAggrFunc extends AggregateFunction {
@Override
public void calculateValueFromPageHeader(PageHeader pageHeader) {
- Object lastVal = pageHeader.getStatistics().getLast();
- updateLastResult(pageHeader.getMaxTimestamp(), lastVal);
+ Object lastVal = pageHeader.getStatistics().getLastValue();
+ updateLastResult(pageHeader.getEndTime(), lastVal);
}
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/query/aggregation/impl/MaxTimeAggrFunc.java b/server/src/main/java/org/apache/iotdb/db/query/aggregation/impl/MaxTimeAggrFunc.java
index 0444443..5a566a7 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/aggregation/impl/MaxTimeAggrFunc.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/aggregation/impl/MaxTimeAggrFunc.java
@@ -47,7 +47,7 @@ public class MaxTimeAggrFunc extends AggregateFunction {
@Override
public void calculateValueFromPageHeader(PageHeader pageHeader) {
- long maxTimestamp = pageHeader.getMaxTimestamp();
+ long maxTimestamp = pageHeader.getEndTime();
updateMaxTimeResult(0, maxTimestamp);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/aggregation/impl/MaxValueAggrFunc.java b/server/src/main/java/org/apache/iotdb/db/query/aggregation/impl/MaxValueAggrFunc.java
index 7845304..c4d39a5 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/aggregation/impl/MaxValueAggrFunc.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/aggregation/impl/MaxValueAggrFunc.java
@@ -46,7 +46,7 @@ public class MaxValueAggrFunc extends AggregateFunction {
@Override
public void calculateValueFromPageHeader(PageHeader pageHeader) {
- Comparable<Object> maxVal = (Comparable<Object>) pageHeader.getStatistics().getMax();
+ Comparable<Object> maxVal = (Comparable<Object>) pageHeader.getStatistics().getMaxValue();
updateResult(maxVal);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/aggregation/impl/MinTimeAggrFunc.java b/server/src/main/java/org/apache/iotdb/db/query/aggregation/impl/MinTimeAggrFunc.java
index f9b64c8..b237bcb 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/aggregation/impl/MinTimeAggrFunc.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/aggregation/impl/MinTimeAggrFunc.java
@@ -49,7 +49,7 @@ public class MinTimeAggrFunc extends AggregateFunction {
if (resultData.isSetValue()) {
return;
}
- long time = pageHeader.getMinTimestamp();
+ long time = pageHeader.getStartTime();
resultData.putTimeAndValue(0, time);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/aggregation/impl/MinValueAggrFunc.java b/server/src/main/java/org/apache/iotdb/db/query/aggregation/impl/MinValueAggrFunc.java
index 74a3211..323f3c4 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/aggregation/impl/MinValueAggrFunc.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/aggregation/impl/MinValueAggrFunc.java
@@ -46,7 +46,7 @@ public class MinValueAggrFunc extends AggregateFunction {
@Override
public void calculateValueFromPageHeader(PageHeader pageHeader) {
- Comparable<Object> minVal = (Comparable<Object>) pageHeader.getStatistics().getMin();
+ Comparable<Object> minVal = (Comparable<Object>) pageHeader.getStatistics().getMinValue();
updateResult(minVal);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
index 9ca26ed..42393c0 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
@@ -157,8 +157,8 @@ public class GroupByWithoutValueFilterDataSet extends GroupByEngineDataSet {
finishCheckSequenceData = calGroupByInBatchData(idx, function, unsequenceReader);
} else {
// page data
- long minTime = pageHeader.getMinTimestamp();
- long maxTime = pageHeader.getMaxTimestamp();
+ long minTime = pageHeader.getStartTime();
+ long maxTime = pageHeader.getEndTime();
// no point in sequence data with a timestamp less than endTime
if (minTime >= endTime) {
finishCheckSequenceData = true;
@@ -246,10 +246,10 @@ public class GroupByWithoutValueFilterDataSet extends GroupByEngineDataSet {
// page data
// timestamps of all points in the page are less than startTime
- if (pageHeader.getMaxTimestamp() < startTime) {
+ if (pageHeader.getEndTime() < startTime) {
sequenceReader.skipPageData();
continue;
- } else if (pageHeader.getMinTimestamp() >= startTime) {
+ } else if (pageHeader.getStartTime() >= startTime) {
// timestamps of all points in the page are greater or equal to startTime, needn't to skip
return;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/executor/AggregateEngineExecutor.java b/server/src/main/java/org/apache/iotdb/db/query/executor/AggregateEngineExecutor.java
index 8978cdc..1855b9c 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/executor/AggregateEngineExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/executor/AggregateEngineExecutor.java
@@ -183,8 +183,8 @@ public class AggregateEngineExecutor {
return false;
}
- long minTime = pageHeader.getMinTimestamp();
- long maxTime = pageHeader.getMaxTimestamp();
+ long minTime = pageHeader.getStartTime();
+ long maxTime = pageHeader.getEndTime();
// If there are points in the page that do not satisfy the time filter,
// page header cannot be used to calculate.
@@ -220,12 +220,12 @@ public class AggregateEngineExecutor {
function.calculateValueFromPageHeader(pageHeader);
sequenceReader.skipPageData();
- if (lastBatchTimeStamp > pageHeader.getMinTimestamp()) {
+ if (lastBatchTimeStamp > pageHeader.getStartTime()) {
// the chunk is end.
isChunkEnd = true;
} else {
// current page and last page are in the same chunk.
- lastBatchTimeStamp = pageHeader.getMinTimestamp();
+ lastBatchTimeStamp = pageHeader.getStartTime();
}
} else {
// cal by pageData
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/UnseqResourceMergeReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/UnseqResourceMergeReader.java
index 6566770..8c61711 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/UnseqResourceMergeReader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/UnseqResourceMergeReader.java
@@ -20,7 +20,6 @@
package org.apache.iotdb.db.query.reader.resourceRelated;
import java.io.IOException;
-import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import org.apache.iotdb.db.engine.cache.DeviceMetaDataCache;
@@ -35,11 +34,9 @@ import org.apache.iotdb.db.query.reader.chunkRelated.ChunkReaderWrap;
import org.apache.iotdb.db.query.reader.universal.PriorityMergeReader;
import org.apache.iotdb.db.utils.QueryUtils;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
-import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl;
-import org.apache.iotdb.tsfile.read.filter.DigestForFilter;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
/**
@@ -97,20 +94,8 @@ public class UnseqResourceMergeReader extends PriorityMergeReader {
for (ChunkMetaData chunkMetaData : metaDataList) {
- if (filter != null) {
- ByteBuffer minValue = null;
- ByteBuffer maxValue = null;
- ByteBuffer[] statistics = chunkMetaData.getDigest().getStatistics();
- if (statistics != null) {
- minValue = statistics[Statistics.StatisticType.min_value.ordinal()]; // note still CAN be null
- maxValue = statistics[Statistics.StatisticType.max_value.ordinal()]; // note still CAN be null
- }
-
- DigestForFilter digest = new DigestForFilter(chunkMetaData.getStartTime(),
- chunkMetaData.getEndTime(), minValue, maxValue, chunkMetaData.getTsDataType());
- if (!filter.satisfy(digest)) {
- continue;
- }
+ if (filter != null && !filter.satisfy(chunkMetaData.getStatistics())) {
+ continue;
}
// create and add DiskChunkReader
readerWrapList.add(new ChunkReaderWrap(chunkMetaData, chunkLoader, filter));
diff --git a/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java b/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java
index f6dc850..b4f3aab 100644
--- a/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java
+++ b/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java
@@ -22,25 +22,20 @@ package org.apache.iotdb.db.tools;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
-import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
-import org.apache.iotdb.tsfile.exception.filter.UnSupportFilterDataTypeException;
import org.apache.iotdb.tsfile.file.footer.ChunkGroupFooter;
import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetaData;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadata;
import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadataIndex;
import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Chunk;
import org.apache.iotdb.tsfile.utils.BloomFilter;
-import org.apache.iotdb.tsfile.utils.BytesUtils;
public class TsFileSketchTool {
@@ -107,10 +102,8 @@ public class TsFileSketchTool {
String.format("%20d", chunkMetaData.getOffsetOfChunkHeader()) + "|\t[Chunk] of "
+ chunkMetaData.getMeasurementUid() + ", numOfPoints:" + chunkMetaData
.getNumOfPoints() + ", time range:[" + chunkMetaData.getStartTime() + ","
- + chunkMetaData.getEndTime() + "], tsDataType:" + chunkMetaData.getTsDataType()
- + ", \n" + String.format("%20s", "") + " \t" + statisticByteBufferToString(
- chunkMetaData.getTsDataType(),
- chunkMetaData.getDigest()));
+ + chunkMetaData.getEndTime() + "], tsDataType:" + chunkMetaData.getDataType()
+ + ", \n" + String.format("%20s", "") + " \t" + chunkMetaData.getStatistics());
printlnBoth(pw, String.format("%20s", "") + "|\t\t[marker] 1");
printlnBoth(pw, String.format("%20s", "") + "|\t\t[ChunkHeader]");
Chunk chunk = reader.readMemChunk(chunkMetaData);
@@ -219,58 +212,4 @@ public class TsFileSketchTool {
pw.println(str);
}
- private static String statisticByteBufferToString(TSDataType tsDataType, Statistics tsDigest) {
- ByteBuffer[] statistics = tsDigest.getStatistics();
- if (statistics == null) {
- return "TsDigest:[]";
- }
- StringBuilder str = new StringBuilder();
- str.append("TsDigest:[");
- for (int i = 0; i < statistics.length - 1; i++) {
- ByteBuffer value = statistics[i];
- str.append(Statistics.StatisticType.values()[i]);
- str.append(":");
- if (value == null) {
- str.append("null");
- } else {
- switch (tsDataType) {
- case INT32:
- str.append(BytesUtils.bytesToInt(value.array()));
- break;
- case INT64:
- str.append(BytesUtils.bytesToLong(value.array()));
- break;
- case FLOAT:
- str.append(BytesUtils.bytesToFloat(value.array()));
- break;
- case DOUBLE:
- str.append(BytesUtils.bytesToDouble(value.array()));
- break;
- case TEXT:
- str.append(BytesUtils.bytesToString(value.array()));
- break;
- case BOOLEAN:
- str.append(BytesUtils.bytesToBool(value.array()));
- break;
- default:
- throw new UnSupportFilterDataTypeException(
- "DigestForFilter unsupported datatype : " + tsDataType.toString());
- }
- }
- str.append(",");
- }
- // Note that the last statistic of StatisticType is sum_value, which is double.
- str.append(Statistics.StatisticType.values()[statistics.length - 1]);
- str.append(":");
- ByteBuffer value = statistics[statistics.length - 1];
- if (value == null) {
- str.append("null");
- } else {
- str.append(BytesUtils.bytesToDouble(value.array()));
- }
- str.append("]");
-
- return str.toString();
- }
-
}
\ No newline at end of file
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskTest.java b/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskTest.java
index eaeb977..5111c6b 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskTest.java
@@ -78,7 +78,7 @@ public class MemTableFlushTaskTest {
assertEquals(MemTableTestUtils.measurementId0, chunkMetaData.getMeasurementUid());
assertEquals(startTime, chunkMetaData.getStartTime());
assertEquals(endTime, chunkMetaData.getEndTime());
- assertEquals(MemTableTestUtils.dataType0, chunkMetaData.getTsDataType());
+ assertEquals(MemTableTestUtils.dataType0, chunkMetaData.getDataType());
assertEquals(endTime - startTime + 1, chunkMetaData.getNumOfPoints());
}
}
\ No newline at end of file
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/merge/MaxSeriesMergeFileSelectorTest.java b/server/src/test/java/org/apache/iotdb/db/engine/merge/MaxSeriesMergeFileSelectorTest.java
index 2669159..ac99cf0 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/merge/MaxSeriesMergeFileSelectorTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/merge/MaxSeriesMergeFileSelectorTest.java
@@ -97,7 +97,6 @@ public class MaxSeriesMergeFileSelectorTest extends MergeTest{
List<TsFileResource> unseqSelected = result[1];
assertEquals(seqResources.subList(0, 1), seqSelected);
assertEquals(unseqResources.subList(0, 1), unseqSelected);
- assertEquals(34, mergeFileSelector.getConcurrentMergeNum());
resource.clear();
}
}
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorTest.java b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorTest.java
index 5f4b4ef..5e8d60e 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorTest.java
@@ -116,7 +116,7 @@ public class TsFileProcessorTest {
assertTrue(left.isEmpty());
assertEquals(1, right.size());
assertEquals(measurementId, right.get(0).getMeasurementUid());
- assertEquals(dataType, right.get(0).getTsDataType());
+ assertEquals(dataType, right.get(0).getDataType());
processor.syncClose();
}
@@ -162,7 +162,7 @@ public class TsFileProcessorTest {
assertTrue(left.isEmpty());
assertEquals(1, right.size());
assertEquals(measurementId, right.get(0).getMeasurementUid());
- assertEquals(dataType, right.get(0).getTsDataType());
+ assertEquals(dataType, right.get(0).getDataType());
RestorableTsFileIOWriter tsFileIOWriter = processor.getWriter();
List<ChunkGroupMetaData> chunkGroupMetaDataList = tsFileIOWriter.getChunkGroupMetaDatas();
@@ -216,7 +216,7 @@ public class TsFileProcessorTest {
assertTrue(left.isEmpty());
assertEquals(10, right.size());
assertEquals(measurementId, right.get(0).getMeasurementUid());
- assertEquals(dataType, right.get(0).getTsDataType());
+ assertEquals(dataType, right.get(0).getDataType());
processor.syncClose();
}
diff --git a/tsfile/format-changelist.md b/tsfile/format-changelist.md
index 950a284..e8d4b69 100644
--- a/tsfile/format-changelist.md
+++ b/tsfile/format-changelist.md
@@ -19,6 +19,15 @@
-->
+# version-1 -> version-2
+
+Last Updated on 2019-11-28 by Jialin Qiao.
+
+| PR# | Name | Author | Changes |
+| ---- | ------------------------------------------------------------ | --------------- | ------------------------------------------------------------ |
+| 587 | [IOTDB-325] Refactor Statistics | qiaojialin | Move start time, end time, count in PageHeader and ChunkMetadata into Statistics; Remove maxTombstoneTime in ChunkHeader |
+
+
# 0.8.0 (version-0) -> version-1
Last Updated on October 27th, 2019 by Lei Rui.
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java
index 242e6b1..e7cd634 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java
@@ -38,7 +38,7 @@ public class ChunkHeader {
private static final Logger LOG = LoggerFactory.getLogger(ChunkHeader.class);
- public static final byte MARKER = MetaMarker.CHUNK_HEADER;
+ private static final byte MARKER = MetaMarker.CHUNK_HEADER;
private String measurementID;
private int dataSize;
@@ -46,35 +46,18 @@ public class ChunkHeader {
private CompressionType compressionType;
private TSEncoding encodingType;
private int numOfPages;
- /**
- * The maximum time of the tombstones that take effect on this chunk. Only data with larger
- * timestamps than this should be exposed to user.
- */
- private long maxTombstoneTime;
// this field does not need to be serialized.
private int serializedSize;
public ChunkHeader(String measurementID, int dataSize, TSDataType dataType,
- CompressionType compressionType,
- TSEncoding encoding, int numOfPages) {
- this(measurementID, dataSize, getSerializedSize(measurementID), dataType, compressionType,
- encoding, numOfPages, 0);
- }
-
- public ChunkHeader(String measurementID, int dataSize, int headerSize, TSDataType dataType,
CompressionType compressionType, TSEncoding encoding, int numOfPages) {
- this(measurementID, dataSize, headerSize, dataType, compressionType, encoding, numOfPages, 0);
- }
-
- private ChunkHeader(String measurementID, int dataSize, TSDataType dataType,
- CompressionType compressionType, TSEncoding encoding, int numOfPages, long maxTombstoneTime) {
this(measurementID, dataSize, getSerializedSize(measurementID), dataType, compressionType,
- encoding, numOfPages, maxTombstoneTime);
+ encoding, numOfPages);
}
private ChunkHeader(String measurementID, int dataSize, int headerSize, TSDataType dataType,
- CompressionType compressionType, TSEncoding encoding, int numOfPages, long maxTombstoneTime) {
+ CompressionType compressionType, TSEncoding encoding, int numOfPages) {
this.measurementID = measurementID;
this.dataSize = dataSize;
this.dataType = dataType;
@@ -82,17 +65,22 @@ public class ChunkHeader {
this.numOfPages = numOfPages;
this.encodingType = encoding;
this.serializedSize = headerSize;
- this.maxTombstoneTime = maxTombstoneTime;
}
public static int getSerializedSize(String measurementID) {
- return Byte.BYTES + Integer.BYTES + getSerializedSize(
- measurementID.getBytes(TSFileConfig.STRING_CHARSET).length);
+ return Byte.BYTES // marker
+ + Integer.BYTES // measurementID length
+ + measurementID.getBytes(TSFileConfig.STRING_CHARSET).length // measurementID
+ + Integer.BYTES // dataSize
+ + TSDataType.getSerializedSize() // dataType
+ + CompressionType.getSerializedSize() // compressionType
+ + TSEncoding.getSerializedSize() // encodingType
+ + Integer.BYTES; // numOfPages
}
private static int getSerializedSize(int measurementIdLength) {
return measurementIdLength + Integer.BYTES + TSDataType.getSerializedSize() + Integer.BYTES
- + CompressionType.getSerializedSize() + TSEncoding.getSerializedSize() + Long.BYTES;
+ + CompressionType.getSerializedSize() + TSEncoding.getSerializedSize();
}
/**
@@ -115,9 +103,7 @@ public class ChunkHeader {
int numOfPages = ReadWriteIOUtils.readInt(inputStream);
CompressionType type = ReadWriteIOUtils.readCompressionType(inputStream);
TSEncoding encoding = ReadWriteIOUtils.readEncoding(inputStream);
- long maxTombstoneTime = ReadWriteIOUtils.readLong(inputStream);
- return new ChunkHeader(measurementID, dataSize, dataType, type, encoding, numOfPages,
- maxTombstoneTime);
+ return new ChunkHeader(measurementID, dataSize, dataType, type, encoding, numOfPages);
}
/**
@@ -151,10 +137,8 @@ public class ChunkHeader {
int numOfPages = ReadWriteIOUtils.readInt(buffer);
CompressionType type = ReadWriteIOUtils.readCompressionType(buffer);
TSEncoding encoding = ReadWriteIOUtils.readEncoding(buffer);
- long maxTombstoneTime = ReadWriteIOUtils.readLong(buffer);
return new ChunkHeader(measurementID, dataSize, chunkHeaderSize, dataType, type, encoding,
- numOfPages,
- maxTombstoneTime);
+ numOfPages);
}
public int getSerializedSize() {
@@ -189,7 +173,6 @@ public class ChunkHeader {
length += ReadWriteIOUtils.write(numOfPages, outputStream);
length += ReadWriteIOUtils.write(compressionType, outputStream);
length += ReadWriteIOUtils.write(encodingType, outputStream);
- length += ReadWriteIOUtils.write(maxTombstoneTime, outputStream);
return length;
}
@@ -208,7 +191,6 @@ public class ChunkHeader {
length += ReadWriteIOUtils.write(numOfPages, buffer);
length += ReadWriteIOUtils.write(compressionType, buffer);
length += ReadWriteIOUtils.write(encodingType, buffer);
- length += ReadWriteIOUtils.write(maxTombstoneTime, buffer);
return length;
}
@@ -224,14 +206,6 @@ public class ChunkHeader {
return encodingType;
}
- public long getMaxTombstoneTime() {
- return maxTombstoneTime;
- }
-
- public void setMaxTombstoneTime(long maxTombstoneTime) {
- this.maxTombstoneTime = maxTombstoneTime;
- }
-
@Override
public String toString() {
return "CHUNK_HEADER{" + "measurementID='" + measurementID + '\'' + ", dataSize=" + dataSize
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/PageHeader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/PageHeader.java
index f647945..08e6076 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/PageHeader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/PageHeader.java
@@ -24,9 +24,7 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.file.metadata.statistics.NoStatistics;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
-import org.apache.iotdb.tsfile.read.reader.TsFileInput;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
public class PageHeader {
@@ -34,35 +32,24 @@ public class PageHeader {
private int uncompressedSize;
private int compressedSize;
private int numOfValues;
+ private long endTime;
+ private long startTime;
private Statistics statistics;
- private long maxTimestamp;
- private long minTimestamp;
-
- // this field does not need to be serialized.
- private int serializedSize;
public PageHeader(int uncompressedSize, int compressedSize, int numOfValues,
- Statistics statistics,
- long maxTimestamp, long minTimestamp) {
+ Statistics statistics, long endTime, long startTime) {
this.uncompressedSize = uncompressedSize;
this.compressedSize = compressedSize;
this.numOfValues = numOfValues;
- if (statistics == null) {
- this.statistics = new NoStatistics();
- } else {
- this.statistics = statistics;
- }
- this.maxTimestamp = maxTimestamp;
- this.minTimestamp = minTimestamp;
- serializedSize = calculatePageHeaderSize();
+ this.statistics = statistics;
+ this.endTime = endTime;
+ this.startTime = startTime;
}
- public static int calculatePageHeaderSize(TSDataType type) {
- return calculatePageHeaderSizeWithoutStatistics() + Statistics.getStatsByType(type).getSerializedSize();
- }
public static int calculatePageHeaderSizeWithoutStatistics() {
- return 3 * Integer.BYTES + 2 * Long.BYTES;
+ return 3 * Integer.BYTES // uncompressedSize, compressedSize, numOfValues
+ + 2 * Long.BYTES; // maxTimestamp, minTimestamp
}
public static PageHeader deserializeFrom(InputStream inputStream, TSDataType dataType)
@@ -77,8 +64,7 @@ public class PageHeader {
minTimestamp);
}
- public static PageHeader deserializeFrom(ByteBuffer buffer, TSDataType dataType)
- throws IOException {
+ public static PageHeader deserializeFrom(ByteBuffer buffer, TSDataType dataType) {
int uncompressedSize = ReadWriteIOUtils.readInt(buffer);
int compressedSize = ReadWriteIOUtils.readInt(buffer);
int numOfValues = ReadWriteIOUtils.readInt(buffer);
@@ -89,60 +75,6 @@ public class PageHeader {
minTimestamp);
}
- /**
- * deserialize from TsFileInput.
- *
- * @param dataType data type
- * @param input TsFileInput
- * @param offset offset
- * @param markerRead read marker (boolean type)
- * @return CHUNK_HEADER object
- * @throws IOException IOException
- */
- public static PageHeader deserializeFrom(TSDataType dataType, TsFileInput input, long offset,
- boolean markerRead)
- throws IOException {
- long offsetVar = offset;
- if (!markerRead) {
- offsetVar++;
- }
-
- if (dataType == TSDataType.TEXT) {
- int sizeWithoutStatistics = calculatePageHeaderSizeWithoutStatistics();
- ByteBuffer bufferWithoutStatistics = ByteBuffer.allocate(sizeWithoutStatistics);
- ReadWriteIOUtils.readAsPossible(input, offsetVar, bufferWithoutStatistics);
- bufferWithoutStatistics.flip();
- offsetVar += sizeWithoutStatistics;
-
- Statistics statistics = Statistics.deserialize(input, offsetVar, dataType);
- return deserializePartFrom(statistics, bufferWithoutStatistics);
- } else {
- int size = calculatePageHeaderSize(dataType);
- ByteBuffer buffer = ByteBuffer.allocate(size);
- ReadWriteIOUtils.readAsPossible(input, offsetVar, buffer);
- buffer.flip();
- return deserializeFrom(buffer, dataType);
- }
- }
-
- private static PageHeader deserializePartFrom(Statistics statistics, ByteBuffer buffer) {
- int uncompressedSize = ReadWriteIOUtils.readInt(buffer);
- int compressedSize = ReadWriteIOUtils.readInt(buffer);
- int numOfValues = ReadWriteIOUtils.readInt(buffer);
- long maxTimestamp = ReadWriteIOUtils.readLong(buffer);
- long minTimestamp = ReadWriteIOUtils.readLong(buffer);
- return new PageHeader(uncompressedSize, compressedSize, numOfValues, statistics, maxTimestamp,
- minTimestamp);
- }
-
- public int calculatePageHeaderSize() {
- return 3 * Integer.BYTES + 2 * Long.BYTES + statistics.getSerializedSize();
- }
-
- public int getSerializedSize() {
- return serializedSize;
- }
-
public int getUncompressedSize() {
return uncompressedSize;
}
@@ -163,47 +95,33 @@ public class PageHeader {
return numOfValues;
}
- public void setNumOfValues(int numOfValues) {
- this.numOfValues = numOfValues;
- }
-
public Statistics getStatistics() {
return statistics;
}
- public long getMaxTimestamp() {
- return maxTimestamp;
- }
-
- public void setMaxTimestamp(long maxTimestamp) {
- this.maxTimestamp = maxTimestamp;
- }
-
- public long getMinTimestamp() {
- return minTimestamp;
+ public long getEndTime() {
+ return endTime;
}
- public void setMinTimestamp(long minTimestamp) {
- this.minTimestamp = minTimestamp;
+ public long getStartTime() {
+ return startTime;
}
- public int serializeTo(OutputStream outputStream) throws IOException {
- int length = 0;
- length += ReadWriteIOUtils.write(uncompressedSize, outputStream);
- length += ReadWriteIOUtils.write(compressedSize, outputStream);
- length += ReadWriteIOUtils.write(numOfValues, outputStream);
- length += ReadWriteIOUtils.write(maxTimestamp, outputStream);
- length += ReadWriteIOUtils.write(minTimestamp, outputStream);
- length += statistics.serialize(outputStream);
- return length;
+ public void serializeTo(OutputStream outputStream) throws IOException {
+ ReadWriteIOUtils.write(uncompressedSize, outputStream);
+ ReadWriteIOUtils.write(compressedSize, outputStream);
+ ReadWriteIOUtils.write(numOfValues, outputStream);
+ ReadWriteIOUtils.write(endTime, outputStream);
+ ReadWriteIOUtils.write(startTime, outputStream);
+ statistics.serialize(outputStream);
}
@Override
public String toString() {
return "PageHeader{" + "uncompressedSize=" + uncompressedSize + ", compressedSize="
+ compressedSize
- + ", numOfValues=" + numOfValues + ", statistics=" + statistics + ", maxTimestamp="
- + maxTimestamp
- + ", minTimestamp=" + minTimestamp + ", serializedSize=" + serializedSize + '}';
+ + ", numOfValues=" + numOfValues + ", statistics=" + statistics + ", endTime="
+ + endTime
+ + ", startTime=" + startTime + '}';
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/package-info.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/package-info.java
index fa36f3c..19c11bc 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/package-info.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/package-info.java
@@ -22,7 +22,7 @@ package org.apache.iotdb.tsfile.file.header;
* CHUNK_GROUP_FOOTER and CHUNK_HEADER are used for parsing file.
*
* ChunkGroupMetadata and ChunkMetadata are used for locating the positions of ChunkGroup (footer)
- * and chunk (header),filtering data quickly, and thereby they have digest information.
+ * and chunk (header),filtering data quickly, and thereby they have statistics information.
*
* However, because Page has only the header structure, therefore, PageHeader has the both two
* functions.
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaData.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaData.java
index 178b12e..b0abe2c 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaData.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaData.java
@@ -19,13 +19,10 @@
package org.apache.iotdb.tsfile.file.metadata;
-import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import java.io.IOException;
-import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.util.ArrayList;
@@ -43,11 +40,6 @@ public class ChunkGroupMetaData {
private String deviceID;
/**
- * Byte size of this metadata. this field is not serialized.
- */
- private int serializedSize;
-
- /**
* Byte offset of the corresponding data in the file Notice: include the chunk group marker.
* For Hadoop and Spark.
*/
@@ -87,39 +79,6 @@ public class ChunkGroupMetaData {
this.deviceID = deviceID;
this.chunkMetaDataList = chunkMetaDataList;
this.startOffsetOfChunkGroup = startOffsetOfChunkGroup;
- reCalculateSerializedSize();
- }
-
- /**
- * deserialize from InputStream.
- *
- * @param inputStream inputStream
- * @return ChunkGroupMetaData object
- * @throws IOException IOException
- */
- public static ChunkGroupMetaData deserializeFrom(InputStream inputStream) throws IOException {
- ChunkGroupMetaData chunkGroupMetaData = new ChunkGroupMetaData();
-
- chunkGroupMetaData.deviceID = ReadWriteIOUtils.readString(inputStream);
- chunkGroupMetaData.startOffsetOfChunkGroup = ReadWriteIOUtils.readLong(inputStream);
- chunkGroupMetaData.endOffsetOfChunkGroup = ReadWriteIOUtils.readLong(inputStream);
- chunkGroupMetaData.version = ReadWriteIOUtils.readLong(inputStream);
-
- int size = ReadWriteIOUtils.readInt(inputStream);
- chunkGroupMetaData.serializedSize = Integer.BYTES
- + chunkGroupMetaData.deviceID.getBytes(TSFileConfig.STRING_CHARSET).length
- + Integer.BYTES + Long.BYTES + Long.BYTES + Long.BYTES;
-
- List<ChunkMetaData> chunkMetaDataList = new ArrayList<>();
-
- for (int i = 0; i < size; i++) {
- ChunkMetaData metaData = ChunkMetaData.deserializeFrom(inputStream);
- chunkMetaDataList.add(metaData);
- chunkGroupMetaData.serializedSize += metaData.getSerializedSize();
- }
- chunkGroupMetaData.chunkMetaDataList = chunkMetaDataList;
-
- return chunkGroupMetaData;
}
/**
@@ -128,7 +87,7 @@ public class ChunkGroupMetaData {
* @param buffer ByteBuffer
* @return ChunkGroupMetaData object
*/
- public static ChunkGroupMetaData deserializeFrom(ByteBuffer buffer) throws IOException {
+ public static ChunkGroupMetaData deserializeFrom(ByteBuffer buffer) {
ChunkGroupMetaData chunkGroupMetaData = new ChunkGroupMetaData();
chunkGroupMetaData.deviceID = ReadWriteIOUtils.readString(buffer);
@@ -138,32 +97,16 @@ public class ChunkGroupMetaData {
int size = ReadWriteIOUtils.readInt(buffer);
- chunkGroupMetaData.serializedSize = Integer.BYTES + chunkGroupMetaData.deviceID.getBytes(TSFileConfig.STRING_CHARSET).length
- + Integer.BYTES + Long.BYTES + Long.BYTES + Long.BYTES;
-
List<ChunkMetaData> chunkMetaDataList = new ArrayList<>();
for (int i = 0; i < size; i++) {
ChunkMetaData metaData = ChunkMetaData.deserializeFrom(buffer);
chunkMetaDataList.add(metaData);
- chunkGroupMetaData.serializedSize += metaData.getSerializedSize();
}
chunkGroupMetaData.chunkMetaDataList = chunkMetaDataList;
return chunkGroupMetaData;
}
- public int getSerializedSize() {
- return serializedSize;
- }
-
- void reCalculateSerializedSize() {
- serializedSize = Integer.BYTES + deviceID.getBytes(TSFileConfig.STRING_CHARSET).length + Integer.BYTES
- + Long.BYTES + Long.BYTES + Long.BYTES; // size of chunkMetaDataList
- for (ChunkMetaData chunk : chunkMetaDataList) {
- serializedSize += chunk.getSerializedSize();
- }
- }
-
/**
* add time series chunk metadata to list. THREAD NOT SAFE
*
@@ -174,7 +117,6 @@ public class ChunkGroupMetaData {
chunkMetaDataList = new ArrayList<>();
}
chunkMetaDataList.add(metadata);
- serializedSize += metadata.getSerializedSize();
}
public List<ChunkMetaData> getChunkMetaDataList() {
@@ -231,25 +173,4 @@ public class ChunkGroupMetaData {
return byteLen;
}
- /**
- * serialize to ByteBuffer.
- *
- * @param buffer ByteBuffer
- * @return byte length
- * @throws IOException IOException
- */
- public int serializeTo(ByteBuffer buffer) throws IOException {
- int byteLen = 0;
-
- byteLen += ReadWriteIOUtils.write(deviceID, buffer);
- byteLen += ReadWriteIOUtils.write(startOffsetOfChunkGroup, buffer);
- byteLen += ReadWriteIOUtils.write(endOffsetOfChunkGroup, buffer);
- byteLen += ReadWriteIOUtils.write(version, buffer);
-
- byteLen += ReadWriteIOUtils.write(chunkMetaDataList.size(), buffer);
- for (ChunkMetaData chunkMetaData : chunkMetaDataList) {
- byteLen += chunkMetaData.serializeTo(buffer);
- }
- return byteLen;
- }
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetaData.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetaData.java
index 0015a45..4999f23 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetaData.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetaData.java
@@ -19,17 +19,13 @@
package org.apache.iotdb.tsfile.file.metadata;
-import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import java.io.IOException;
-import java.io.InputStream;
import java.io.OutputStream;
-import java.io.UnsupportedEncodingException;
import java.nio.ByteBuffer;
import java.util.Objects;
@@ -48,12 +44,6 @@ public class ChunkMetaData {
*/
private long offsetOfChunkHeader;
- private long numOfPoints;
-
- private long startTime;
-
- private long endTime;
-
private TSDataType tsDataType;
/**
@@ -67,7 +57,7 @@ public class ChunkMetaData {
*/
private long deletedAt = Long.MIN_VALUE;
- private Statistics valuesStatistics;
+ private Statistics statistics;
private ChunkMetaData() {
}
@@ -78,90 +68,23 @@ public class ChunkMetaData {
* @param measurementUid measurement id
* @param tsDataType time series data type
* @param fileOffset file offset
- * @param startTime chunk start time
- * @param endTime chunk end time
+ * @param statistics value statistics
*/
public ChunkMetaData(String measurementUid, TSDataType tsDataType, long fileOffset,
- long startTime, long endTime) {
+ Statistics statistics) {
this.measurementUid = measurementUid;
this.tsDataType = tsDataType;
this.offsetOfChunkHeader = fileOffset;
- this.startTime = startTime;
- this.endTime = endTime;
- }
-
- /**
- * deserialize from InputStream.
- *
- * @param inputStream InputStream
- * @return ChunkMetaData object
- * @throws IOException IOException
- */
- public static ChunkMetaData deserializeFrom(InputStream inputStream) throws IOException {
- ChunkMetaData chunkMetaData = new ChunkMetaData();
-
- chunkMetaData.measurementUid = ReadWriteIOUtils.readString(inputStream);
-
- chunkMetaData.offsetOfChunkHeader = ReadWriteIOUtils.readLong(inputStream);
-
- chunkMetaData.numOfPoints = ReadWriteIOUtils.readLong(inputStream);
- chunkMetaData.startTime = ReadWriteIOUtils.readLong(inputStream);
- chunkMetaData.endTime = ReadWriteIOUtils.readLong(inputStream);
-
- chunkMetaData.tsDataType = ReadWriteIOUtils.readDataType(inputStream);
-
- chunkMetaData.valuesStatistics = Statistics.deserializeFrom(inputStream, chunkMetaData.tsDataType);
-
- return chunkMetaData;
- }
-
- /**
- * deserialize from ByteBuffer.
- *
- * @param buffer ByteBuffer
- * @return ChunkMetaData object
- */
- public static ChunkMetaData deserializeFrom(ByteBuffer buffer) {
- ChunkMetaData chunkMetaData = new ChunkMetaData();
-
- chunkMetaData.measurementUid = ReadWriteIOUtils.readString(buffer);
- chunkMetaData.offsetOfChunkHeader = ReadWriteIOUtils.readLong(buffer);
- chunkMetaData.numOfPoints = ReadWriteIOUtils.readLong(buffer);
- chunkMetaData.startTime = ReadWriteIOUtils.readLong(buffer);
- chunkMetaData.endTime = ReadWriteIOUtils.readLong(buffer);
- chunkMetaData.tsDataType = ReadWriteIOUtils.readDataType(buffer);
-
- chunkMetaData.valuesStatistics = Statistics.deserializeFrom(buffer, chunkMetaData.tsDataType);
-
- return chunkMetaData;
- }
-
- /**
- * get serialized size.
- *
- * @return serialized size (int type)
- */
- public int getSerializedSize() {
- int serializedSize = (Integer.BYTES +
- 4 * Long.BYTES + // 4 long: offsetOfChunkHeader, numOfPoints, startTime, endTime
- TSDataType.getSerializedSize() + // TSDataType
- (valuesStatistics == null ? Statistics.getNullDigestSize()
- : valuesStatistics.getDigestSerializedSize()));
- serializedSize += measurementUid.getBytes(TSFileConfig.STRING_CHARSET).length; // measurementUid
- return serializedSize;
+ this.statistics = statistics;
}
@Override
public String toString() {
- return String.format("numPoints %d", numOfPoints);
+ return String.format("numPoints %d", statistics.getCount());
}
public long getNumOfPoints() {
- return numOfPoints;
- }
-
- public void setNumOfPoints(long numRows) {
- this.numOfPoints = numRows;
+ return statistics.getCount();
}
/**
@@ -177,39 +100,22 @@ public class ChunkMetaData {
return measurementUid;
}
- public Statistics getDigest() {
- return valuesStatistics;
- }
-
- public void setDigest(Statistics digest) {
- this.valuesStatistics = digest;
-
+ public Statistics getStatistics() {
+ return statistics;
}
public long getStartTime() {
- return startTime;
- }
-
- public void setStartTime(long startTime) {
- this.startTime = startTime;
+ return statistics.getStartTime();
}
public long getEndTime() {
- return endTime;
+ return statistics.getEndTime();
}
- public void setEndTime(long endTime) {
- this.endTime = endTime;
- }
-
- public TSDataType getTsDataType() {
+ public TSDataType getDataType() {
return tsDataType;
}
- public void setTsDataType(TSDataType tsDataType) {
- this.tsDataType = tsDataType;
- }
-
/**
* serialize to outputStream.
*
@@ -222,41 +128,27 @@ public class ChunkMetaData {
byteLen += ReadWriteIOUtils.write(measurementUid, outputStream);
byteLen += ReadWriteIOUtils.write(offsetOfChunkHeader, outputStream);
- byteLen += ReadWriteIOUtils.write(numOfPoints, outputStream);
- byteLen += ReadWriteIOUtils.write(startTime, outputStream);
- byteLen += ReadWriteIOUtils.write(endTime, outputStream);
byteLen += ReadWriteIOUtils.write(tsDataType, outputStream);
-
- if (valuesStatistics == null) {
- byteLen += Statistics.serializeNullTo(outputStream);
- } else {
- byteLen += valuesStatistics.serializeTo(outputStream);
- }
+ byteLen += statistics.serialize(outputStream);
return byteLen;
}
/**
- * serialize to ByteBuffer.
+ * deserialize from ByteBuffer.
*
* @param buffer ByteBuffer
- * @return length
+ * @return ChunkMetaData object
*/
- public int serializeTo(ByteBuffer buffer) {
- int byteLen = 0;
+ public static ChunkMetaData deserializeFrom(ByteBuffer buffer) {
+ ChunkMetaData chunkMetaData = new ChunkMetaData();
- byteLen += ReadWriteIOUtils.write(measurementUid, buffer);
- byteLen += ReadWriteIOUtils.write(offsetOfChunkHeader, buffer);
- byteLen += ReadWriteIOUtils.write(numOfPoints, buffer);
- byteLen += ReadWriteIOUtils.write(startTime, buffer);
- byteLen += ReadWriteIOUtils.write(endTime, buffer);
- byteLen += ReadWriteIOUtils.write(tsDataType, buffer);
-
- if (valuesStatistics == null) {
- byteLen += Statistics.serializeNullTo(buffer);
- } else {
- byteLen += valuesStatistics.serializeTo(buffer);
- }
- return byteLen;
+ chunkMetaData.measurementUid = ReadWriteIOUtils.readString(buffer);
+ chunkMetaData.offsetOfChunkHeader = ReadWriteIOUtils.readLong(buffer);
+ chunkMetaData.tsDataType = ReadWriteIOUtils.readDataType(buffer);
+
+ chunkMetaData.statistics = Statistics.deserialize(buffer, chunkMetaData.tsDataType);
+
+ return chunkMetaData;
}
public long getVersion() {
@@ -285,13 +177,10 @@ public class ChunkMetaData {
}
ChunkMetaData that = (ChunkMetaData) o;
return offsetOfChunkHeader == that.offsetOfChunkHeader &&
- numOfPoints == that.numOfPoints &&
- startTime == that.startTime &&
- endTime == that.endTime &&
version == that.version &&
deletedAt == that.deletedAt &&
Objects.equals(measurementUid, that.measurementUid) &&
tsDataType == that.tsDataType &&
- Objects.equals(valuesStatistics, that.valuesStatistics);
+ Objects.equals(statistics, that.statistics);
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsDeviceMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsDeviceMetadata.java
index 26034de..85c1e6d 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsDeviceMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsDeviceMetadata.java
@@ -32,12 +32,6 @@ import java.util.List;
public class TsDeviceMetadata {
/**
- * size of ChunkGroupMetadataBlock in byte.
- **/
- private int serializedSize =
- 2 * Long.BYTES + Integer.BYTES;// this field does not need to be serialized.
-
- /**
* start time for a device.
**/
private long startTime = Long.MAX_VALUE;
@@ -56,30 +50,6 @@ public class TsDeviceMetadata {
// allowed to clair an empty TsDeviceMetadata whose fields will be assigned later.
}
- /**
- * deserialize from the inputstream.
- *
- * @param inputStream -input stream to deserialize
- * @return -device meta data
- */
- public static TsDeviceMetadata deserializeFrom(InputStream inputStream) throws IOException {
- TsDeviceMetadata deviceMetadata = new TsDeviceMetadata();
-
- deviceMetadata.startTime = ReadWriteIOUtils.readLong(inputStream);
- deviceMetadata.endTime = ReadWriteIOUtils.readLong(inputStream);
-
- int size = ReadWriteIOUtils.readInt(inputStream);
- if (size > 0) {
- List<ChunkGroupMetaData> chunkGroupMetaDataList = new ArrayList<>();
- for (int i = 0; i < size; i++) {
- chunkGroupMetaDataList.add(ChunkGroupMetaData.deserializeFrom(inputStream));
- }
- deviceMetadata.chunkGroupMetadataList = chunkGroupMetaDataList;
- }
-
- deviceMetadata.reCalculateSerializedSize();
- return deviceMetadata;
- }
/**
* deserialize from the given buffer.
@@ -87,7 +57,7 @@ public class TsDeviceMetadata {
* @param buffer -buffer to deserialize
* @return -device meta data
*/
- public static TsDeviceMetadata deserializeFrom(ByteBuffer buffer) throws IOException {
+ public static TsDeviceMetadata deserializeFrom(ByteBuffer buffer) {
TsDeviceMetadata deviceMetadata = new TsDeviceMetadata();
deviceMetadata.startTime = ReadWriteIOUtils.readLong(buffer);
@@ -102,34 +72,9 @@ public class TsDeviceMetadata {
deviceMetadata.chunkGroupMetadataList = chunkGroupMetaDataList;
}
- deviceMetadata.reCalculateSerializedSize();
return deviceMetadata;
}
- public int getSerializedSize() {
- return serializedSize;
- }
-
- private void reCalculateSerializedSize() {
- serializedSize = 2 * Long.BYTES + // startTime , endTime
- Integer.BYTES; // size of chunkGroupMetadataList
-
- for (ChunkGroupMetaData meta : chunkGroupMetadataList) {
- serializedSize += meta.getSerializedSize();
- }
- }
-
- /**
- * set the ChunkGroupMetadataList and recalculate serialized size.
- *
- * @param chunkGroupMetadataList -use to set the ChunkGroupMetadataList and recalculate serialized
- * size
- */
- public void setChunkGroupMetadataList(List<ChunkGroupMetaData> chunkGroupMetadataList) {
- this.chunkGroupMetadataList = chunkGroupMetadataList;
- reCalculateSerializedSize();
- }
-
/**
* add chunk group metadata to chunkGroups. THREAD NOT SAFE
*
@@ -137,7 +82,6 @@ public class TsDeviceMetadata {
*/
public void addChunkGroupMetaData(ChunkGroupMetaData chunkGroup) {
chunkGroupMetadataList.add(chunkGroup);
- serializedSize += chunkGroup.getSerializedSize();
for (ChunkMetaData chunkMetaData : chunkGroup.getChunkMetaDataList()) {
// update startTime and endTime
startTime = Long.min(startTime, chunkMetaData.getStartTime());
@@ -187,33 +131,10 @@ public class TsDeviceMetadata {
return byteLen;
}
- /**
- * get the byte length of the given buffer.
- *
- * @param buffer -buffer to determine the byte length
- * @return -byte length
- */
- public int serializeTo(ByteBuffer buffer) throws IOException {
- int byteLen = 0;
-
- byteLen += ReadWriteIOUtils.write(startTime, buffer);
- byteLen += ReadWriteIOUtils.write(endTime, buffer);
-
- if (chunkGroupMetadataList == null) {
- byteLen += ReadWriteIOUtils.write(0, buffer);
- } else {
- byteLen += ReadWriteIOUtils.write(chunkGroupMetadataList.size(), buffer);
- for (ChunkGroupMetaData chunkGroupMetaData : chunkGroupMetadataList) {
- byteLen += chunkGroupMetaData.serializeTo(buffer);
- }
- }
-
- return byteLen;
- }
@Override
public String toString() {
- return "TsDeviceMetadata{" + "serializedSize=" + serializedSize + ", startTime=" + startTime
+ return "TsDeviceMetadata{" + " startTime=" + startTime
+ ", endTime="
+ endTime + ", chunkGroupMetadataList=" + chunkGroupMetadataList + '}';
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BinaryStatistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BinaryStatistics.java
index 64dc577..b54e7f1 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BinaryStatistics.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BinaryStatistics.java
@@ -20,10 +20,10 @@ package org.apache.iotdb.tsfile.file.metadata.statistics;
import java.io.IOException;
import java.io.InputStream;
+import java.io.OutputStream;
import java.nio.ByteBuffer;
-import org.apache.iotdb.tsfile.read.reader.TsFileInput;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.utils.Binary;
-import org.apache.iotdb.tsfile.utils.BytesUtils;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
/**
@@ -31,197 +31,167 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
*/
public class BinaryStatistics extends Statistics<Binary> {
- private Binary min = new Binary("");
- private Binary max = new Binary("");
- private Binary first = new Binary("");
- private Binary last = new Binary("");
- private double sum;// FIXME sum is meaningless
+ private Binary firstValue = new Binary("");
+ private Binary lastValue = new Binary("");
@Override
- public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) {
- min = new Binary(minBytes);
- max = new Binary(maxBytes);
+ public TSDataType getType() {
+ return TSDataType.TEXT;
}
@Override
- public Binary getMin() {
- return min;
+ public int getStatsSize() {
+ return 4 + firstValue.getValues().length
+ + 4 + lastValue.getValues().length;
+ }
+
+ /**
+ * initialize Statistics.
+ *
+ * @param first the first value
+ * @param last the last value
+ */
+ private void initializeStats(Binary first, Binary last) {
+ this.firstValue = first;
+ this.lastValue = last;
+ }
+
+ private void updateStats(Binary firstValue, Binary lastValue) {
+ this.lastValue = lastValue;
}
@Override
- public Binary getMax() {
- return max;
+ public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) {
}
@Override
- public Binary getFirst() {
- return first;
+ public Binary getMinValue() {
+ throw new StatisticsClassException("Binary statistics does not support: min");
}
@Override
- public Binary getLast() {
- return last;
+ public Binary getMaxValue() {
+ throw new StatisticsClassException("Binary statistics does not support: max");
}
@Override
- public double getSum() {
- return sum;
+ public Binary getFirstValue() {
+ return firstValue;
}
- /**
- * initialize Statistics.
- *
- * @param min minimum value
- * @param max maximum value
- * @param first the first value
- * @param last the last value
- * @param sum sum
- */
- private void initializeStats(Binary min, Binary max, Binary first, Binary last, double sum) {
- this.min = min;
- this.max = max;
- this.first = first;
- this.last = last;
- this.sum = sum;
+ @Override
+ public Binary getLastValue() {
+ return lastValue;
}
@Override
- protected void mergeStatisticsValue(Statistics<?> stats) {
+ public double getSumValue() {
+ throw new StatisticsClassException("Binary statistics does not support: sum");
+ }
+
+ @Override
+ protected void mergeStatisticsValue(Statistics stats) {
BinaryStatistics stringStats = (BinaryStatistics) stats;
if (isEmpty) {
- initializeStats(stringStats.getMin(), stringStats.getMax(), stringStats.getFirst(),
- stringStats.getLast(), stringStats.getSum());
+ initializeStats(stringStats.getFirstValue(), stringStats.getLastValue());
isEmpty = false;
} else {
- updateStats(stringStats.getMin(), stringStats.getMax(), stringStats.getFirst(),
- stringStats.getLast(), stringStats.getSum());
+ updateStats(stringStats.getFirstValue(), stringStats.getLastValue());
}
}
@Override
- public void updateStats(Binary value) {
+ void updateStats(Binary value) {
if (isEmpty) {
- initializeStats(value, value, value, value, 0);
+ initializeStats(value, value);
isEmpty = false;
} else {
- updateStats(value, value, value, value, 0);
- isEmpty = false;
+ updateStats(value, value);
}
}
@Override
- public void updateStats(Binary[] values) {
- for (Binary value : values) {
- if (isEmpty) {
- initializeStats(value, value, value, value, 0);
- isEmpty = false;
- } else {
- updateStats(value, value, value, value, 0);
- isEmpty = false;
- }
- }
- }
-
- private void updateStats(Binary minValue, Binary maxValue, Binary firstValue, Binary lastValue,
- double sum) {
- if (minValue.compareTo(min) < 0) {
- min = minValue;
- }
- if (maxValue.compareTo(max) > 0) {
- max = maxValue;
+ void updateStats(Binary[] values, int batchSize) {
+ for (int i = 0; i < batchSize; i++) {
+ updateStats(values[i]);
}
- this.last = lastValue;
}
@Override
- public byte[] getMinBytes() {
- return min.getValues();
+ public byte[] getMinValueBytes() {
+ throw new StatisticsClassException("Binary statistics does not support: min");
}
@Override
- public byte[] getMaxBytes() {
- return max.getValues();
+ public byte[] getMaxValueBytes() {
+ throw new StatisticsClassException("Binary statistics does not support: max");
}
@Override
- public byte[] getFirstBytes() {
- return first.getValues();
+ public byte[] getFirstValueBytes() {
+ return firstValue.getValues();
}
@Override
- public byte[] getLastBytes() {
- return last.getValues();
+ public byte[] getLastValueBytes() {
+ return lastValue.getValues();
}
@Override
- public byte[] getSumBytes() {
- return BytesUtils.doubleToBytes(sum);
+ public byte[] getSumValueBytes() {
+ throw new StatisticsClassException("Binary statistics does not support: sum");
}
@Override
- public ByteBuffer getMinBytebuffer() {
- return ByteBuffer.wrap(min.getValues());
+ public ByteBuffer getMinValueBuffer() {
+ throw new StatisticsClassException("Binary statistics does not support: min");
}
@Override
- public ByteBuffer getMaxBytebuffer() {
- return ByteBuffer.wrap(max.getValues());
+ public ByteBuffer getMaxValueBuffer() {
+ throw new StatisticsClassException("Binary statistics does not support: max");
}
@Override
- public ByteBuffer getFirstBytebuffer() {
- return ByteBuffer.wrap(first.getValues());
+ public ByteBuffer getFirstValueBuffer() {
+ return ByteBuffer.wrap(firstValue.getValues());
}
@Override
- public ByteBuffer getLastBytebuffer() {
- return ByteBuffer.wrap(last.getValues());
+ public ByteBuffer getLastValueBuffer() {
+ return ByteBuffer.wrap(lastValue.getValues());
}
@Override
- public ByteBuffer getSumBytebuffer() {
- return ReadWriteIOUtils.getByteBuffer(sum);
+ public ByteBuffer getSumValueBuffer() {
+ throw new StatisticsClassException("Binary statistics does not support: sum");
}
@Override
- public int sizeOfDatum() {
- return -1;
- }
-
- @Override
- public String toString() {
- return "[min:" + min + ",max:" + max + ",first:" + first + ",last:" + last + ",sum:" + sum
- + "]";
+ public int serializeStats(OutputStream outputStream) throws IOException {
+ int byteLen = 0;
+ byteLen += ReadWriteIOUtils.write(firstValue, outputStream);
+ byteLen += ReadWriteIOUtils.write(lastValue, outputStream);
+ return byteLen;
}
@Override
void deserialize(InputStream inputStream) throws IOException {
- this.min = new Binary(ReadWriteIOUtils.readBytesWithSelfDescriptionLength(inputStream));
- this.max = new Binary(ReadWriteIOUtils.readBytesWithSelfDescriptionLength(inputStream));
- this.first = new Binary(ReadWriteIOUtils.readBytesWithSelfDescriptionLength(inputStream));
- this.last = new Binary(ReadWriteIOUtils.readBytesWithSelfDescriptionLength(inputStream));
- this.sum = ReadWriteIOUtils.readDouble(inputStream);
+ this.firstValue = new Binary(ReadWriteIOUtils.readBytesWithSelfDescriptionLength(inputStream));
+ this.lastValue = new Binary(ReadWriteIOUtils.readBytesWithSelfDescriptionLength(inputStream));
}
@Override
- void deserialize(ByteBuffer byteBuffer) throws IOException {
- this.min = new Binary(
+ void deserialize(ByteBuffer byteBuffer) {
+ this.firstValue = new Binary(
ReadWriteIOUtils.readByteBufferWithSelfDescriptionLength(byteBuffer).array());
- this.max = new Binary(
+ this.lastValue = new Binary(
ReadWriteIOUtils.readByteBufferWithSelfDescriptionLength(byteBuffer).array());
- this.first = new Binary(
- ReadWriteIOUtils.readByteBufferWithSelfDescriptionLength(byteBuffer).array());
- this.last = new Binary(
- ReadWriteIOUtils.readByteBufferWithSelfDescriptionLength(byteBuffer).array());
- this.sum = ReadWriteIOUtils.readDouble(byteBuffer);
}
@Override
- protected void deserialize(TsFileInput input, long offset) throws IOException {
- int size = getSerializedSize();
- ByteBuffer buffer = ByteBuffer.allocate(size);
- ReadWriteIOUtils.readAsPossible(input, offset, buffer);
- buffer.flip();
- deserialize(buffer);
+ public String toString() {
+ return "[fistValue:" + firstValue + ",lastValue:" + lastValue + "]";
}
+
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatistics.java
index 097dba0..bb04470 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatistics.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatistics.java
@@ -20,194 +20,172 @@ package org.apache.iotdb.tsfile.file.metadata.statistics;
import java.io.IOException;
import java.io.InputStream;
+import java.io.OutputStream;
import java.nio.ByteBuffer;
+import org.apache.iotdb.tsfile.exception.TsFileRuntimeException;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.utils.BytesUtils;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
-/**
- * Boolean Statistics.
- */
public class BooleanStatistics extends Statistics<Boolean> {
- private boolean min;
- private boolean max;
- private boolean first;
- private boolean last;
- private double sum;
+ private boolean firstValue;
+ private boolean lastValue;
@Override
- public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) {
- min = BytesUtils.bytesToBool(minBytes);
- max = BytesUtils.bytesToBool(maxBytes);
+ public TSDataType getType() {
+ return TSDataType.BOOLEAN;
}
@Override
- public void updateStats(boolean value) {
+ public int getStatsSize() {
+ return 2;
+ }
+
+ /**
+ * initialize boolean Statistics.
+ *
+ * @param firstValue first boolean value
+ * @param lastValue last boolean value
+ */
+ private void initializeStats(boolean firstValue, boolean lastValue) {
+ this.firstValue = firstValue;
+ this.lastValue = lastValue;
+ }
+
+ private void updateStats(boolean firstValue, boolean lastValue) {
+ this.lastValue = lastValue;
+ }
+
+ @Override
+ void updateStats(boolean value) {
if (isEmpty) {
- initializeStats(value, value, value, value, 0);
+ initializeStats(value, value);
isEmpty = false;
} else {
- updateStats(value, value, value, value, 0);
- isEmpty = false;
+ updateStats(value, value);
}
}
@Override
- public void updateStats(boolean[] values) {
- for (boolean value : values) {
- if (isEmpty) {
- initializeStats(value, value, value, value, 0);
- isEmpty = false;
- } else {
- updateStats(value, value, value, value, 0);
- isEmpty = false;
- }
+ void updateStats(boolean[] values, int batchSize) {
+ for (int i = 0; i < batchSize; i++) {
+ updateStats(values[i]);
}
}
- private void updateStats(boolean minValue, boolean maxValue, boolean firstValue,
- boolean lastValue, double sumValue) {
- if (!minValue && min) {
- min = false;
- }
- if (maxValue && !max) {
- max = true;
- }
- this.last = lastValue;
+ @Override
+ public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) {
}
@Override
- public Boolean getMin() {
- return min;
+ public Boolean getMinValue() {
+ throw new StatisticsClassException("Boolean statistics does not support: min");
}
@Override
- public Boolean getMax() {
- return max;
+ public Boolean getMaxValue() {
+ throw new StatisticsClassException("Boolean statistics does not support: max");
}
@Override
- public Boolean getFirst() {
- return first;
+ public Boolean getFirstValue() {
+ return firstValue;
}
@Override
- public Boolean getLast() {
- return last;
+ public Boolean getLastValue() {
+ return lastValue;
}
@Override
- public double getSum() {
- return sum;
+ public double getSumValue() {
+ throw new StatisticsClassException("Boolean statistics does not support: sum");
}
@Override
- public ByteBuffer getMinBytebuffer() {
- return ReadWriteIOUtils.getByteBuffer(min);
+ public ByteBuffer getMinValueBuffer() {
+ throw new StatisticsClassException("Boolean statistics do not support: min");
}
@Override
- public ByteBuffer getMaxBytebuffer() {
- return ReadWriteIOUtils.getByteBuffer(max);
+ public ByteBuffer getMaxValueBuffer() {
+ throw new StatisticsClassException("Boolean statistics do not support: max");
}
@Override
- public ByteBuffer getFirstBytebuffer() {
- return ReadWriteIOUtils.getByteBuffer(first);
+ public ByteBuffer getFirstValueBuffer() {
+ return ReadWriteIOUtils.getByteBuffer(firstValue);
}
@Override
- public ByteBuffer getLastBytebuffer() {
- return ReadWriteIOUtils.getByteBuffer(last);
+ public ByteBuffer getLastValueBuffer() {
+ return ReadWriteIOUtils.getByteBuffer(lastValue);
}
@Override
- public ByteBuffer getSumBytebuffer() {
- return ReadWriteIOUtils.getByteBuffer(sum);
+ public ByteBuffer getSumValueBuffer() {
+ throw new StatisticsClassException("Boolean statistics do not support: sum");
}
@Override
- protected void mergeStatisticsValue(Statistics<?> stats) {
+ protected void mergeStatisticsValue(Statistics stats) {
BooleanStatistics boolStats = (BooleanStatistics) stats;
if (isEmpty) {
- initializeStats(boolStats.getMin(), boolStats.getMax(), boolStats.getFirst(),
- boolStats.getLast(), boolStats.getSum());
+ initializeStats(boolStats.getFirstValue(), boolStats.getLastValue());
isEmpty = false;
} else {
- updateStats(boolStats.getMin(), boolStats.getMax(), boolStats.getFirst(),
- boolStats.getLast(), boolStats.getSum());
+ updateStats(boolStats.getFirstValue(), boolStats.getLastValue());
}
}
- /**
- * initialize boolean Statistics.
- *
- * @param min min boolean
- * @param max max boolean
- * @param firstValue first boolean value
- * @param lastValue last boolean value
- * @param sumValue sum value (double type)
- */
- private void initializeStats(boolean min, boolean max, boolean firstValue, boolean lastValue,
- double sumValue) {
- this.min = min;
- this.max = max;
- this.first = firstValue;
- this.last = lastValue;
- }
-
@Override
- public byte[] getMinBytes() {
- return BytesUtils.boolToBytes(min);
+ public byte[] getMinValueBytes() {
+ throw new StatisticsClassException("Boolean statistics does not support: min");
}
@Override
- public byte[] getMaxBytes() {
- return BytesUtils.boolToBytes(max);
+ public byte[] getMaxValueBytes() {
+ throw new StatisticsClassException("Boolean statistics does not support: max");
}
@Override
- public byte[] getFirstBytes() {
- return BytesUtils.boolToBytes(first);
+ public byte[] getFirstValueBytes() {
+ return BytesUtils.boolToBytes(firstValue);
}
@Override
- public byte[] getLastBytes() {
- return BytesUtils.boolToBytes(last);
+ public byte[] getLastValueBytes() {
+ return BytesUtils.boolToBytes(lastValue);
}
@Override
- public byte[] getSumBytes() {
- return BytesUtils.doubleToBytes(sum);
+ public byte[] getSumValueBytes() {
+ throw new StatisticsClassException("Boolean statistics does not support: sum");
}
@Override
- public int sizeOfDatum() {
- return 1;
+ public int serializeStats(OutputStream outputStream) throws IOException {
+ int byteLen = 0;
+ byteLen += ReadWriteIOUtils.write(firstValue, outputStream);
+ byteLen += ReadWriteIOUtils.write(lastValue, outputStream);
+ return byteLen;
}
@Override
- public String toString() {
- return "[min:" + min + ",max:" + max + ",first:" + first + ",last:" + last + ",sum:" + sum
- + "]";
+ void deserialize(InputStream inputStream) throws IOException {
+ this.firstValue = ReadWriteIOUtils.readBool(inputStream);
+ this.lastValue = ReadWriteIOUtils.readBool(inputStream);
}
@Override
- void deserialize(InputStream inputStream) throws IOException {
- this.min = ReadWriteIOUtils.readBool(inputStream);
- this.max = ReadWriteIOUtils.readBool(inputStream);
- this.first = ReadWriteIOUtils.readBool(inputStream);
- this.last = ReadWriteIOUtils.readBool(inputStream);
- this.sum = ReadWriteIOUtils.readDouble(inputStream);
+ void deserialize(ByteBuffer byteBuffer) {
+ this.firstValue = ReadWriteIOUtils.readBool(byteBuffer);
+ this.lastValue = ReadWriteIOUtils.readBool(byteBuffer);
}
@Override
- void deserialize(ByteBuffer byteBuffer) throws IOException {
- this.min = ReadWriteIOUtils.readBool(byteBuffer);
- this.max = ReadWriteIOUtils.readBool(byteBuffer);
- this.first = ReadWriteIOUtils.readBool(byteBuffer);
- this.last = ReadWriteIOUtils.readBool(byteBuffer);
- this.sum = ReadWriteIOUtils.readDouble(byteBuffer);
+ public String toString() {
+ return "[firstValue:" + firstValue + ",lastValue:" + lastValue + "]";
}
-
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatistics.java
index 250aaac..85ae01e 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatistics.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatistics.java
@@ -20,29 +20,67 @@ package org.apache.iotdb.tsfile.file.metadata.statistics;
import java.io.IOException;
import java.io.InputStream;
+import java.io.OutputStream;
import java.nio.ByteBuffer;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.utils.BytesUtils;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
-/**
- * Statistics for double type.
- */
public class DoubleStatistics extends Statistics<Double> {
- private double min;
- private double max;
- private double first;
- private double last;
- private double sum;
+ private double minValue;
+ private double maxValue;
+ private double firstValue;
+ private double lastValue;
+ private double sumValue;
+
+ @Override
+ public TSDataType getType() {
+ return TSDataType.DOUBLE;
+ }
+
+ @Override
+ public int getStatsSize() {
+ return 40;
+ }
+
+ /**
+ * initialize double statistics.
+ *
+ * @param min min value
+ * @param max max value
+ * @param first the first value
+ * @param last the last value
+ * @param sum sum value
+ */
+ private void initializeStats(double min, double max, double first, double last, double sum) {
+ this.minValue = min;
+ this.maxValue = max;
+ this.firstValue = first;
+ this.lastValue = last;
+ this.sumValue = sum;
+ }
+
+ private void updateStats(double minValue, double maxValue, double firstValue, double lastValue,
+ double sumValue) {
+ if (minValue < this.minValue) {
+ this.minValue = minValue;
+ }
+ if (maxValue > this.maxValue) {
+ this.maxValue = maxValue;
+ }
+ this.sumValue += sumValue;
+ this.lastValue = lastValue;
+ }
@Override
public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) {
- min = BytesUtils.bytesToDouble(minBytes);
- max = BytesUtils.bytesToDouble(maxBytes);
+ minValue = BytesUtils.bytesToDouble(minBytes);
+ maxValue = BytesUtils.bytesToDouble(maxBytes);
}
@Override
- public void updateStats(double value) {
+ void updateStats(double value) {
if (this.isEmpty) {
initializeStats(value, value, value, value, value);
isEmpty = false;
@@ -52,161 +90,132 @@ public class DoubleStatistics extends Statistics<Double> {
}
@Override
- public void updateStats(double[] values) {
- for (double value : values) {
- if (this.isEmpty) {
- initializeStats(value, value, value, value, value);
- isEmpty = false;
- } else {
- updateStats(value, value, value, value, value);
- }
- }
- }
-
- private void updateStats(double minValue, double maxValue, double firstValue, double lastValue,
- double sumValue) {
- if (minValue < min) {
- min = minValue;
+ void updateStats(double[] values, int batchSize) {
+ for (int i = 0; i < batchSize; i++) {
+ updateStats(values[i]);
}
- if (maxValue > max) {
- max = maxValue;
- }
- sum += sumValue;
- this.last = lastValue;
}
@Override
- public Double getMin() {
- return min;
+ public Double getMinValue() {
+ return minValue;
}
@Override
- public Double getMax() {
- return max;
+ public Double getMaxValue() {
+ return maxValue;
}
@Override
- public Double getFirst() {
- return first;
+ public Double getFirstValue() {
+ return firstValue;
}
@Override
- public Double getLast() {
- return last;
+ public Double getLastValue() {
+ return lastValue;
}
@Override
- public double getSum() {
- return sum;
+ public double getSumValue() {
+ return sumValue;
}
@Override
- protected void mergeStatisticsValue(Statistics<?> stats) {
+ protected void mergeStatisticsValue(Statistics stats) {
DoubleStatistics doubleStats = (DoubleStatistics) stats;
if (this.isEmpty) {
- initializeStats(doubleStats.getMin(), doubleStats.getMax(), doubleStats.getFirst(),
- doubleStats.getLast(), doubleStats.getSum());
+ initializeStats(doubleStats.getMinValue(), doubleStats.getMaxValue(), doubleStats.getFirstValue(),
+ doubleStats.getLastValue(), doubleStats.getSumValue());
isEmpty = false;
} else {
- updateStats(doubleStats.getMin(), doubleStats.getMax(), doubleStats.getFirst(),
- doubleStats.getLast(), doubleStats.getSum());
+ updateStats(doubleStats.getMinValue(), doubleStats.getMaxValue(), doubleStats.getFirstValue(),
+ doubleStats.getLastValue(), doubleStats.getSumValue());
}
-
- }
-
- /**
- * initialize double statistics.
- *
- * @param min min value
- * @param max max value
- * @param first the first value
- * @param last the last value
- * @param sum sum value
- */
- private void initializeStats(double min, double max, double first, double last, double sum) {
- this.min = min;
- this.max = max;
- this.first = first;
- this.last = last;
- this.sum = sum;
}
@Override
- public byte[] getMinBytes() {
- return BytesUtils.doubleToBytes(min);
+ public byte[] getMinValueBytes() {
+ return BytesUtils.doubleToBytes(minValue);
}
@Override
- public byte[] getMaxBytes() {
- return BytesUtils.doubleToBytes(max);
+ public byte[] getMaxValueBytes() {
+ return BytesUtils.doubleToBytes(maxValue);
}
@Override
- public byte[] getFirstBytes() {
- return BytesUtils.doubleToBytes(first);
+ public byte[] getFirstValueBytes() {
+ return BytesUtils.doubleToBytes(firstValue);
}
@Override
- public byte[] getLastBytes() {
- return BytesUtils.doubleToBytes(last);
+ public byte[] getLastValueBytes() {
+ return BytesUtils.doubleToBytes(lastValue);
}
@Override
- public byte[] getSumBytes() {
- return BytesUtils.doubleToBytes(sum);
+ public byte[] getSumValueBytes() {
+ return BytesUtils.doubleToBytes(sumValue);
}
@Override
- public ByteBuffer getMinBytebuffer() {
- return ReadWriteIOUtils.getByteBuffer(min);
+ public ByteBuffer getMinValueBuffer() {
+ return ReadWriteIOUtils.getByteBuffer(minValue);
}
@Override
- public ByteBuffer getMaxBytebuffer() {
- return ReadWriteIOUtils.getByteBuffer(max);
+ public ByteBuffer getMaxValueBuffer() {
+ return ReadWriteIOUtils.getByteBuffer(maxValue);
}
@Override
- public ByteBuffer getFirstBytebuffer() {
- return ReadWriteIOUtils.getByteBuffer(first);
+ public ByteBuffer getFirstValueBuffer() {
+ return ReadWriteIOUtils.getByteBuffer(firstValue);
}
@Override
- public ByteBuffer getLastBytebuffer() {
- return ReadWriteIOUtils.getByteBuffer(last);
+ public ByteBuffer getLastValueBuffer() {
+ return ReadWriteIOUtils.getByteBuffer(lastValue);
}
@Override
- public ByteBuffer getSumBytebuffer() {
- return ReadWriteIOUtils.getByteBuffer(sum);
+ public ByteBuffer getSumValueBuffer() {
+ return ReadWriteIOUtils.getByteBuffer(sumValue);
}
@Override
- public int sizeOfDatum() {
- return 8;
+ public int serializeStats(OutputStream outputStream) throws IOException {
+ int byteLen = 0;
+ byteLen += ReadWriteIOUtils.write(minValue, outputStream);
+ byteLen += ReadWriteIOUtils.write(maxValue, outputStream);
+ byteLen += ReadWriteIOUtils.write(firstValue, outputStream);
+ byteLen += ReadWriteIOUtils.write(lastValue, outputStream);
+ byteLen += ReadWriteIOUtils.write(sumValue, outputStream);
+ return byteLen;
}
@Override
- public String toString() {
- return "[min:" + min + ",max:" + max + ",first:" + first + ",last:" + last + ",sum:" + sum
- + "]";
+ void deserialize(InputStream inputStream) throws IOException {
+ this.minValue = ReadWriteIOUtils.readDouble(inputStream);
+ this.maxValue = ReadWriteIOUtils.readDouble(inputStream);
+ this.firstValue = ReadWriteIOUtils.readDouble(inputStream);
+ this.lastValue = ReadWriteIOUtils.readDouble(inputStream);
+ this.sumValue = ReadWriteIOUtils.readDouble(inputStream);
}
@Override
- void deserialize(InputStream inputStream) throws IOException {
- this.min = ReadWriteIOUtils.readDouble(inputStream);
- this.max = ReadWriteIOUtils.readDouble(inputStream);
- this.first = ReadWriteIOUtils.readDouble(inputStream);
- this.last = ReadWriteIOUtils.readDouble(inputStream);
- this.sum = ReadWriteIOUtils.readDouble(inputStream);
+ void deserialize(ByteBuffer byteBuffer) {
+ this.minValue = ReadWriteIOUtils.readDouble(byteBuffer);
+ this.maxValue = ReadWriteIOUtils.readDouble(byteBuffer);
+ this.firstValue = ReadWriteIOUtils.readDouble(byteBuffer);
+ this.lastValue = ReadWriteIOUtils.readDouble(byteBuffer);
+ this.sumValue = ReadWriteIOUtils.readDouble(byteBuffer);
}
@Override
- void deserialize(ByteBuffer byteBuffer) throws IOException {
- this.min = ReadWriteIOUtils.readDouble(byteBuffer);
- this.max = ReadWriteIOUtils.readDouble(byteBuffer);
- this.first = ReadWriteIOUtils.readDouble(byteBuffer);
- this.last = ReadWriteIOUtils.readDouble(byteBuffer);
- this.sum = ReadWriteIOUtils.readDouble(byteBuffer);
+ public String toString() {
+ return "[minValue:" + minValue + ",maxValue:" + maxValue + ",firstValue:" + firstValue +
+ ",lastValue:" + lastValue + ",sumValue:" + sumValue + "]";
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatistics.java
index e1f487b..4daaf77 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatistics.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatistics.java
@@ -20,7 +20,9 @@ package org.apache.iotdb.tsfile.file.metadata.statistics;
import java.io.IOException;
import java.io.InputStream;
+import java.io.OutputStream;
import java.nio.ByteBuffer;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.utils.BytesUtils;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
@@ -29,20 +31,50 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
*/
public class FloatStatistics extends Statistics<Float> {
- private float min;
- private float max;
- private float first;
- private double sum;
- private float last;
+ private float minValue;
+ private float maxValue;
+ private float firstValue;
+ private float lastValue;
+ private double sumValue;
+
+ @Override
+ public TSDataType getType() {
+ return TSDataType.FLOAT;
+ }
+
+ @Override
+ public int getStatsSize() {
+ return 24;
+ }
+
+ private void initializeStats(float min, float max, float first, float last, double sum) {
+ this.minValue = min;
+ this.maxValue = max;
+ this.firstValue = first;
+ this.lastValue = last;
+ this.sumValue = sum;
+ }
+
+ private void updateStats(float minValue, float maxValue, float firstValue, float last,
+ double sumValue) {
+ if (minValue < this.minValue) {
+ this.minValue = minValue;
+ }
+ if (maxValue > this.maxValue) {
+ this.maxValue = maxValue;
+ }
+ this.sumValue += sumValue;
+ this.lastValue = last;
+ }
@Override
public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) {
- min = BytesUtils.bytesToFloat(minBytes);
- max = BytesUtils.bytesToFloat(maxBytes);
+ minValue = BytesUtils.bytesToFloat(minBytes);
+ maxValue = BytesUtils.bytesToFloat(maxBytes);
}
@Override
- public void updateStats(float value) {
+ void updateStats(float value) {
if (this.isEmpty) {
initializeStats(value, value, value, value, value);
isEmpty = false;
@@ -52,152 +84,132 @@ public class FloatStatistics extends Statistics<Float> {
}
@Override
- public void updateStats(float[] values) {
- for (float value : values) {
- if (this.isEmpty) {
- initializeStats(value, value, value, value, value);
- isEmpty = false;
- } else {
- updateStats(value, value, value, value, value);
- }
+ void updateStats(float[] values, int batchSize) {
+ for (int i = 0; i < batchSize; i++) {
+ updateStats(values[i]);
}
}
- private void updateStats(float minValue, float maxValue, float firstValue, float last,
- double sumValue) {
- if (minValue < min) {
- min = minValue;
- }
- if (maxValue > max) {
- max = maxValue;
- }
- sum += sumValue;
- this.last = last;
- }
-
@Override
- public Float getMin() {
- return min;
+ public Float getMinValue() {
+ return minValue;
}
@Override
- public Float getMax() {
- return max;
+ public Float getMaxValue() {
+ return maxValue;
}
@Override
- public Float getFirst() {
- return first;
+ public Float getFirstValue() {
+ return firstValue;
}
@Override
- public Float getLast() {
- return last;
+ public Float getLastValue() {
+ return lastValue;
}
@Override
- public double getSum() {
- return sum;
+ public double getSumValue() {
+ return sumValue;
}
@Override
- protected void mergeStatisticsValue(Statistics<?> stats) {
+ protected void mergeStatisticsValue(Statistics stats) {
FloatStatistics floatStats = (FloatStatistics) stats;
if (isEmpty) {
- initializeStats(floatStats.getMin(), floatStats.getMax(), floatStats.getFirst(),
- floatStats.getLast(), floatStats.getSum());
+ initializeStats(floatStats.getMinValue(), floatStats.getMaxValue(), floatStats.getFirstValue(),
+ floatStats.getLastValue(), floatStats.getSumValue());
isEmpty = false;
} else {
- updateStats(floatStats.getMin(), floatStats.getMax(), floatStats.getFirst(),
- floatStats.getLast(), floatStats.getSum());
+ updateStats(floatStats.getMinValue(), floatStats.getMaxValue(), floatStats.getFirstValue(),
+ floatStats.getLastValue(), floatStats.getSumValue());
}
-
- }
-
- private void initializeStats(float min, float max, float first, float last, double sum) {
- this.min = min;
- this.max = max;
- this.first = first;
- this.last = last;
- this.sum = sum;
}
@Override
- public byte[] getMinBytes() {
- return BytesUtils.floatToBytes(min);
+ public byte[] getMinValueBytes() {
+ return BytesUtils.floatToBytes(minValue);
}
@Override
- public byte[] getMaxBytes() {
- return BytesUtils.floatToBytes(max);
+ public byte[] getMaxValueBytes() {
+ return BytesUtils.floatToBytes(maxValue);
}
@Override
- public byte[] getFirstBytes() {
- return BytesUtils.floatToBytes(first);
+ public byte[] getFirstValueBytes() {
+ return BytesUtils.floatToBytes(firstValue);
}
@Override
- public byte[] getLastBytes() {
- return BytesUtils.floatToBytes(last);
+ public byte[] getLastValueBytes() {
+ return BytesUtils.floatToBytes(lastValue);
}
@Override
- public byte[] getSumBytes() {
- return BytesUtils.doubleToBytes(sum);
+ public byte[] getSumValueBytes() {
+ return BytesUtils.doubleToBytes(sumValue);
}
@Override
- public ByteBuffer getMinBytebuffer() {
- return ReadWriteIOUtils.getByteBuffer(min);
+ public ByteBuffer getMinValueBuffer() {
+ return ReadWriteIOUtils.getByteBuffer(minValue);
}
@Override
- public ByteBuffer getMaxBytebuffer() {
- return ReadWriteIOUtils.getByteBuffer(max);
+ public ByteBuffer getMaxValueBuffer() {
+ return ReadWriteIOUtils.getByteBuffer(maxValue);
}
@Override
- public ByteBuffer getFirstBytebuffer() {
- return ReadWriteIOUtils.getByteBuffer(first);
+ public ByteBuffer getFirstValueBuffer() {
+ return ReadWriteIOUtils.getByteBuffer(firstValue);
}
@Override
- public ByteBuffer getLastBytebuffer() {
- return ReadWriteIOUtils.getByteBuffer(last);
+ public ByteBuffer getLastValueBuffer() {
+ return ReadWriteIOUtils.getByteBuffer(lastValue);
}
@Override
- public ByteBuffer getSumBytebuffer() {
- return ReadWriteIOUtils.getByteBuffer(sum);
+ public ByteBuffer getSumValueBuffer() {
+ return ReadWriteIOUtils.getByteBuffer(sumValue);
}
@Override
- public int sizeOfDatum() {
- return 4;
+ public int serializeStats(OutputStream outputStream) throws IOException {
+ int byteLen = 0;
+ byteLen += ReadWriteIOUtils.write(minValue, outputStream);
+ byteLen += ReadWriteIOUtils.write(maxValue, outputStream);
+ byteLen += ReadWriteIOUtils.write(firstValue, outputStream);
+ byteLen += ReadWriteIOUtils.write(lastValue, outputStream);
+ byteLen += ReadWriteIOUtils.write(sumValue, outputStream);
+ return byteLen;
}
@Override
- public String toString() {
- return "[min:" + min + ",max:" + max + ",first:" + first + ",last:" + last + ",sum:" + sum
- + "]";
+ void deserialize(InputStream inputStream) throws IOException {
+ this.minValue = ReadWriteIOUtils.readFloat(inputStream);
+ this.maxValue = ReadWriteIOUtils.readFloat(inputStream);
+ this.firstValue = ReadWriteIOUtils.readFloat(inputStream);
+ this.lastValue = ReadWriteIOUtils.readFloat(inputStream);
+ this.sumValue = ReadWriteIOUtils.readDouble(inputStream);
}
@Override
- void deserialize(InputStream inputStream) throws IOException {
- this.min = ReadWriteIOUtils.readFloat(inputStream);
- this.max = ReadWriteIOUtils.readFloat(inputStream);
- this.first = ReadWriteIOUtils.readFloat(inputStream);
- this.last = ReadWriteIOUtils.readFloat(inputStream);
- this.sum = ReadWriteIOUtils.readDouble(inputStream);
+ void deserialize(ByteBuffer byteBuffer) {
+ this.minValue = ReadWriteIOUtils.readFloat(byteBuffer);
+ this.maxValue = ReadWriteIOUtils.readFloat(byteBuffer);
+ this.firstValue = ReadWriteIOUtils.readFloat(byteBuffer);
+ this.lastValue = ReadWriteIOUtils.readFloat(byteBuffer);
+ this.sumValue = ReadWriteIOUtils.readDouble(byteBuffer);
}
@Override
- void deserialize(ByteBuffer byteBuffer) throws IOException {
- this.min = ReadWriteIOUtils.readFloat(byteBuffer);
- this.max = ReadWriteIOUtils.readFloat(byteBuffer);
- this.first = ReadWriteIOUtils.readFloat(byteBuffer);
- this.last = ReadWriteIOUtils.readFloat(byteBuffer);
- this.sum = ReadWriteIOUtils.readDouble(byteBuffer);
+ public String toString() {
+ return "[minValue:" + minValue + ",maxValue:" + maxValue + ",firstValue:" + firstValue +
+ ",lastValue:" + lastValue + ",sumValue:" + sumValue + "]";
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatistics.java
index f2ab8cf..3bf7a5d 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatistics.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatistics.java
@@ -20,7 +20,9 @@ package org.apache.iotdb.tsfile.file.metadata.statistics;
import java.io.IOException;
import java.io.InputStream;
+import java.io.OutputStream;
import java.nio.ByteBuffer;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.utils.BytesUtils;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
@@ -29,20 +31,50 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
*/
public class IntegerStatistics extends Statistics<Integer> {
- private int min;
- private int max;
- private int first;
- private int last;
- private double sum;
+ private int minValue;
+ private int maxValue;
+ private int firstValue;
+ private int lastValue;
+ private double sumValue;
+
+ @Override
+ public TSDataType getType() {
+ return TSDataType.INT32;
+ }
+
+ @Override
+ public int getStatsSize() {
+ return 24;
+ }
+
+ private void initializeStats(int min, int max, int first, int last, double sum) {
+ this.minValue = min;
+ this.maxValue = max;
+ this.firstValue = first;
+ this.lastValue = last;
+ this.sumValue = sum;
+ }
+
+ private void updateStats(int minValue, int maxValue, int firstValue, int lastValue,
+ double sumValue) {
+ if (minValue < this.minValue) {
+ this.minValue = minValue;
+ }
+ if (maxValue > this.maxValue) {
+ this.maxValue = maxValue;
+ }
+ this.sumValue += sumValue;
+ this.lastValue = lastValue;
+ }
@Override
public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) {
- min = BytesUtils.bytesToInt(minBytes);
- max = BytesUtils.bytesToInt(maxBytes);
+ minValue = BytesUtils.bytesToInt(minBytes);
+ maxValue = BytesUtils.bytesToInt(maxBytes);
}
@Override
- public void updateStats(int value) {
+ void updateStats(int value) {
if (isEmpty) {
initializeStats(value, value, value, value, value);
isEmpty = false;
@@ -53,154 +85,133 @@ public class IntegerStatistics extends Statistics<Integer> {
}
@Override
- public void updateStats(int[] values) {
- for (int value : values) {
- if (isEmpty) {
- initializeStats(value, value, value, value, value);
- isEmpty = false;
- } else {
- updateStats(value, value, value, value, value);
- isEmpty = false;
- }
- }
- }
-
- private void updateStats(int minValue, int maxValue, int firstValue, int lastValue,
- double sumValue) {
- // TODO: unused parameter
- if (minValue < min) {
- min = minValue;
- }
- if (maxValue > max) {
- max = maxValue;
+ void updateStats(int[] values, int batchSize) {
+ for (int i = 0; i < batchSize; i++) {
+ updateStats(values[i]);
}
- sum += sumValue;
- this.last = lastValue;
}
@Override
- public Integer getMin() {
- return min;
+ public Integer getMinValue() {
+ return minValue;
}
@Override
- public Integer getMax() {
- return max;
+ public Integer getMaxValue() {
+ return maxValue;
}
@Override
- public Integer getFirst() {
- return first;
+ public Integer getFirstValue() {
+ return firstValue;
}
@Override
- public Integer getLast() {
- return last;
+ public Integer getLastValue() {
+ return lastValue;
}
@Override
- public double getSum() {
- return sum;
+ public double getSumValue() {
+ return sumValue;
}
@Override
- protected void mergeStatisticsValue(Statistics<?> stats) {
+ protected void mergeStatisticsValue(Statistics stats) {
IntegerStatistics intStats = (IntegerStatistics) stats;
if (isEmpty) {
- initializeStats(intStats.getMin(), intStats.getMax(), intStats.getFirst(), intStats.getLast(),
- intStats.getSum());
+ initializeStats(intStats.getMinValue(), intStats.getMaxValue(), intStats.getFirstValue(), intStats.getLastValue(),
+ intStats.getSumValue());
isEmpty = false;
} else {
- updateStats(intStats.getMin(), intStats.getMax(), intStats.getFirst(), intStats.getLast(),
- intStats.getSum());
+ updateStats(intStats.getMinValue(), intStats.getMaxValue(), intStats.getFirstValue(), intStats.getLastValue(),
+ intStats.getSumValue());
}
}
- private void initializeStats(int min, int max, int first, int last, double sum) {
- this.min = min;
- this.max = max;
- this.first = first;
- this.last = last;
- this.sum = sum;
- }
-
@Override
- public ByteBuffer getMinBytebuffer() {
- return ReadWriteIOUtils.getByteBuffer(min);
+ public ByteBuffer getMinValueBuffer() {
+ return ReadWriteIOUtils.getByteBuffer(minValue);
}
@Override
- public ByteBuffer getMaxBytebuffer() {
- return ReadWriteIOUtils.getByteBuffer(max);
+ public ByteBuffer getMaxValueBuffer() {
+ return ReadWriteIOUtils.getByteBuffer(maxValue);
}
@Override
- public ByteBuffer getFirstBytebuffer() {
- return ReadWriteIOUtils.getByteBuffer(first);
+ public ByteBuffer getFirstValueBuffer() {
+ return ReadWriteIOUtils.getByteBuffer(firstValue);
}
@Override
- public ByteBuffer getLastBytebuffer() {
- return ReadWriteIOUtils.getByteBuffer(last);
+ public ByteBuffer getLastValueBuffer() {
+ return ReadWriteIOUtils.getByteBuffer(lastValue);
}
@Override
- public ByteBuffer getSumBytebuffer() {
- return ReadWriteIOUtils.getByteBuffer(sum);
+ public ByteBuffer getSumValueBuffer() {
+ return ReadWriteIOUtils.getByteBuffer(sumValue);
}
@Override
- public byte[] getMinBytes() {
- return BytesUtils.intToBytes(min);
+ public byte[] getMinValueBytes() {
+ return BytesUtils.intToBytes(minValue);
}
@Override
- public byte[] getMaxBytes() {
- return BytesUtils.intToBytes(max);
+ public byte[] getMaxValueBytes() {
+ return BytesUtils.intToBytes(maxValue);
}
@Override
- public byte[] getFirstBytes() {
- return BytesUtils.intToBytes(first);
+ public byte[] getFirstValueBytes() {
+ return BytesUtils.intToBytes(firstValue);
}
@Override
- public byte[] getLastBytes() {
- return BytesUtils.intToBytes(last);
+ public byte[] getLastValueBytes() {
+ return BytesUtils.intToBytes(lastValue);
}
@Override
- public byte[] getSumBytes() {
- return BytesUtils.doubleToBytes(sum);
+ public byte[] getSumValueBytes() {
+ return BytesUtils.doubleToBytes(sumValue);
}
@Override
- public int sizeOfDatum() {
- return 4;
+ public int serializeStats(OutputStream outputStream) throws IOException {
+ int byteLen = 0;
+ byteLen += ReadWriteIOUtils.write(minValue, outputStream);
+ byteLen += ReadWriteIOUtils.write(maxValue, outputStream);
+ byteLen += ReadWriteIOUtils.write(firstValue, outputStream);
+ byteLen += ReadWriteIOUtils.write(lastValue, outputStream);
+ byteLen += ReadWriteIOUtils.write(sumValue, outputStream);
+ return byteLen;
}
@Override
- public String toString() {
- return "[min:" + min + ",max:" + max + ",first:" + first + ",last:" + last + ",sum:" + sum
- + "]";
+ void deserialize(InputStream inputStream) throws IOException {
+ this.minValue = ReadWriteIOUtils.readInt(inputStream);
+ this.maxValue = ReadWriteIOUtils.readInt(inputStream);
+ this.firstValue = ReadWriteIOUtils.readInt(inputStream);
+ this.lastValue = ReadWriteIOUtils.readInt(inputStream);
+ this.sumValue = ReadWriteIOUtils.readDouble(inputStream);
}
@Override
- void deserialize(InputStream inputStream) throws IOException {
- this.min = ReadWriteIOUtils.readInt(inputStream);
- this.max = ReadWriteIOUtils.readInt(inputStream);
- this.first = ReadWriteIOUtils.readInt(inputStream);
- this.last = ReadWriteIOUtils.readInt(inputStream);
- this.sum = ReadWriteIOUtils.readDouble(inputStream);
+ void deserialize(ByteBuffer byteBuffer) {
+ this.minValue = ReadWriteIOUtils.readInt(byteBuffer);
+ this.maxValue = ReadWriteIOUtils.readInt(byteBuffer);
+ this.firstValue = ReadWriteIOUtils.readInt(byteBuffer);
+ this.lastValue = ReadWriteIOUtils.readInt(byteBuffer);
+ this.sumValue = ReadWriteIOUtils.readDouble(byteBuffer);
}
@Override
- void deserialize(ByteBuffer byteBuffer) throws IOException {
- this.min = ReadWriteIOUtils.readInt(byteBuffer);
- this.max = ReadWriteIOUtils.readInt(byteBuffer);
- this.first = ReadWriteIOUtils.readInt(byteBuffer);
- this.last = ReadWriteIOUtils.readInt(byteBuffer);
- this.sum = ReadWriteIOUtils.readDouble(byteBuffer);
+ public String toString() {
+ return "[minValue:" + minValue + ",maxValue:" + maxValue + ",firstValue:" + firstValue +
+ ",lastValue:" + lastValue + ",sumValue:" + sumValue + "]";
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatistics.java
index 0e4ace5..5741715 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatistics.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatistics.java
@@ -20,54 +20,83 @@ package org.apache.iotdb.tsfile.file.metadata.statistics;
import java.io.IOException;
import java.io.InputStream;
+import java.io.OutputStream;
import java.nio.ByteBuffer;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.utils.BytesUtils;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
-/**
- * Statistics for long type.
- */
public class LongStatistics extends Statistics<Long> {
- private long min;
- private long max;
- private long first;
- private long last;
- private double sum;
+ private long minValue;
+ private long maxValue;
+ private long firstValue;
+ private long lastValue;
+ private double sumValue;
+
+ @Override
+ public TSDataType getType() {
+ return TSDataType.INT64;
+ }
+
+ @Override
+ public int getStatsSize() {
+ return 40;
+ }
+
+ private void initializeStats(long min, long max, long firstValue, long last, double sum) {
+ this.minValue = min;
+ this.maxValue = max;
+ this.firstValue = firstValue;
+ this.lastValue = last;
+ this.sumValue += sum;
+ }
+
+ private void updateStats(long minValue, long maxValue, long firstValue, long lastValue,
+ double sumValue) {
+ if (minValue < this.minValue) {
+ this.minValue = minValue;
+ }
+ if (maxValue > this.maxValue) {
+ this.maxValue = maxValue;
+ }
+ this.sumValue += sumValue;
+ this.lastValue = lastValue;
+ }
@Override
public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) {
- min = BytesUtils.bytesToLong(minBytes);
- max = BytesUtils.bytesToLong(maxBytes);
+ minValue = BytesUtils.bytesToLong(minBytes);
+ maxValue = BytesUtils.bytesToLong(maxBytes);
}
@Override
- public Long getMin() {
- return min;
+ public Long getMinValue() {
+ return minValue;
}
@Override
- public Long getMax() {
- return max;
+ public Long getMaxValue() {
+ return maxValue;
}
@Override
- public Long getFirst() {
- return first;
+ public Long getFirstValue() {
+ return firstValue;
}
@Override
- public Long getLast() {
- return last;
+ public Long getLastValue() {
+ return lastValue;
}
@Override
- public double getSum() {
- return sum;
+ public double getSumValue() {
+ return sumValue;
}
@Override
- public void updateStats(long value) {
+ void updateStats(long value) {
if (isEmpty) {
initializeStats(value, value, value, value, value);
isEmpty = false;
@@ -77,138 +106,118 @@ public class LongStatistics extends Statistics<Long> {
}
@Override
- public void updateStats(long[] values) {
- for (long value : values) {
- if (isEmpty) {
- initializeStats(value, value, value, value, value);
- isEmpty = false;
- } else {
- updateStats(value, value, value, value, value);
- }
- }
- }
-
- private void updateStats(long minValue, long maxValue, long firstValue, long lastValue,
- double sumValue) {
- if (minValue < min) {
- min = minValue;
+ void updateStats(long[] values, int batchSize) {
+ for (int i = 0; i < batchSize; i++) {
+ updateStats(values[i]);
}
- if (maxValue > max) {
- max = maxValue;
- }
- sum += sumValue;
- this.last = lastValue;
}
@Override
public void updateStats(long minValue, long maxValue) {
- if (minValue < min) {
- min = minValue;
+ if (minValue < this.minValue) {
+ this.minValue = minValue;
}
- if (maxValue > max) {
- max = maxValue;
+ if (maxValue > this.maxValue) {
+ this.maxValue = maxValue;
}
}
@Override
- protected void mergeStatisticsValue(Statistics<?> stats) {
+ protected void mergeStatisticsValue(Statistics stats) {
LongStatistics longStats = (LongStatistics) stats;
if (isEmpty) {
- initializeStats(longStats.getMin(), longStats.getMax(), longStats.getFirst(),
- longStats.getLast(), longStats.getSum());
+ initializeStats(longStats.getMinValue(), longStats.getMaxValue(), longStats.getFirstValue(),
+ longStats.getLastValue(), longStats.getSumValue());
isEmpty = false;
} else {
- updateStats(longStats.getMin(), longStats.getMax(), longStats.getFirst(), longStats.getLast(),
- longStats.getSum());
+ updateStats(longStats.getMinValue(), longStats.getMaxValue(), longStats.getFirstValue(), longStats.getLastValue(),
+ longStats.getSumValue());
}
}
- private void initializeStats(long min, long max, long firstValue, long last, double sum) {
- this.min = min;
- this.max = max;
- this.first = firstValue;
- this.last = last;
- this.sum += sum;
- }
-
@Override
- public byte[] getMinBytes() {
- return BytesUtils.longToBytes(min);
+ public byte[] getMinValueBytes() {
+ return BytesUtils.longToBytes(minValue);
}
@Override
- public byte[] getMaxBytes() {
- return BytesUtils.longToBytes(max);
+ public byte[] getMaxValueBytes() {
+ return BytesUtils.longToBytes(maxValue);
}
@Override
- public byte[] getFirstBytes() {
- return BytesUtils.longToBytes(first);
+ public byte[] getFirstValueBytes() {
+ return BytesUtils.longToBytes(firstValue);
}
@Override
- public byte[] getLastBytes() {
- return BytesUtils.longToBytes(last);
+ public byte[] getLastValueBytes() {
+ return BytesUtils.longToBytes(lastValue);
}
@Override
- public byte[] getSumBytes() {
- return BytesUtils.doubleToBytes(sum);
+ public byte[] getSumValueBytes() {
+ return BytesUtils.doubleToBytes(sumValue);
}
@Override
- public ByteBuffer getMinBytebuffer() {
- return ReadWriteIOUtils.getByteBuffer(min);
+ public ByteBuffer getMinValueBuffer() {
+ return ReadWriteIOUtils.getByteBuffer(minValue);
}
@Override
- public ByteBuffer getMaxBytebuffer() {
- return ReadWriteIOUtils.getByteBuffer(max);
+ public ByteBuffer getMaxValueBuffer() {
+ return ReadWriteIOUtils.getByteBuffer(maxValue);
}
@Override
- public ByteBuffer getFirstBytebuffer() {
- return ReadWriteIOUtils.getByteBuffer(first);
+ public ByteBuffer getFirstValueBuffer() {
+ return ReadWriteIOUtils.getByteBuffer(firstValue);
}
@Override
- public ByteBuffer getLastBytebuffer() {
- return ReadWriteIOUtils.getByteBuffer(last);
+ public ByteBuffer getLastValueBuffer() {
+ return ReadWriteIOUtils.getByteBuffer(lastValue);
}
@Override
- public ByteBuffer getSumBytebuffer() {
- return ReadWriteIOUtils.getByteBuffer(sum);
+ public ByteBuffer getSumValueBuffer() {
+ return ReadWriteIOUtils.getByteBuffer(sumValue);
}
@Override
- public String toString() {
- return "[min:" + min + ",max:" + max + ",first:" + first + ",last:" + last + ",sum:" + sum
- + "]";
+ public int serializeStats(OutputStream outputStream) throws IOException {
+ int byteLen = 0;
+ byteLen += ReadWriteIOUtils.write(minValue, outputStream);
+ byteLen += ReadWriteIOUtils.write(maxValue, outputStream);
+ byteLen += ReadWriteIOUtils.write(firstValue, outputStream);
+ byteLen += ReadWriteIOUtils.write(lastValue, outputStream);
+ byteLen += ReadWriteIOUtils.write(sumValue, outputStream);
+ return byteLen;
}
@Override
- public int sizeOfDatum() {
- return 8;
+ void deserialize(InputStream inputStream) throws IOException {
+ this.minValue = ReadWriteIOUtils.readLong(inputStream);
+ this.maxValue = ReadWriteIOUtils.readLong(inputStream);
+ this.firstValue = ReadWriteIOUtils.readLong(inputStream);
+ this.lastValue = ReadWriteIOUtils.readLong(inputStream);
+ this.sumValue = ReadWriteIOUtils.readDouble(inputStream);
}
@Override
- void deserialize(InputStream inputStream) throws IOException {
- this.min = ReadWriteIOUtils.readLong(inputStream);
- this.max = ReadWriteIOUtils.readLong(inputStream);
- this.first = ReadWriteIOUtils.readLong(inputStream);
- this.last = ReadWriteIOUtils.readLong(inputStream);
- this.sum = ReadWriteIOUtils.readDouble(inputStream);
+ void deserialize(ByteBuffer byteBuffer) {
+ this.minValue = ReadWriteIOUtils.readLong(byteBuffer);
+ this.maxValue = ReadWriteIOUtils.readLong(byteBuffer);
+ this.firstValue = ReadWriteIOUtils.readLong(byteBuffer);
+ this.lastValue = ReadWriteIOUtils.readLong(byteBuffer);
+ this.sumValue = ReadWriteIOUtils.readDouble(byteBuffer);
}
@Override
- void deserialize(ByteBuffer byteBuffer) throws IOException {
- this.min = ReadWriteIOUtils.readLong(byteBuffer);
- this.max = ReadWriteIOUtils.readLong(byteBuffer);
- this.first = ReadWriteIOUtils.readLong(byteBuffer);
- this.last = ReadWriteIOUtils.readLong(byteBuffer);
- this.sum = ReadWriteIOUtils.readDouble(byteBuffer);
+ public String toString() {
+ return "[minValue:" + minValue + ",maxValue:" + maxValue + ",firstValue:" + firstValue +
+ ",lastValue:" + lastValue + ",sumValue:" + sumValue + "]";
}
-
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/NoStatistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/NoStatistics.java
deleted file mode 100644
index 9f66ba4..0000000
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/NoStatistics.java
+++ /dev/null
@@ -1,181 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.tsfile.file.metadata.statistics;
-
-import java.io.InputStream;
-import java.nio.ByteBuffer;
-import org.apache.iotdb.tsfile.utils.Binary;
-import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
-
-/**
- * This statistic is used as Unsupported data type. It just return a 0-byte array while asked max or
- * min.
- */
-public class NoStatistics extends Statistics<Long> {
-
- @Override
- public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) {
- // NoStatistics does not make any statistics
- }
-
- @Override
- public Long getMin() {
- return null;
- }
-
- @Override
- public Long getMax() {
- return null;
- }
-
- @Override
- public void updateStats(boolean value) {
- // NoStatistics does not make any statistics
- }
-
- @Override
- public void updateStats(int value) {
- // NoStatistics does not make any statistics
- }
-
- @Override
- public void updateStats(long value) {
- // NoStatistics does not make any statistics
- }
-
- @Override
- public void updateStats(Binary value) {
- // NoStatistics does not make any statistics
- }
-
- @Override
- public void updateStats(boolean[] values) {
- // NoStatistics does not make any statistics
- }
-
- @Override
- public void updateStats(int[] values) {
- // NoStatistics does not make any statistics
- }
-
- @Override
- public void updateStats(long[] values) {
- // NoStatistics does not make any statistics
- }
-
- @Override
- public void updateStats(Binary[] values) {
- // NoStatistics does not make any statistics
- }
-
- @Override
- public void updateStats(long min, long max) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- protected void mergeStatisticsValue(Statistics<?> stats) {
- // NoStatistics does not make any statistics
- }
-
- @Override
- public byte[] getMaxBytes() {
- return new byte[0];
- }
-
- @Override
- public byte[] getMinBytes() {
- return new byte[0];
- }
-
- @Override
- public String toString() {
- return "no stats";
- }
-
- @Override
- public Long getFirst() {
- return null;
- }
-
- @Override
- public double getSum() {
- return 0;
- }
-
- @Override
- public Long getLast() {
- return null;
- }
-
- @Override
- public byte[] getFirstBytes() {
- return new byte[0];
- }
-
- @Override
- public byte[] getSumBytes() {
- return new byte[0];
- }
-
- @Override
- public byte[] getLastBytes() {
- return new byte[0];
- }
-
- @Override
- public int sizeOfDatum() {
- return 0;
- }
-
- @Override
- public ByteBuffer getMaxBytebuffer() {
- return ReadWriteIOUtils.getByteBuffer(0);
- }
-
- @Override
- public ByteBuffer getMinBytebuffer() {
- return ReadWriteIOUtils.getByteBuffer(0);
- }
-
- @Override
- public ByteBuffer getFirstBytebuffer() {
- return ReadWriteIOUtils.getByteBuffer(0);
- }
-
- @Override
- public ByteBuffer getSumBytebuffer() {
- return ReadWriteIOUtils.getByteBuffer(0);
- }
-
- @Override
- public ByteBuffer getLastBytebuffer() {
- return ReadWriteIOUtils.getByteBuffer(0);
- }
-
- @Override
- void deserialize(InputStream inputStream) {
- // NoStatistics does not make any statistics
- }
-
- @Override
- void deserialize(ByteBuffer byteBuffer) {
- // NoStatistics does not make any statistics
- }
-}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java
index c7d5230..5ece330 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java
@@ -21,15 +21,9 @@ package org.apache.iotdb.tsfile.file.metadata.statistics;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
-import java.math.BigDecimal;
import java.nio.ByteBuffer;
-import java.util.Arrays;
-
-import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
-import org.apache.iotdb.tsfile.encoding.common.EndianType;
import org.apache.iotdb.tsfile.exception.write.UnknownColumnTypeException;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.read.reader.TsFileInput;
import org.apache.iotdb.tsfile.utils.Binary;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
import org.slf4j.Logger;
@@ -37,7 +31,7 @@ import org.slf4j.LoggerFactory;
/**
* This class is used for recording statistic information of each measurement in a delta file. While
- * writing processing, the processor records the digest information. Statistics includes maximum,
+ * writing processing, the processor records the statistics information. Statistics includes maximum,
* minimum and null value count up to version 0.0.1.<br> Each data type extends this Statistic as
* super class.<br>
*
@@ -51,15 +45,13 @@ public abstract class Statistics<T> {
*/
protected boolean isEmpty = true;
- private ByteBuffer[] statistics;
-
/**
- * size of valid values in statistics. Note that some values in statistics can be null and thus
- * invalid.
+ * number of time-value points
*/
- private int validSizeOfArray = 0;
+ private long count = 0;
- private int digestSerializedSize = Integer.BYTES; // initialize for number of statistics
+ private long startTime = Long.MAX_VALUE;
+ private long endTime = Long.MIN_VALUE;
/**
* static method providing statistic instance for respective data type.
@@ -86,63 +78,68 @@ public abstract class Statistics<T> {
}
}
- public static Statistics deserialize(InputStream inputStream, TSDataType dataType)
- throws IOException {
- Statistics statistics = getStatsByType(dataType);
- statistics.deserialize(inputStream);
- statistics.isEmpty = false;
- return statistics;
- }
+ public abstract TSDataType getType();
- public static Statistics deserialize(ByteBuffer buffer, TSDataType dataType) throws IOException {
- Statistics statistics = getStatsByType(dataType);
- statistics.deserialize(buffer);
- statistics.isEmpty = false;
- return statistics;
+ public int getSerializedSize() {
+ return 24 // count, startTime, endTime
+ + getStatsSize();
}
- public static Statistics deserialize(TsFileInput input, long offset, TSDataType dataType)
- throws IOException {
- Statistics statistics = getStatsByType(dataType);
- statistics.deserialize(input, offset);
- statistics.isEmpty = false;
- return statistics;
+ public abstract int getStatsSize();
+
+ public int serialize(OutputStream outputStream) throws IOException{
+ int byteLen = 0;
+ byteLen += ReadWriteIOUtils.write(count, outputStream);
+ byteLen += ReadWriteIOUtils.write(startTime, outputStream);
+ byteLen += ReadWriteIOUtils.write(endTime, outputStream);
+ // value statistics of different data type
+ byteLen += serializeStats(outputStream);
+ return byteLen;
}
+ abstract int serializeStats(OutputStream outputStream) throws IOException;
+
+ /**
+ * read data from the inputStream.
+ */
+ abstract void deserialize(InputStream inputStream) throws IOException;
+
+ abstract void deserialize(ByteBuffer byteBuffer);
+
public abstract void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes);
- public abstract T getMin();
+ public abstract T getMinValue();
- public abstract T getMax();
+ public abstract T getMaxValue();
- public abstract T getFirst();
+ public abstract T getFirstValue();
- public abstract T getLast();
+ public abstract T getLastValue();
- public abstract double getSum();
+ public abstract double getSumValue();
- public abstract byte[] getMinBytes();
+ public abstract byte[] getMinValueBytes();
- public abstract byte[] getMaxBytes();
+ public abstract byte[] getMaxValueBytes();
- public abstract byte[] getFirstBytes();
+ public abstract byte[] getFirstValueBytes();
- public abstract byte[] getLastBytes();
+ public abstract byte[] getLastValueBytes();
- public abstract byte[] getSumBytes();
+ public abstract byte[] getSumValueBytes();
- public abstract ByteBuffer getMinBytebuffer();
+ public abstract ByteBuffer getMinValueBuffer();
- public abstract ByteBuffer getMaxBytebuffer();
+ public abstract ByteBuffer getMaxValueBuffer();
- public abstract ByteBuffer getFirstBytebuffer();
+ public abstract ByteBuffer getFirstValueBuffer();
- public abstract ByteBuffer getLastBytebuffer();
+ public abstract ByteBuffer getLastValueBuffer();
- public abstract ByteBuffer getSumBytebuffer();
+ public abstract ByteBuffer getSumValueBuffer();
/**
- * merge parameter to this statistic. Including
+ * merge parameter to this statistic
*
* @param stats input statistics
* @throws StatisticsClassException cannot merge statistics
@@ -152,384 +149,269 @@ public abstract class Statistics<T> {
LOG.warn("tsfile-file parameter stats is null");
return;
}
- if (this.getClass() == stats.getClass()) {
- if (!stats.isEmpty) {
- mergeStatisticsValue(stats);
- isEmpty = false;
+ if (this.getClass() == stats.getClass() && !stats.isEmpty) {
+ if (stats.startTime < this.startTime) {
+ this.startTime = stats.startTime;
+ }
+ if (stats.endTime > this.endTime) {
+ this.endTime = stats.endTime;
}
+ // must be sure no overlap between two statistics
+ this.count += stats.count;
+ mergeStatisticsValue(stats);
+ isEmpty = false;
} else {
String thisClass = this.getClass().toString();
String statsClass = stats.getClass().toString();
- LOG.warn("tsfile-file Statistics classes mismatched,no merge: {} v.s. {}",
+ LOG.warn("Statistics classes mismatched,no merge: {} v.s. {}",
thisClass, statsClass);
throw new StatisticsClassException(this.getClass(), stats.getClass());
}
}
- protected abstract void mergeStatisticsValue(Statistics<?> stats);
+ public void update(long time, boolean value) {
+ if (time < this.startTime) {
+ startTime = time;
+ }
+ if (time > this.endTime) {
+ endTime = time;
+ }
+ count++;
+ updateStats(value);
+ }
- public boolean isEmpty() {
- return isEmpty;
+ public void update(long time, int value) {
+ if (time < this.startTime) {
+ startTime = time;
+ }
+ if (time > this.endTime) {
+ endTime = time;
+ }
+ count++;
+ updateStats(value);
}
- public void setEmpty(boolean empty) {
- isEmpty = empty;
+ public void update(long time, long value) {
+ if (time < this.startTime) {
+ startTime = time;
+ }
+ if (time > this.endTime) {
+ endTime = time;
+ }
+ count++;
+ updateStats(value);
}
- public void updateStats(boolean value) {
- throw new UnsupportedOperationException();
+ public void update(long time, float value) {
+ if (time < this.startTime) {
+ startTime = time;
+ }
+ if (time > this.endTime) {
+ endTime = time;
+ }
+ count++;
+ updateStats(value);
}
- public void updateStats(int value) {
- throw new UnsupportedOperationException();
+ public void update(long time, double value) {
+ if (time < this.startTime) {
+ startTime = time;
+ } else if (time > this.endTime) {
+ endTime = time;
+ }
+ count++;
+ updateStats(value);
}
- public void updateStats(long value) {
- throw new UnsupportedOperationException();
+ public void update(long time, Binary value) {
+ if (time < startTime) {
+ startTime = time;
+ } else if (time > endTime) {
+ endTime = time;
+ }
+ count++;
+ updateStats(value);
}
- /**
- * This method with two parameters is only used by {@code unsequence} which
- * updates/inserts/deletes timestamp.
- *
- * @param min min timestamp
- * @param max max timestamp
- */
- public void updateStats(long min, long max) {
- throw new UnsupportedOperationException();
+ public void update(long[] time, boolean[] values, int batchSize) {
+ if (time[0] < startTime) {
+ startTime = time[0];
+ }
+ if (time[batchSize-1] > this.endTime) {
+ endTime = time[batchSize-1];
+ }
+ count += batchSize;
+ updateStats(values, batchSize);
}
- public void updateStats(float value) {
- throw new UnsupportedOperationException();
+ public void update(long[] time, int[] values, int batchSize) {
+ if (time[0] < startTime) {
+ startTime = time[0];
+ }
+ if (time[batchSize-1] > this.endTime) {
+ endTime = time[batchSize-1];
+ }
+ count += batchSize;
+ updateStats(values, batchSize);
}
- public void updateStats(double value) {
- throw new UnsupportedOperationException();
+ public void update(long[] time, long[] values, int batchSize) {
+ if (time[0] < startTime) {
+ startTime = time[0];
+ }
+ if (time[batchSize-1] > this.endTime) {
+ endTime = time[batchSize-1];
+ }
+ count += batchSize;
+ updateStats(values, batchSize);
}
- public void updateStats(BigDecimal value) {
- throw new UnsupportedOperationException();
+ public void update(long[] time, float[] values, int batchSize) {
+ if (time[0] < startTime) {
+ startTime = time[0];
+ }
+ if (time[batchSize-1] > this.endTime) {
+ endTime = time[batchSize-1];
+ }
+ count += batchSize;
+ updateStats(values, batchSize);
}
- public void updateStats(Binary value) {
- throw new UnsupportedOperationException();
+ public void update(long[] time, double[] values, int batchSize) {
+ if (time[0] < startTime) {
+ startTime = time[0];
+ }
+ if (time[batchSize-1] > this.endTime) {
+ endTime = time[batchSize-1];
+ }
+ count += batchSize;
+ updateStats(values, batchSize);
}
- public void updateStats(boolean[] values) {
- throw new UnsupportedOperationException();
+ public void update(long[] time, Binary[] values, int batchSize) {
+ if (time[0] < startTime) {
+ startTime = time[0];
+ }
+ if (time[batchSize-1] > this.endTime) {
+ endTime = time[batchSize-1];
+ }
+ count += batchSize;
+ updateStats(values, batchSize);
}
- public void updateStats(int[] values) {
- throw new UnsupportedOperationException();
+ protected abstract void mergeStatisticsValue(Statistics stats);
+
+ public boolean isEmpty() {
+ return isEmpty;
}
- public void updateStats(long[] values) {
- throw new UnsupportedOperationException();
+ public void setEmpty(boolean empty) {
+ isEmpty = empty;
}
- public void updateStats(float[] values) {
+ void updateStats(boolean value) {
throw new UnsupportedOperationException();
}
- public void updateStats(double[] values) {
+ void updateStats(int value) {
throw new UnsupportedOperationException();
}
- public void updateStats(BigDecimal[] values) {
+ void updateStats(long value) {
throw new UnsupportedOperationException();
}
- public void updateStats(Binary[] values) {
+ void updateStats(float value) {
throw new UnsupportedOperationException();
}
- public void reset() {
+ void updateStats(double value) {
+ throw new UnsupportedOperationException();
}
- /**
- * @return the size of one field of this class.<br> int, float - 4<br> double, long, bigDecimal -
- * 8 <br> boolean - 1 <br> No - 0 <br> binary - -1 which means uncertainty </>
- */
- public abstract int sizeOfDatum();
-
- /**
- * read data from the inputStream.
- */
- abstract void deserialize(InputStream inputStream) throws IOException;
-
- abstract void deserialize(ByteBuffer byteBuffer) throws IOException;
-
- protected void deserialize(TsFileInput input, long offset) throws IOException {
- int size = getSerializedSize();
- ByteBuffer buffer = ByteBuffer.allocate(size);
- ReadWriteIOUtils.readAsPossible(input, offset, buffer);
- buffer.flip();
- deserialize(buffer);
+ void updateStats(Binary value) {
+ throw new UnsupportedOperationException();
}
- public int getSerializedSize() {
- if (sizeOfDatum() == 0) {
- return 0;
- } else if (sizeOfDatum() != -1) {
- return sizeOfDatum() * 4 + 8;
- } else {
- return 4 * Integer.BYTES + getMinBytes().length + getMaxBytes().length
- + getFirstBytes().length
- + getLastBytes().length + getSumBytes().length;
- }
+ void updateStats(boolean[] values, int batchSize) {
+ throw new UnsupportedOperationException();
}
- public int serialize(OutputStream outputStream) throws IOException {
- int length = 0;
- if (sizeOfDatum() == 0) {
- return 0;
- } else if (sizeOfDatum() != -1) {
- length = sizeOfDatum() * 4 + 8;
- outputStream.write(getMinBytes());
- outputStream.write(getMaxBytes());
- outputStream.write(getFirstBytes());
- outputStream.write(getLastBytes());
- outputStream.write(getSumBytes());
- } else {
- byte[] tmp = getMinBytes();
- length += tmp.length;
- length += ReadWriteIOUtils.write(tmp.length, outputStream);
- outputStream.write(tmp);
- tmp = getMaxBytes();
- length += tmp.length;
- length += ReadWriteIOUtils.write(tmp.length, outputStream);
- outputStream.write(tmp);
- tmp = getFirstBytes();
- length += tmp.length;
- length += ReadWriteIOUtils.write(tmp.length, outputStream);
- outputStream.write(tmp);
- tmp = getLastBytes();
- length += tmp.length;
- length += ReadWriteIOUtils.write(tmp.length, outputStream);
- outputStream.write(tmp);
- outputStream.write(getSumBytes());
- length += 8;
- }
- return length;
+ void updateStats(int[] values, int batchSize) {
+ throw new UnsupportedOperationException();
}
- public static int getNullDigestSize() {
- return Integer.BYTES;
+ void updateStats(long[] values, int batchSize) {
+ throw new UnsupportedOperationException();
}
- public static int serializeNullTo(OutputStream outputStream) throws IOException {
- return ReadWriteIOUtils.write(0, outputStream);
+ void updateStats(float[] values, int batchSize) {
+ throw new UnsupportedOperationException();
}
- public static int serializeNullTo(ByteBuffer buffer) {
- return ReadWriteIOUtils.write(0, buffer);
+ void updateStats(double[] values, int batchSize) {
+ throw new UnsupportedOperationException();
}
- /**
- * use given input stream to deserialize.
- *
- * @param inputStream -given input stream
- * @return -an instance of TsDigest
- */
- public static Statistics deserializeFrom(InputStream inputStream, TSDataType dataType) throws IOException {
- Statistics digest = getStatsByType(dataType);
- int size = ReadWriteIOUtils.readInt(inputStream);
- digest.validSizeOfArray = size;
- digest.digestSerializedSize = Integer.BYTES;
- if (size > 0) {
- digest.statistics = new ByteBuffer[StatisticType.getTotalTypeNum()];
- ByteBuffer value;
- // check if it's an old version of TsFile
- String key = "";
- if (TSFileDescriptor.getInstance().getConfig().getEndian().equals(EndianType.LITTLE_ENDIAN.toString())) {
- for (int i = 0; i < size; i++) {
- key = ReadWriteIOUtils.readString(inputStream);
- value = ReadWriteIOUtils.readByteBufferWithSelfDescriptionLength(inputStream);
- short n;
- switch (key) {
- case "min_value":
- n = 0;
- break;
- case "max_value":
- n = 1;
- break;
- case "first":
- n = 2;
- break;
- case "last":
- n = 3;
- break;
- case "sum":
- n = 4;
- break;
- default:
- n = -1;
- }
- digest.statistics[n] = value;
- digest.digestSerializedSize += Short.BYTES + Integer.BYTES + value.remaining();
- }
- }
- else {
- for (int i = 0; i < size; i++) {
- short n = ReadWriteIOUtils.readShort(inputStream);
- value = ReadWriteIOUtils.readByteBufferWithSelfDescriptionLength(inputStream);
- digest.statistics[n] = value;
- digest.digestSerializedSize += Short.BYTES + Integer.BYTES + value.remaining();
- }
- }
- } // else left digest.statistics as null
- return digest;
+ void updateStats(Binary[] values, int batchSize) {
+ throw new UnsupportedOperationException();
}
/**
- * use given buffer to deserialize.
+ * This method with two parameters is only used by {@code unsequence} which
+ * updates/inserts/deletes timestamp.
*
- * @param buffer -given buffer
- * @return -an instance of TsDigest
+ * @param min min timestamp
+ * @param max max timestamp
*/
- public static Statistics deserializeFrom(ByteBuffer buffer, TSDataType dataType) {
- Statistics digest = getStatsByType(dataType);
- int size = ReadWriteIOUtils.readInt(buffer);
- digest.validSizeOfArray = size;
- digest.digestSerializedSize = Integer.BYTES;
- if (size > 0) {
- digest.statistics = new ByteBuffer[StatisticType.getTotalTypeNum()];
- ByteBuffer value;
- // check if it's old version of TsFile
- buffer.mark();
- String key = ReadWriteIOUtils.readString(buffer);
- if (key.equals("min_value") || key.equals("max_value") || key.equals("first")
- || key.equals("last") || key.equals("sum")) {
- buffer.reset();
- for (int i = 0; i < size; i++) {
- key = ReadWriteIOUtils.readString(buffer);
- value = ReadWriteIOUtils.readByteBufferWithSelfDescriptionLength(buffer);
- short n;
- switch (key) {
- case "min_value":
- n = 0;
- break;
- case "max_value":
- n = 1;
- break;
- case "first":
- n = 2;
- break;
- case "last":
- n = 3;
- break;
- case "sum":
- n = 4;
- break;
- default:
- n = -1;
- }
- digest.statistics[n] = value;
- digest.digestSerializedSize += Short.BYTES + Integer.BYTES + value.remaining();
- }
- }
- else {
- buffer.reset();
- for (int i = 0; i < size; i++) {
- short n = ReadWriteIOUtils.readShort(buffer);
- value = ReadWriteIOUtils.readByteBufferWithSelfDescriptionLength(buffer);
- digest.statistics[n] = value;
- digest.digestSerializedSize += Short.BYTES + Integer.BYTES + value.remaining();
- }
- }
- } // else left digest.statistics as null
+ public void updateStats(long min, long max) {
+ throw new UnsupportedOperationException();
+ }
- return digest;
+ public static Statistics deserialize(InputStream inputStream, TSDataType dataType)
+ throws IOException {
+ Statistics statistics = getStatsByType(dataType);
+ statistics.setCount(ReadWriteIOUtils.readLong(inputStream));
+ statistics.setStartTime(ReadWriteIOUtils.readLong(inputStream));
+ statistics.setEndTime(ReadWriteIOUtils.readLong(inputStream));
+ statistics.deserialize(inputStream);
+ statistics.isEmpty = false;
+ return statistics;
}
- private void reCalculate() {
- validSizeOfArray = 0;
- digestSerializedSize = Integer.BYTES;
- if (statistics != null) {
- for (ByteBuffer value : statistics) {
- if (value != null) {
- // StatisticType serialized value, byteBuffer.capacity and byteBuffer.array
- digestSerializedSize += Short.BYTES + Integer.BYTES + value.remaining();
- validSizeOfArray++;
- }
- }
- }
+ public static Statistics deserialize(ByteBuffer buffer, TSDataType dataType) {
+ Statistics statistics = getStatsByType(dataType);
+ statistics.setCount(ReadWriteIOUtils.readLong(buffer));
+ statistics.setStartTime(ReadWriteIOUtils.readLong(buffer));
+ statistics.setEndTime(ReadWriteIOUtils.readLong(buffer));
+ statistics.deserialize(buffer);
+ statistics.isEmpty = false;
+ return statistics;
}
- /**
- * get statistics of the current object.
- */
- public ByteBuffer[] getStatistics() {
- return statistics; //TODO unmodifiable
+ public long getStartTime() {
+ return startTime;
}
- public void setStatistics(ByteBuffer[] statistics) throws IOException {
- if (statistics != null && statistics.length != StatisticType.getTotalTypeNum()) {
- throw new IOException(String.format(
- "The length of array of statistics doesn't equal StatisticType.getTotalTypeNum() %d",
- StatisticType.getTotalTypeNum()));
- }
- this.statistics = statistics;
- reCalculate(); // DO NOT REMOVE THIS
+ public long getEndTime() {
+ return endTime;
}
- @Override
- public String toString() {
- return statistics != null ? Arrays.toString(statistics) : "";
+ public long getCount() {
+ return count;
}
- /**
- * use given outputStream to serialize.
- *
- * @param outputStream -given outputStream
- * @return -byte length
- */
- public int serializeTo(OutputStream outputStream) throws IOException {
- int byteLen = 0;
- if (validSizeOfArray == 0) {
- byteLen += ReadWriteIOUtils.write(0, outputStream);
- } else {
- byteLen += ReadWriteIOUtils.write(validSizeOfArray, outputStream);
- for (int i = 0; i < statistics.length; i++) {
- if (statistics[i] != null) {
- byteLen += ReadWriteIOUtils.write((short) i, outputStream);
- byteLen += ReadWriteIOUtils.write(statistics[i], outputStream);
- }
- }
- }
- return byteLen;
+ public void setStartTime(long startTime) {
+ this.startTime = startTime;
}
- /**
- * use given buffer to serialize.
- *
- * @param buffer -given buffer
- * @return -byte length
- */
- public int serializeTo(ByteBuffer buffer) {
- int byteLen = 0;
- if (validSizeOfArray == 0) {
- byteLen += ReadWriteIOUtils.write(0, buffer);
- } else {
- byteLen += ReadWriteIOUtils.write(validSizeOfArray, buffer);
- for (int i = 0; i < statistics.length; i++) {
- if (statistics[i] != null) {
- byteLen += ReadWriteIOUtils.write((short) i, buffer);
- byteLen += ReadWriteIOUtils.write(statistics[i], buffer);
- }
- }
- }
- return byteLen;
+ public void setEndTime(long endTime) {
+ this.endTime = endTime;
}
- /**
- * get the serializedSize of the current object.
- *
- * @return -serializedSize
- */
- public int getDigestSerializedSize() {
- return digestSerializedSize;
+ public void setCount(long count) {
+ this.count = count;
}
@Override
@@ -537,48 +419,7 @@ public abstract class Statistics<T> {
if (this == o) {
return true;
}
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- Statistics digest = (Statistics) o;
- if (digestSerializedSize != digest.digestSerializedSize || validSizeOfArray != digest.validSizeOfArray
- || ((statistics == null) ^ (digest.statistics == null))) {
- return false;
- }
-
- if (statistics != null) {
- for (int i = 0; i < statistics.length; i++) {
- if ((statistics[i] == null) ^ (digest.statistics[i] == null)) {
- // one is null and the other is not null
- return false;
- }
- if (statistics[i] != null) {
- if (!statistics[i].equals(digest.statistics[i])) {
- return false;
- }
- }
- }
- }
- return true;
+ return o != null && getClass() == o.getClass();
}
- public enum StatisticType {
- min_value, max_value, first_value, last_value, sum_value;
-
- public static int getTotalTypeNum() {
- return StatisticType.values().length;
- }
-
- public static StatisticType deserialize(short i) {
- return StatisticType.values()[i];
- }
-
- public static int getSerializedSize() {
- return Short.BYTES;
- }
-
- public short serialize() {
- return (short) this.ordinal();
- }
- }
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/StatisticsClassException.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/StatisticsClassException.java
index 9f7a3f6..778278a 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/StatisticsClassException.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/StatisticsClassException.java
@@ -27,4 +27,9 @@ public class StatisticsClassException extends TsFileRuntimeException {
public StatisticsClassException(Class<?> className1, Class<?> className2) {
super("tsfile-file Statistics classes mismatched: " + className1 + " vs. " + className2);
}
+
+ public StatisticsClassException(String message) {
+ super(message);
+ }
+
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index 7fcce1f..3b18933 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -28,7 +28,6 @@ import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Optional;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.compress.IUnCompressor;
@@ -285,23 +284,6 @@ public class TsFileSequenceReader implements AutoCloseable {
}
/**
- * @return get the position after the last chunk group in the file
- */
- public long getPositionOfFirstDeviceMetaIndex() throws IOException {
- TsFileMetaData metaData = readFileMetadata();
- Optional<Long> data = metaData.getDeviceMap().values().stream()
- .map(TsDeviceMetadataIndex::getOffset)
- .min(Comparator.comparing(Long::valueOf));
- if (data.isPresent()) {
- return data.get();
- } else {
- //no real data
- return TSFileConfig.MAGIC_STRING.getBytes().length + TSFileConfig.VERSION_NUMBER
- .getBytes().length;
- }
- }
-
- /**
* this function does not modify the position of the file reader.
*/
public TsDeviceMetadata readTsDeviceMetaData(TsDeviceMetadataIndex index) throws IOException {
@@ -313,8 +295,7 @@ public class TsFileSequenceReader implements AutoCloseable {
deviceMetadata = deviceMetadataMap.get(index);
}
if (deviceMetadata == null) {
- deviceMetadata = TsDeviceMetadata.deserializeFrom(readData(index.getOffset()
- , index.getLen()));
+ deviceMetadata = TsDeviceMetadata.deserializeFrom(readData(index.getOffset(), index.getLen()));
if (cacheDeviceMetadata) {
deviceMetadataMap.put(index, deviceMetadata);
}
@@ -435,18 +416,6 @@ public class TsFileSequenceReader implements AutoCloseable {
return PageHeader.deserializeFrom(tsFileInput.wrapAsInputStream(), type);
}
- /**
- * read the page's header.
- *
- * @param dataType given tsfile data type
- * @param position the file offset of this chunk's header
- * @param markerRead true if the offset does not contains the marker , otherwise false
- */
- private PageHeader readPageHeader(TSDataType dataType, long position, boolean markerRead)
- throws IOException {
- return PageHeader.deserializeFrom(dataType, tsFileInput, position, markerRead);
- }
-
public long position() throws IOException {
return tsFileInput.position();
}
@@ -459,14 +428,6 @@ public class TsFileSequenceReader implements AutoCloseable {
tsFileInput.position(tsFileInput.position() + header.getCompressedSize());
}
- /**
- *
- */
- public long skipPageData(PageHeader header, long position) throws IOException {
- return position + header.getCompressedSize();
- }
-
-
public ByteBuffer readPage(PageHeader header, CompressionType type) throws IOException {
return readPage(header, type, -1);
}
@@ -576,9 +537,6 @@ public class TsFileSequenceReader implements AutoCloseable {
String measurementID;
TSDataType dataType;
long fileOffsetOfChunk;
- long startTimeOfChunk = 0;
- long endTimeOfChunk = 0;
- long numOfPoints = 0;
ChunkGroupMetaData currentChunkGroup;
List<ChunkMetaData> chunks = null;
@@ -638,45 +596,23 @@ public class TsFileSequenceReader implements AutoCloseable {
Statistics<?> chunkStatistics = Statistics.getStatsByType(dataType);
if (header.getNumOfPages() > 0) {
PageHeader pageHeader = this.readPageHeader(header.getDataType());
- numOfPoints += pageHeader.getNumOfValues();
- startTimeOfChunk = pageHeader.getMinTimestamp();
- endTimeOfChunk = pageHeader.getMaxTimestamp();
chunkStatistics.mergeStatistics(pageHeader.getStatistics());
this.skipPageData(pageHeader);
}
for (int j = 1; j < header.getNumOfPages() - 1; j++) {
//a new Page
PageHeader pageHeader = this.readPageHeader(header.getDataType());
- numOfPoints += pageHeader.getNumOfValues();
chunkStatistics.mergeStatistics(pageHeader.getStatistics());
this.skipPageData(pageHeader);
}
if (header.getNumOfPages() > 1) {
PageHeader pageHeader = this.readPageHeader(header.getDataType());
- numOfPoints += pageHeader.getNumOfValues();
- endTimeOfChunk = pageHeader.getMaxTimestamp();
chunkStatistics.mergeStatistics(pageHeader.getStatistics());
this.skipPageData(pageHeader);
}
currentChunk = new ChunkMetaData(measurementID, dataType, fileOffsetOfChunk,
- startTimeOfChunk, endTimeOfChunk);
- currentChunk.setNumOfPoints(numOfPoints);
- ByteBuffer[] statisticsArray = new ByteBuffer[Statistics.StatisticType.getTotalTypeNum()];
- statisticsArray[Statistics.StatisticType.min_value.ordinal()] = ByteBuffer
- .wrap(chunkStatistics.getMinBytes());
- statisticsArray[Statistics.StatisticType.max_value.ordinal()] = ByteBuffer
- .wrap(chunkStatistics.getMaxBytes());
- statisticsArray[Statistics.StatisticType.first_value.ordinal()] = ByteBuffer
- .wrap(chunkStatistics.getFirstBytes());
- statisticsArray[Statistics.StatisticType.last_value.ordinal()] = ByteBuffer
- .wrap(chunkStatistics.getLastBytes());
- statisticsArray[Statistics.StatisticType.sum_value.ordinal()] = ByteBuffer
- .wrap(chunkStatistics.getSumBytes());
- Statistics tsDigest = Statistics.getStatsByType(dataType);
- tsDigest.setStatistics(statisticsArray);
- currentChunk.setDigest(tsDigest);
+ chunkStatistics);
chunks.add(currentChunk);
- numOfPoints = 0;
chunkCnt++;
break;
case MetaMarker.CHUNK_GROUP_FOOTER:
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/DigestForFilter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/DigestForFilter.java
deleted file mode 100755
index e4921d6..0000000
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/DigestForFilter.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.tsfile.read.filter;
-
-import java.nio.ByteBuffer;
-import org.apache.iotdb.tsfile.exception.filter.UnSupportFilterDataTypeException;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.utils.Binary;
-import org.apache.iotdb.tsfile.utils.BytesUtils;
-
-/**
- * class to construct digest.
- */
-public class DigestForFilter {
-
- private ByteBuffer minValue;
- private ByteBuffer maxValue;
- private long minTime;
- private long maxTime;
- private TSDataType type;
-
- /**
- * constructor of DigestForFilter with ByteBuffer minValue and maxValue.
- *
- * @param minTime min timestamp
- * @param maxTime max timestamp
- * @param minValue MUST be ByteBuffer
- * @param maxValue MUST be ByteBuffer
- * @param type time series data type
- */
- public DigestForFilter(long minTime, long maxTime, ByteBuffer minValue, ByteBuffer maxValue,
- TSDataType type) {
- this.minTime = minTime;
- this.maxTime = maxTime;
- this.minValue = minValue;
- this.maxValue = maxValue;
- this.type = type;
- }
-
- /**
- * constructor of DigestForFilter with byte[] minValue and maxValue.
- *
- * @param minTime min timestamp
- * @param maxTime max timestamp
- * @param minValue MUST be byte[]
- * @param maxValue MUST be byte[]
- * @param type time series data type
- */
- public DigestForFilter(long minTime, long maxTime, byte[] minValue, byte[] maxValue,
- TSDataType type) {
- this.minTime = minTime;
- this.maxTime = maxTime;
- this.minValue = ByteBuffer.wrap(minValue);
- this.maxValue = ByteBuffer.wrap(maxValue);
- this.type = type;
- }
-
- @SuppressWarnings("unchecked")
- private <T extends Comparable<T>> T getValue(ByteBuffer value) {
- switch (type) {
- case INT32:
- return (T) ((Integer) BytesUtils.bytesToInt(value.array()));
- case INT64:
- return (T) ((Long) BytesUtils.bytesToLong(value.array()));
- case FLOAT:
- return (T) ((Float) BytesUtils.bytesToFloat(value.array()));
- case DOUBLE:
- return (T) ((Double) BytesUtils.bytesToDouble(value.array()));
- case TEXT:
- return (T) new Binary(BytesUtils.bytesToString(value.array()));
- case BOOLEAN:
- return (T) (Boolean) BytesUtils.bytesToBool(value.array());
- default:
- throw new UnSupportFilterDataTypeException(
- "DigestForFilter unsupported datatype : " + type.toString());
- }
- }
-
- public long getMinTime() {
- return minTime;
- }
-
- public long getMaxTime() {
- return maxTime;
- }
-
- public boolean isMinValueNull() {
- return minValue == null;
- }
-
- public boolean isMaxValueNull() {
- return maxValue == null;
- }
-
- /**
- * Note check isMinValueNull before its usage
- */
- public <T extends Comparable<T>> T getMinValue() {
- return getValue(minValue);
- }
-
- /**
- * Note check isMaxValueNull before its usage
- */
- public <T extends Comparable<T>> T getMaxValue() {
- return getValue(maxValue);
- }
-
- /**
- * get type class.
- *
- * @return type class, i.g. Long.class
- */
- public Class<?> getTypeClass() {
- switch (type) {
- case INT32:
- return Integer.class;
- case INT64:
- return Long.class;
- case FLOAT:
- return Float.class;
- case DOUBLE:
- return Double.class;
- case TEXT:
- return String.class;
- case BOOLEAN:
- return Boolean.class;
- default:
- throw new UnSupportFilterDataTypeException(
- "DigestForFilter unsupported datatype : " + type.toString());
- }
- }
-
- public TSDataType getType() {
- return type;
- }
-
-}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/Filter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/Filter.java
index a37ccd2..ce2ccc5 100755
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/Filter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/Filter.java
@@ -18,7 +18,7 @@
*/
package org.apache.iotdb.tsfile.read.filter.basic;
-import org.apache.iotdb.tsfile.read.filter.DigestForFilter;
+import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
/**
* Filter is a top level filter abstraction.
@@ -27,12 +27,12 @@ import org.apache.iotdb.tsfile.read.filter.DigestForFilter;
public interface Filter {
/**
- * To examine whether the digest is satisfied with the filter.
+ * To examine whether the statistics is satisfied with the filter.
*
- * @param digest
- * digest with min time, max time, min value, max value.
+ * @param statistics
+ * statistics with min time, max time, min value, max value.
*/
- boolean satisfy(DigestForFilter digest);
+ boolean satisfy(Statistics statistics);
/**
* To examine whether the single point(with time and value) is satisfied with the filter.
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/AndFilter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/AndFilter.java
index 8b6e5db..4ba5778 100755
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/AndFilter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/AndFilter.java
@@ -18,7 +18,7 @@
*/
package org.apache.iotdb.tsfile.read.filter.operator;
-import org.apache.iotdb.tsfile.read.filter.DigestForFilter;
+import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.filter.basic.BinaryFilter;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
@@ -34,8 +34,8 @@ public class AndFilter extends BinaryFilter {
}
@Override
- public boolean satisfy(DigestForFilter digest) {
- return left.satisfy(digest) && right.satisfy(digest);
+ public boolean satisfy(Statistics statistics) {
+ return left.satisfy(statistics) && right.satisfy(statistics);
}
@Override
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Eq.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Eq.java
index 21cce69..48b6415 100755
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Eq.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Eq.java
@@ -18,7 +18,8 @@
*/
package org.apache.iotdb.tsfile.read.filter.operator;
-import org.apache.iotdb.tsfile.read.filter.DigestForFilter;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.read.filter.basic.UnaryFilter;
import org.apache.iotdb.tsfile.read.filter.factory.FilterType;
@@ -37,15 +38,15 @@ public class Eq<T extends Comparable<T>> extends UnaryFilter<T> {
}
@Override
- public boolean satisfy(DigestForFilter digest) {
+ public boolean satisfy(Statistics statistics) {
if (filterType == FilterType.TIME_FILTER) {
- return ((Long) value) >= digest.getMinTime() && ((Long) value) <= digest.getMaxTime();
+ return ((Long) value) >= statistics.getStartTime() && ((Long) value) <= statistics.getEndTime();
} else {
- if (digest.isMinValueNull() || digest.isMaxValueNull()) {
+ if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) {
return true;
}
- return value.compareTo(digest.getMinValue()) >= 0
- && value.compareTo(digest.getMaxValue()) <= 0;
+ return value.compareTo((T) statistics.getMinValue()) >= 0
+ && value.compareTo((T) statistics.getMaxValue()) <= 0;
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Gt.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Gt.java
index 7e7ff63..a48f0d5 100755
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Gt.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Gt.java
@@ -18,7 +18,8 @@
*/
package org.apache.iotdb.tsfile.read.filter.operator;
-import org.apache.iotdb.tsfile.read.filter.DigestForFilter;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.read.filter.basic.UnaryFilter;
import org.apache.iotdb.tsfile.read.filter.factory.FilterType;
@@ -37,14 +38,14 @@ public class Gt<T extends Comparable<T>> extends UnaryFilter<T> {
}
@Override
- public boolean satisfy(DigestForFilter digest) {
+ public boolean satisfy(Statistics statistics) {
if (filterType == FilterType.TIME_FILTER) {
- return ((Long) value) < digest.getMaxTime();
+ return ((Long) value) < statistics.getEndTime();
} else {
- if (digest.isMaxValueNull()) {
+ if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) {
return true;
}
- return value.compareTo(digest.getMaxValue()) < 0;
+ return value.compareTo((T) statistics.getMaxValue()) < 0;
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/GtEq.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/GtEq.java
index 7a63c4f..8391225 100755
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/GtEq.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/GtEq.java
@@ -18,7 +18,8 @@
*/
package org.apache.iotdb.tsfile.read.filter.operator;
-import org.apache.iotdb.tsfile.read.filter.DigestForFilter;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.read.filter.basic.UnaryFilter;
import org.apache.iotdb.tsfile.read.filter.factory.FilterType;
@@ -37,14 +38,14 @@ public class GtEq<T extends Comparable<T>> extends UnaryFilter<T> {
}
@Override
- public boolean satisfy(DigestForFilter digest) {
+ public boolean satisfy(Statistics statistics) {
if (filterType == FilterType.TIME_FILTER) {
- return ((Long) value) <= digest.getMaxTime();
+ return ((Long) value) <= statistics.getEndTime();
} else {
- if (digest.isMaxValueNull()) {
+ if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) {
return true;
}
- return value.compareTo(digest.getMaxValue()) <= 0;
+ return value.compareTo((T) statistics.getMaxValue()) <= 0;
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Lt.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Lt.java
index 3954935..7d92a78 100755
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Lt.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Lt.java
@@ -18,7 +18,8 @@
*/
package org.apache.iotdb.tsfile.read.filter.operator;
-import org.apache.iotdb.tsfile.read.filter.DigestForFilter;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.read.filter.basic.UnaryFilter;
import org.apache.iotdb.tsfile.read.filter.factory.FilterType;
@@ -37,14 +38,14 @@ public class Lt<T extends Comparable<T>> extends UnaryFilter<T> {
}
@Override
- public boolean satisfy(DigestForFilter digest) {
+ public boolean satisfy(Statistics statistics) {
if (filterType == FilterType.TIME_FILTER) {
- return ((Long) value) > digest.getMinTime();
+ return ((Long) value) > statistics.getStartTime();
} else {
- if (digest.isMinValueNull()) {
+ if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) {
return true;
}
- return value.compareTo(digest.getMinValue()) > 0;
+ return value.compareTo((T) statistics.getMinValue()) > 0;
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/LtEq.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/LtEq.java
index 7ca6b12..f25db22 100755
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/LtEq.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/LtEq.java
@@ -18,7 +18,8 @@
*/
package org.apache.iotdb.tsfile.read.filter.operator;
-import org.apache.iotdb.tsfile.read.filter.DigestForFilter;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.read.filter.basic.UnaryFilter;
import org.apache.iotdb.tsfile.read.filter.factory.FilterType;
@@ -37,14 +38,14 @@ public class LtEq<T extends Comparable<T>> extends UnaryFilter<T> {
}
@Override
- public boolean satisfy(DigestForFilter digest) {
+ public boolean satisfy(Statistics statistics) {
if (filterType == FilterType.TIME_FILTER) {
- return ((Long) value) >= digest.getMinTime();
+ return ((Long) value) >= statistics.getStartTime();
} else {
- if (digest.isMinValueNull()) {
+ if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) {
return true;
}
- return value.compareTo(digest.getMinValue()) >= 0;
+ return value.compareTo((T) statistics.getMinValue()) >= 0;
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/NotEq.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/NotEq.java
index 1c0e597..f570f86 100755
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/NotEq.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/NotEq.java
@@ -18,7 +18,8 @@
*/
package org.apache.iotdb.tsfile.read.filter.operator;
-import org.apache.iotdb.tsfile.read.filter.DigestForFilter;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.read.filter.basic.UnaryFilter;
import org.apache.iotdb.tsfile.read.filter.factory.FilterType;
@@ -37,15 +38,15 @@ public class NotEq<T extends Comparable<T>> extends UnaryFilter<T> {
}
@Override
- public boolean satisfy(DigestForFilter digest) {
+ public boolean satisfy(Statistics statistics) {
if (filterType == FilterType.TIME_FILTER) {
- return !(((Long) value) == digest.getMinTime() && (Long) value == digest.getMaxTime());
+ return !(((Long) value) == statistics.getStartTime() && (Long) value == statistics.getEndTime());
} else {
- if (digest.isMinValueNull() || digest.isMaxValueNull()) {
+ if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) {
return true;
}
- return !(value.compareTo(digest.getMinValue()) == 0
- && value.compareTo(digest.getMaxValue()) == 0);
+ return !(value.compareTo((T) statistics.getMinValue()) == 0
+ && value.compareTo((T) statistics.getMaxValue()) == 0);
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/NotFilter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/NotFilter.java
index 4135c45..58baa11 100755
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/NotFilter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/NotFilter.java
@@ -19,7 +19,7 @@
package org.apache.iotdb.tsfile.read.filter.operator;
import java.io.Serializable;
-import org.apache.iotdb.tsfile.read.filter.DigestForFilter;
+import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
/**
@@ -35,8 +35,8 @@ public class NotFilter implements Filter, Serializable {
}
@Override
- public boolean satisfy(DigestForFilter digest) {
- return !that.satisfy(digest);
+ public boolean satisfy(Statistics statistics) {
+ return !that.satisfy(statistics);
}
@Override
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/OrFilter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/OrFilter.java
index 0472be8..5797074 100755
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/OrFilter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/OrFilter.java
@@ -19,7 +19,7 @@
package org.apache.iotdb.tsfile.read.filter.operator;
import java.io.Serializable;
-import org.apache.iotdb.tsfile.read.filter.DigestForFilter;
+import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.filter.basic.BinaryFilter;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
@@ -45,8 +45,8 @@ public class OrFilter extends BinaryFilter implements Serializable {
}
@Override
- public boolean satisfy(DigestForFilter digest) {
- return left.satisfy(digest) || right.satisfy(digest);
+ public boolean satisfy(Statistics statistics) {
+ return left.satisfy(statistics) || right.satisfy(statistics);
}
@Override
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/executor/ExecutorWithTimeGenerator.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/executor/ExecutorWithTimeGenerator.java
index 2c0b117..a34b3df 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/executor/ExecutorWithTimeGenerator.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/executor/ExecutorWithTimeGenerator.java
@@ -76,7 +76,7 @@ public class ExecutorWithTimeGenerator implements QueryExecutor {
List<ChunkMetaData> chunkMetaDataList = metadataQuerier.getChunkMetaDataList(selectedPath);
if (chunkMetaDataList.size() != 0) {
- dataTypes.add(chunkMetaDataList.get(0).getTsDataType());
+ dataTypes.add(chunkMetaDataList.get(0).getDataType());
if (cachedValue) {
readersOfSelectedSeries.add(null);
continue;
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/executor/TsFileExecutor.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/executor/TsFileExecutor.java
index 7484ff4..9b2b205 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/executor/TsFileExecutor.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/executor/TsFileExecutor.java
@@ -185,7 +185,7 @@ public class TsFileExecutor implements QueryExecutor {
seriesReader = new FileSeriesReaderWithFilter(chunkLoader, chunkMetaDataList,
timeFilter.getFilter());
}
- dataTypes.add(chunkMetaDataList.get(0).getTsDataType());
+ dataTypes.add(chunkMetaDataList.get(0).getDataType());
}
readersOfSelectedSeries.add(seriesReader);
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReader.java
index eb00310..479b98a 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReader.java
@@ -87,7 +87,7 @@ public abstract class ChunkReader {
/**
* judge if has nextBatch.
*/
- public boolean hasNextBatch() throws IOException {
+ public boolean hasNextBatch() {
if (hasCachedPageHeader) {
return true;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReaderByTimestamp.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReaderByTimestamp.java
index 4d3e9a2..951c327 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReaderByTimestamp.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReaderByTimestamp.java
@@ -31,7 +31,7 @@ public class ChunkReaderByTimestamp extends ChunkReader {
@Override
public boolean pageSatisfied(PageHeader pageHeader) {
- long maxTimestamp = pageHeader.getMaxTimestamp();
+ long maxTimestamp = pageHeader.getEndTime();
// if maxTimestamp > currentTimestamp, this page should NOT be skipped
return maxTimestamp >= currentTimestamp && maxTimestamp > deletedAt;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReaderWithFilter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReaderWithFilter.java
index 0381628..62c9cb2 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReaderWithFilter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReaderWithFilter.java
@@ -20,7 +20,6 @@ package org.apache.iotdb.tsfile.read.reader.chunk;
import org.apache.iotdb.tsfile.file.header.PageHeader;
import org.apache.iotdb.tsfile.read.common.Chunk;
-import org.apache.iotdb.tsfile.read.filter.DigestForFilter;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
public class ChunkReaderWithFilter extends ChunkReader {
@@ -34,15 +33,10 @@ public class ChunkReaderWithFilter extends ChunkReader {
@Override
public boolean pageSatisfied(PageHeader pageHeader) {
- if (pageHeader.getMaxTimestamp() < deletedAt) {
+ if (pageHeader.getEndTime() < deletedAt) {
return false;
}
- DigestForFilter digest = new DigestForFilter(pageHeader.getMinTimestamp(),
- pageHeader.getMaxTimestamp(),
- pageHeader.getStatistics().getMinBytebuffer(),
- pageHeader.getStatistics().getMaxBytebuffer(),
- chunkHeader.getDataType());
- return filter.satisfy(digest);
+ return filter.satisfy(pageHeader.getStatistics());
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReaderWithoutFilter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReaderWithoutFilter.java
index 720b244..c2a09c6 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReaderWithoutFilter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReaderWithoutFilter.java
@@ -29,7 +29,7 @@ public class ChunkReaderWithoutFilter extends ChunkReader {
@Override
public boolean pageSatisfied(PageHeader pageHeader) {
- return pageHeader.getMaxTimestamp() > deletedAt;
+ return pageHeader.getEndTime() > deletedAt;
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReaderByTimestamp.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReaderByTimestamp.java
index 8b0e916..d9d7a51 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReaderByTimestamp.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReaderByTimestamp.java
@@ -54,7 +54,7 @@ public class FileSeriesReaderByTimestamp {
}
public TSDataType getDataType() {
- return chunkMetaDataList.get(0).getTsDataType();
+ return chunkMetaDataList.get(0).getDataType();
}
/**
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReaderWithFilter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReaderWithFilter.java
index ad8968d..0a2fc53 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReaderWithFilter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReaderWithFilter.java
@@ -19,13 +19,10 @@
package org.apache.iotdb.tsfile.read.reader.series;
import java.io.IOException;
-import java.nio.ByteBuffer;
import java.util.List;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
-import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.common.Chunk;
import org.apache.iotdb.tsfile.read.controller.IChunkLoader;
-import org.apache.iotdb.tsfile.read.filter.DigestForFilter;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReaderWithFilter;
@@ -51,17 +48,7 @@ public class FileSeriesReaderWithFilter extends FileSeriesReader {
@Override
protected boolean chunkSatisfied(ChunkMetaData chunkMetaData) {
- ByteBuffer minValue = null;
- ByteBuffer maxValue = null;
- ByteBuffer[] statistics = chunkMetaData.getDigest().getStatistics();
- if (statistics != null) {
- minValue = statistics[Statistics.StatisticType.min_value.ordinal()]; // note still CAN be null
- maxValue = statistics[Statistics.StatisticType.max_value.ordinal()]; // note still CAN be null
- }
-
- DigestForFilter digest = new DigestForFilter(chunkMetaData.getStartTime(),
- chunkMetaData.getEndTime(), minValue, maxValue, chunkMetaData.getTsDataType());
- return filter.satisfy(digest);
+ return filter.satisfy(chunkMetaData.getStatistics());
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/tool/upgrade/TsfileUpgradeToolV0_8_0.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/tool/upgrade/TsfileUpgradeToolV0_8_0.java
index cdb7436..328dc3a 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/tool/upgrade/TsfileUpgradeToolV0_8_0.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/tool/upgrade/TsfileUpgradeToolV0_8_0.java
@@ -344,9 +344,6 @@ public class TsfileUpgradeToolV0_8_0 implements AutoCloseable {
schema = new Schema(tsFileMetaData.getMeasurementSchema());
}
- long startTimeOfChunk = 0;
- long endTimeOfChunk = 0;
- long numOfPoints = 0;
ChunkMetaData currentChunkMetaData;
List<ChunkMetaData> chunkMetaDataList = null;
long startOffsetOfChunkGroup = 0;
@@ -379,48 +376,25 @@ public class TsfileUpgradeToolV0_8_0 implements AutoCloseable {
if (header.getNumOfPages() > 0) {
PageHeader pageHeader = this.readPageHeader(header.getDataType());
pageHeaders.add(pageHeader);
- numOfPoints += pageHeader.getNumOfValues();
- startTimeOfChunk = pageHeader.getMinTimestamp();
- endTimeOfChunk = pageHeader.getMaxTimestamp();
chunkStatistics.mergeStatistics(pageHeader.getStatistics());
pages.add(readData(-1, pageHeader.getCompressedSize()));
}
for (int j = 1; j < header.getNumOfPages() - 1; j++) {
PageHeader pageHeader = this.readPageHeader(header.getDataType());
pageHeaders.add(pageHeader);
- numOfPoints += pageHeader.getNumOfValues();
chunkStatistics.mergeStatistics(pageHeader.getStatistics());
pages.add(readData(-1, pageHeader.getCompressedSize()));
}
if (header.getNumOfPages() > 1) {
PageHeader pageHeader = this.readPageHeader(header.getDataType());
pageHeaders.add(pageHeader);
- numOfPoints += pageHeader.getNumOfValues();
- endTimeOfChunk = pageHeader.getMaxTimestamp();
chunkStatistics.mergeStatistics(pageHeader.getStatistics());
pages.add(readData(-1, pageHeader.getCompressedSize()));
}
currentChunkMetaData = new ChunkMetaData(header.getMeasurementID(), dataType,
- fileOffsetOfChunk,
- startTimeOfChunk, endTimeOfChunk);
- currentChunkMetaData.setNumOfPoints(numOfPoints);
- ByteBuffer[] statisticsArray = new ByteBuffer[Statistics.StatisticType.getTotalTypeNum()];
- statisticsArray[Statistics.StatisticType.min_value.ordinal()] = ByteBuffer
- .wrap(chunkStatistics.getMinBytes());
- statisticsArray[Statistics.StatisticType.max_value.ordinal()] = ByteBuffer
- .wrap(chunkStatistics.getMaxBytes());
- statisticsArray[Statistics.StatisticType.first_value.ordinal()] = ByteBuffer
- .wrap(chunkStatistics.getFirstBytes());
- statisticsArray[Statistics.StatisticType.last_value.ordinal()] = ByteBuffer
- .wrap(chunkStatistics.getLastBytes());
- statisticsArray[Statistics.StatisticType.sum_value.ordinal()] = ByteBuffer
- .wrap(chunkStatistics.getSumBytes());
- Statistics tsDigest = Statistics.getStatsByType(dataType);
- tsDigest.setStatistics(statisticsArray);
- currentChunkMetaData.setDigest(tsDigest);
+ fileOffsetOfChunk, chunkStatistics);
chunkMetaDataList.add(currentChunkMetaData);
- numOfPoints = 0;
pageHeadersList.add(pageHeaders);
pagesList.add(pages);
break;
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/ReadWriteIOUtils.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/ReadWriteIOUtils.java
index 401686d..0a67b17 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/ReadWriteIOUtils.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/ReadWriteIOUtils.java
@@ -192,6 +192,18 @@ public class ReadWriteIOUtils {
return bytes.length;
}
+ public static int write(byte[] bytes, OutputStream outputStream) throws IOException {
+ outputStream.write(bytes);
+ return bytes.length;
+ }
+
+ public static int write(Binary binary, OutputStream outputStream) throws IOException {
+ byte[] bytes = BytesUtils.intToBytes(binary.getValues().length);
+ outputStream.write(bytes);
+ outputStream.write(binary.getValues());
+ return bytes.length + binary.getValues().length;
+ }
+
/**
* write a int n to byteBuffer.
*
@@ -571,6 +583,8 @@ public class ReadWriteIOUtils {
/**
* read bytes from byteBuffer, this method makes sure that you can read length bytes or reach to
* the end of the buffer.
+ *
+ * read a int + buffer
*/
public static ByteBuffer readByteBufferWithSelfDescriptionLength(ByteBuffer buffer) {
int byteLength = readInt(buffer);
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkWriterImpl.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkWriterImpl.java
index 2beff8d..e6b3298 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkWriterImpl.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkWriterImpl.java
@@ -19,7 +19,6 @@
package org.apache.iotdb.tsfile.write.chunk;
import java.io.IOException;
-import java.math.BigDecimal;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.WritableByteChannel;
@@ -51,10 +50,6 @@ public class ChunkWriterImpl implements IChunkWriter {
*/
private PublicBAOS pageBuffer;
- private long chunkPointCount;
- private long chunkMaxTime;
- private long chunkMinTime = Long.MIN_VALUE;
-
private int numOfPages;
/**
@@ -135,12 +130,6 @@ public class ChunkWriterImpl implements IChunkWriter {
}
@Override
- public void write(long time, BigDecimal value) {
- pageWriter.write(time, value);
- checkPageSizeAndMayOpenANewPage();
- }
-
- @Override
public void write(long time, Binary value) {
pageWriter.write(time, value);
checkPageSizeAndMayOpenANewPage();
@@ -177,12 +166,6 @@ public class ChunkWriterImpl implements IChunkWriter {
}
@Override
- public void write(long[] timestamps, BigDecimal[] values, int batchSize) {
- pageWriter.write(timestamps, values, batchSize);
- checkPageSizeAndMayOpenANewPage();
- }
-
- @Override
public void write(long[] timestamps, Binary[] values, int batchSize) {
pageWriter.write(timestamps, values, batchSize);
checkPageSizeAndMayOpenANewPage();
@@ -222,11 +205,6 @@ public class ChunkWriterImpl implements IChunkWriter {
// update statistics of this chunk
numOfPages++;
- chunkMaxTime = pageWriter.getPageMaxTime();
- if (chunkMinTime == Long.MIN_VALUE) {
- chunkMinTime = pageWriter.getPageMinTime();
- }
- chunkPointCount += pageWriter.getPointNumber();
this.chunkStatistics.mergeStatistics(pageWriter.getStatistics());
} catch (IOException e) {
logger.error("meet error in pageWriter.writePageHeaderAndDataIntoBuff,ignore this page:", e);
@@ -243,8 +221,6 @@ public class ChunkWriterImpl implements IChunkWriter {
// reinit this chunk writer
pageBuffer.reset();
- chunkPointCount = 0;
- chunkMinTime = Long.MIN_VALUE;
this.chunkStatistics = Statistics.getStatsByType(measurementSchema.getType());
}
@@ -286,15 +262,6 @@ public class ChunkWriterImpl implements IChunkWriter {
throws PageException {
numOfPages++;
- // 1. update time statistics
- if (this.chunkMinTime == Long.MIN_VALUE) {
- this.chunkMinTime = header.getMinTimestamp();
- }
- if (this.chunkMinTime == Long.MIN_VALUE) {
- throw new PageException("No valid data point in this page");
- }
- this.chunkMaxTime = header.getMaxTimestamp();
-
// write the page header to pageBuffer
try {
logger.debug("start to flush a page header into buffer, buffer position {} ", pageBuffer.size());
@@ -302,17 +269,13 @@ public class ChunkWriterImpl implements IChunkWriter {
logger.debug("finish to flush a page header {} of {} into buffer, buffer position {} ", header,
measurementSchema.getMeasurementId(), pageBuffer.size());
+ chunkStatistics.mergeStatistics(header.getStatistics());
+
} catch (IOException e) {
- if (chunkPointCount == 0) {
- chunkMinTime = Long.MIN_VALUE;
- }
throw new PageException(
"IO Exception in writeDataPageHeader,ignore this page", e);
}
- // update data point num
- this.chunkPointCount += header.getNumOfValues();
-
// write page content to temp PBAOS
try (WritableByteChannel channel = Channels.newChannel(pageBuffer)) {
channel.write(data);
@@ -326,21 +289,17 @@ public class ChunkWriterImpl implements IChunkWriter {
*
* @param writer the specified IOWriter
* @param statistics the chunk statistics
- * @return the data size of this chunk
* @throws IOException exception in IO
*/
- public long writeAllPagesOfChunkToTsFile(TsFileIOWriter writer, Statistics<?> statistics)
+ public void writeAllPagesOfChunkToTsFile(TsFileIOWriter writer, Statistics<?> statistics)
throws IOException {
- if (chunkPointCount == 0) {
- return 0;
+ if (statistics.getCount() == 0) {
+ return;
}
// start to write this column chunk
- int headerSize = writer
- .startFlushChunk(measurementSchema, compressor.getType(), measurementSchema.getType(),
- measurementSchema.getEncodingType(), statistics, chunkMaxTime,
- chunkMinTime, pageBuffer.size(),
- numOfPages);
+ writer.startFlushChunk(measurementSchema, compressor.getType(), measurementSchema.getType(),
+ measurementSchema.getEncodingType(), statistics, pageBuffer.size(), numOfPages);
long dataOffset = writer.getPos();
@@ -354,8 +313,7 @@ public class ChunkWriterImpl implements IChunkWriter {
+ " " + pageBuffer.size());
}
- writer.endChunk(chunkPointCount);
- return headerSize + dataSize;
+ writer.endCurrentChunk();
}
/**
@@ -363,13 +321,11 @@ public class ChunkWriterImpl implements IChunkWriter {
*
* @return the max possible allocated size currently
*/
- public long estimateMaxPageMemSize() {
+ private long estimateMaxPageMemSize() {
// return the sum of size of buffer and page max size
- return (long) (pageBuffer.size() + estimateMaxPageHeaderSize());
- }
-
- private int estimateMaxPageHeaderSize() {
- return PageHeader.calculatePageHeaderSize(measurementSchema.getType());
+ return (long) (pageBuffer.size() +
+ PageHeader.calculatePageHeaderSizeWithoutStatistics() +
+ pageWriter.getStatistics().getSerializedSize());
}
/**
@@ -377,7 +333,7 @@ public class ChunkWriterImpl implements IChunkWriter {
*
* @return current data size that the writer has serialized.
*/
- public long getCurrentDataSize() {
+ private long getCurrentDataSize() {
return pageBuffer.size();
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/IChunkWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/IChunkWriter.java
index 3f7f762..c375b45 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/IChunkWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/IChunkWriter.java
@@ -57,11 +57,6 @@ public interface IChunkWriter {
/**
* write a time value pair.
*/
- void write(long time, BigDecimal value);
-
- /**
- * write a time value pair.
- */
void write(long time, Binary value);
/**
@@ -92,11 +87,6 @@ public interface IChunkWriter {
/**
* write time series
*/
- void write(long[] timestamps, BigDecimal[] values, int batchSize);
-
- /**
- * write time series
- */
void write(long[] timestamps, Binary[] values, int batchSize);
/**
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/PageWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/PageWriter.java
index 048baaf..1203280 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/PageWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/PageWriter.java
@@ -91,7 +91,7 @@ public class PageWriter {
}
timeEncoder.encode(time, timeOut);
valueEncoder.encode(value, valueOut);
- statistics.updateStats(value);
+ statistics.update(time, value);
}
/**
@@ -101,7 +101,7 @@ public class PageWriter {
++pointNumber;
timeEncoder.encode(time, timeOut);
valueEncoder.encode(value, valueOut);
- statistics.updateStats(value);
+ statistics.update(time, value);
}
/**
@@ -115,7 +115,7 @@ public class PageWriter {
}
timeEncoder.encode(time, timeOut);
valueEncoder.encode(value, valueOut);
- statistics.updateStats(value);
+ statistics.update(time, value);
}
/**
@@ -129,7 +129,7 @@ public class PageWriter {
}
timeEncoder.encode(time, timeOut);
valueEncoder.encode(value, valueOut);
- statistics.updateStats(value);
+ statistics.update(time, value);
}
/**
@@ -143,7 +143,7 @@ public class PageWriter {
}
timeEncoder.encode(time, timeOut);
valueEncoder.encode(value, valueOut);
- statistics.updateStats(value);
+ statistics.update(time, value);
}
/**
@@ -157,21 +157,7 @@ public class PageWriter {
}
timeEncoder.encode(time, timeOut);
valueEncoder.encode(value, valueOut);
- statistics.updateStats(value);
- }
-
- /**
- * write a time value pair into encoder
- */
- public void write(long time, BigDecimal value) {
- ++pointNumber;
- this.pageMaxTime = time;
- if (pageMinTime == Long.MIN_VALUE) {
- pageMinTime = time;
- }
- timeEncoder.encode(time, timeOut);
- valueEncoder.encode(value, valueOut);
- statistics.updateStats(value);
+ statistics.update(time, value);
}
/**
@@ -185,7 +171,7 @@ public class PageWriter {
}
timeEncoder.encode(time, timeOut);
valueEncoder.encode(value, valueOut);
- statistics.updateStats(value);
+ statistics.update(time, value);
}
/**
@@ -201,7 +187,7 @@ public class PageWriter {
timeEncoder.encode(timestamps[i], timeOut);
valueEncoder.encode(values[i], valueOut);
}
- statistics.updateStats(values);
+ statistics.update(timestamps, values, batchSize);
}
/**
@@ -217,7 +203,7 @@ public class PageWriter {
timeEncoder.encode(timestamps[i], timeOut);
valueEncoder.encode(values[i], valueOut);
}
- statistics.updateStats(values);
+ statistics.update(timestamps, values, batchSize);
}
/**
@@ -233,7 +219,7 @@ public class PageWriter {
timeEncoder.encode(timestamps[i], timeOut);
valueEncoder.encode(values[i], valueOut);
}
- statistics.updateStats(values);
+ statistics.update(timestamps, values, batchSize);
}
/**
@@ -249,7 +235,7 @@ public class PageWriter {
timeEncoder.encode(timestamps[i], timeOut);
valueEncoder.encode(values[i], valueOut);
}
- statistics.updateStats(values);
+ statistics.update(timestamps, values, batchSize);
}
/**
@@ -265,23 +251,7 @@ public class PageWriter {
timeEncoder.encode(timestamps[i], timeOut);
valueEncoder.encode(values[i], valueOut);
}
- statistics.updateStats(values);
- }
-
- /**
- * write time series into encoder
- */
- public void write(long[] timestamps, BigDecimal[] values, int batchSize) {
- pointNumber += batchSize;
- this.pageMaxTime = timestamps[batchSize - 1];
- if (pageMinTime == Long.MIN_VALUE) {
- pageMinTime = timestamps[0];
- }
- for (int i = 0; i < batchSize; i++) {
- timeEncoder.encode(timestamps[i], timeOut);
- valueEncoder.encode(values[i], valueOut);
- }
- statistics.updateStats(values);
+ statistics.update(timestamps, values, batchSize);
}
/**
@@ -297,7 +267,7 @@ public class PageWriter {
timeEncoder.encode(timestamps[i], timeOut);
valueEncoder.encode(values[i], valueOut);
}
- statistics.updateStats(values);
+ statistics.update(timestamps, values, batchSize);
}
/**
@@ -334,12 +304,10 @@ public class PageWriter {
/**
* write the page header and data into the PageWriter's output stream.
- *
- * @return byte size of the page header and uncompressed data in the page body.
*/
- public int writePageHeaderAndDataIntoBuff(PublicBAOS pageBuffer) throws IOException {
+ public void writePageHeaderAndDataIntoBuff(PublicBAOS pageBuffer) throws IOException {
if (pointNumber == 0) {
- return 0;
+ return;
}
ByteBuffer pageData = getUncompressedBytes();
@@ -361,7 +329,6 @@ public class PageWriter {
// write the page header to IOWriter
PageHeader header = new PageHeader(uncompressedSize, compressedSize, pointNumber, statistics,
pageMaxTime, pageMinTime);
- int headerSize = header.getSerializedSize();
header.serializeTo(pageBuffer);
// write page content to temp PBAOS
@@ -374,7 +341,6 @@ public class PageWriter {
}
logger.debug("start to flush a page data into buffer, buffer position {} ", pageBuffer.size());
}
- return headerSize + uncompressedSize;
}
/**
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java
index fe5dfff..3dc8cb5 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java
@@ -133,7 +133,7 @@ public class RestorableTsFileIOWriter extends TsFileIOWriter {
// filter: if a device'sensor is defined as float type, and data has been persistent.
// Then someone deletes the timeseries and recreate it with Int type. We have to ignore
// all the stale data.
- if (dataType == null || dataType.equals(chunkMetaData.getTsDataType())) {
+ if (dataType == null || dataType.equals(chunkMetaData.getDataType())) {
chunkMetaDataList.add(chunkMetaData);
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
index 44636c5..a947d54 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
@@ -20,7 +20,6 @@ package org.apache.iotdb.tsfile.write.writer;
import java.io.File;
import java.io.IOException;
-import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
@@ -44,7 +43,6 @@ import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
-import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
import org.apache.iotdb.tsfile.read.common.Chunk;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.utils.BytesUtils;
@@ -108,23 +106,6 @@ public class TsFileIOWriter {
startFile();
}
- /**
- * for writing data into an existing and incomplete Tsfile. The caller need to guarantee existing
- * data in the TsFileOutput matches the given metadata list
- *
- * @param out the target output
- * @param chunkGroupMetaDataList existing chunkgroups' metadata
- * @throws IOException if I/O error occurs
- */
- public TsFileIOWriter(TsFileOutput out, List<ChunkGroupMetaData> chunkGroupMetaDataList)
- throws IOException {
- this.out = FSFactoryProducer.getFileOutputFactory()
- .getTsFileOutput(file.getPath(), false); //NOTE overwrite false here
- this.chunkGroupMetaDataList = chunkGroupMetaDataList;
- if (chunkGroupMetaDataList.isEmpty()) {
- startFile();
- }
- }
/**
* Writes given bytes to output stream. This method is called when total memory size exceeds the
@@ -179,18 +160,15 @@ public class TsFileIOWriter {
* @param compressionCodecName - compression name of this time series
* @param tsDataType - data type
* @param statistics - Chunk statistics
- * @param maxTime - maximum timestamp of the whole series in this stage
- * @param minTime - minimum timestamp of the whole series in this stage
* @param dataSize - the serialized size of all pages
- * @return the serialized size of CHunkHeader
* @throws IOException if I/O error occurs
*/
- public int startFlushChunk(MeasurementSchema descriptor, CompressionType compressionCodecName,
- TSDataType tsDataType, TSEncoding encodingType, Statistics<?> statistics, long maxTime,
- long minTime, int dataSize, int numOfPages) throws IOException {
+ public void startFlushChunk(MeasurementSchema descriptor, CompressionType compressionCodecName,
+ TSDataType tsDataType, TSEncoding encodingType, Statistics<?> statistics,
+ int dataSize, int numOfPages) throws IOException {
currentChunkMetaData = new ChunkMetaData(descriptor.getMeasurementId(), tsDataType,
- out.getPosition(), minTime, maxTime);
+ out.getPosition(), statistics);
// flush ChunkHeader to TsFileIOWriter
if (logger.isDebugEnabled()) {
@@ -204,24 +182,6 @@ public class TsFileIOWriter {
if (logger.isDebugEnabled()) {
logger.debug("finish series chunk:{} header, file position {}", header, out.getPosition());
}
-
- // TODO add your statistics
- ByteBuffer[] statisticsArray = new ByteBuffer[Statistics.StatisticType.getTotalTypeNum()];
- statisticsArray[Statistics.StatisticType.max_value.ordinal()] = ByteBuffer.wrap(statistics.getMaxBytes());
- statisticsArray[Statistics.StatisticType.min_value.ordinal()] = ByteBuffer.wrap(statistics.getMinBytes());
- statisticsArray[Statistics.StatisticType.first_value.ordinal()] = ByteBuffer
- .wrap(statistics.getFirstBytes());
- statisticsArray[Statistics.StatisticType.last_value.ordinal()] = ByteBuffer
- .wrap(statistics.getLastBytes());
- statisticsArray[Statistics.StatisticType.sum_value.ordinal()] = ByteBuffer.wrap(statistics.getSumBytes());
-
- Statistics tsDigest = Statistics.getStatsByType(tsDataType);
-
- tsDigest.setStatistics(statisticsArray);
-
- currentChunkMetaData.setDigest(tsDigest);
-
- return header.getSerializedSize();
}
/**
@@ -230,23 +190,18 @@ public class TsFileIOWriter {
public void writeChunk(Chunk chunk, ChunkMetaData chunkMetadata) throws IOException {
ChunkHeader chunkHeader = chunk.getHeader();
currentChunkMetaData = new ChunkMetaData(chunkHeader.getMeasurementID(),
- chunkHeader.getDataType(), out.getPosition(), chunkMetadata.getStartTime(),
- chunkMetadata.getEndTime());
- currentChunkMetaData.setDigest(chunkMetadata.getDigest());
+ chunkHeader.getDataType(), out.getPosition(), chunkMetadata.getStatistics());
chunkHeader.serializeTo(out.wrapAsStream());
out.write(chunk.getData());
- endChunk(chunkMetadata.getNumOfPoints());
+ endCurrentChunk();
+ logger.debug("end flushing a chunk:{}, totalvalue:{}", currentChunkMetaData, chunkMetadata.getNumOfPoints());
}
/**
* end chunk and write some log.
- *
- * @param totalValueCount -set the number of points to the currentChunkMetaData
*/
- public void endChunk(long totalValueCount) {
- currentChunkMetaData.setNumOfPoints(totalValueCount);
+ public void endCurrentChunk() {
currentChunkGroupMetaData.addTimeSeriesChunkMetaData(currentChunkMetaData);
- logger.debug("end series chunk:{},totalvalue:{}", currentChunkMetaData, totalValueCount);
currentChunkMetaData = null;
totalChunkNum++;
}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/header/PageHeaderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/header/PageHeaderTest.java
index 04af370..d7394e7 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/header/PageHeaderTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/header/PageHeaderTest.java
@@ -23,13 +23,9 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
-import java.nio.channels.FileChannel;
-import java.nio.file.Paths;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.utils.TestHelper;
import org.apache.iotdb.tsfile.file.metadata.utils.Utils;
-import org.apache.iotdb.tsfile.read.reader.DefaultTsFileInput;
-import org.apache.iotdb.tsfile.read.reader.TsFileInput;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -41,9 +37,8 @@ public class PageHeaderTest {
public static final int NUM_OF_VALUES = 10000;
public static final long MAX_TIMESTAMO = 523372036854775806L;
public static final long MIN_TIMESTAMO = 423372036854775806L;
- public static final TSDataType DATA_TYPE = TSDataType.TEXT;
- public static final int OFFSET = 123456;
- final String PATH = "target/outputPageHeader.tsfile";
+ public static final TSDataType DATA_TYPE = TSDataType.INT64;
+ private final String PATH = "target/outputPageHeader.tsfile";
@Before
public void setUp() {
@@ -58,8 +53,8 @@ public class PageHeaderTest {
}
@Test
- public void testWriteIntoFile() throws IOException {
- PageHeader header = TestHelper.createSimplePageHeader();
+ public void testWriteIntoFile() {
+ PageHeader header = TestHelper.createTestPageHeader();
serialized(header);
PageHeader readHeader = deSerialized();
Utils.isPageHeaderEqual(header, readHeader);
@@ -109,57 +104,4 @@ public class PageHeaderTest {
}
}
- @Test
- public void testReadWithOffset() throws IOException {
- PageHeader header = TestHelper.createSimplePageHeader();
- serialized(header, OFFSET);
- PageHeader readHeader = deSerialized(OFFSET);
- Utils.isPageHeaderEqual(header, readHeader);
- serialized(readHeader);
- }
-
- private PageHeader deSerialized(int offset) {
- TsFileInput input = null;
- PageHeader header = null;
- try {
- input = new DefaultTsFileInput(Paths.get(PATH));
- header = PageHeader.deserializeFrom(DATA_TYPE, input, offset, true);
- return header;
- } catch (IOException e) {
- e.printStackTrace();
- } finally {
- if (input != null) {
- try {
- input.close();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- }
- return header;
- }
-
- private void serialized(PageHeader header, int offset) {
- File file = new File(PATH);
- if (file.exists()) {
- Assert.assertTrue(file.delete());
- }
- FileOutputStream fos = null;
- FileChannel fc = null;
- try {
- fos = new FileOutputStream(file);
- fos.write(new byte[offset]);
- header.serializeTo(fos);
- } catch (IOException e) {
- e.printStackTrace();
- } finally {
- if (fos != null) {
- try {
- fos.close();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- }
- }
}
\ No newline at end of file
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaDataTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaDataTest.java
index afa93fb..8b4e0b9 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaDataTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaDataTest.java
@@ -104,56 +104,4 @@ public class ChunkGroupMetaDataTest {
reader.close();
}
- @Test
- public void testWriteIntoFile() {
- // serialize metadata to a file
- ChunkGroupMetaData metaData = TestHelper.createSimpleChunkGroupMetaData();
- serialized(metaData);
- ChunkGroupMetaData readMetaData = deSerialized();
- serialized(readMetaData);
- }
-
- private ChunkGroupMetaData deSerialized() {
- FileInputStream fis = null;
- ChunkGroupMetaData metaData = null;
- try {
- fis = new FileInputStream(new File(PATH));
- metaData = ChunkGroupMetaData.deserializeFrom(fis);
- return metaData;
- } catch (IOException e) {
- e.printStackTrace();
- } finally {
- if (fis != null) {
- try {
- fis.close();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- }
- return metaData;
- }
-
- private void serialized(ChunkGroupMetaData metaData) {
- File file = new File(PATH);
- if (file.exists()) {
- file.delete();
- }
- FileOutputStream fos = null;
- try {
- fos = new FileOutputStream(file);
- metaData.serializeTo(fos);
- } catch (IOException e) {
- e.printStackTrace();
- } finally {
- if (fos != null) {
- try {
- fos.close();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- }
- }
-
}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetaDataTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetaDataTest.java
deleted file mode 100644
index a9c6140..0000000
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetaDataTest.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.tsfile.file.metadata;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.file.metadata.utils.TestHelper;
-import org.apache.iotdb.tsfile.file.metadata.utils.Utils;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class ChunkMetaDataTest {
-
- public static final String MEASUREMENT_UID = "sensor231";
- public static final long FILE_OFFSET = 2313424242L;
- public static final long NUM_OF_POINTS = 123456L;
- public static final long START_TIME = 523372036854775806L;
- public static final long END_TIME = 523372036854775806L;
- public static final TSDataType DATA_TYPE = TSDataType.INT64;
- final String PATH = "target/outputTimeSeriesChunk.tsfile";
-
- @Before
- public void setUp() {
- }
-
- @After
- public void tearDown() {
- File file = new File(PATH);
- if (file.exists()) {
- file.delete();
- }
- }
-
- @Test
- public void testWriteIntoFile() {
- ChunkMetaData metaData = TestHelper.createSimpleTimeSeriesChunkMetaData();
- serialized(metaData);
- ChunkMetaData readMetaData = deSerialized();
- Utils.isTimeSeriesChunkMetadataEqual(metaData, readMetaData);
- serialized(readMetaData);
- }
-
- @Test
- public void testWriteIntoFile2() throws IOException {
- ChunkMetaData metaData = TestHelper.createNotCompleteSimpleTimeSeriesChunkMetaData();
- serialized(metaData);
- ChunkMetaData readMetaData = deSerialized();
- Utils.isTimeSeriesChunkMetadataEqual(metaData, readMetaData);
- serialized(readMetaData);
- }
-
- private ChunkMetaData deSerialized() {
- FileInputStream fis = null;
- ChunkMetaData metaData = null;
- try {
- fis = new FileInputStream(new File(PATH));
- metaData = ChunkMetaData.deserializeFrom(fis);
- return metaData;
- } catch (IOException e) {
- e.printStackTrace();
- } finally {
- if (fis != null) {
- try {
- fis.close();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- }
- return metaData;
- }
-
- private void serialized(ChunkMetaData metaData) {
- File file = new File(PATH);
- if (file.exists()) {
- file.delete();
- }
- FileOutputStream fos = null;
- try {
- fos = new FileOutputStream(file);
- metaData.serializeTo(fos);
- } catch (IOException e) {
- e.printStackTrace();
- } finally {
- if (fos != null) {
- try {
- fos.close();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- }
- }
-}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/TsDeviceMetadataTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/TsDeviceMetadataTest.java
deleted file mode 100644
index 19539e2..0000000
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/TsDeviceMetadataTest.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.tsfile.file.metadata;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import org.apache.iotdb.tsfile.file.metadata.utils.TestHelper;
-import org.apache.iotdb.tsfile.file.metadata.utils.Utils;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class TsDeviceMetadataTest {
-
- public static final long START_TIME = 523372036854775806L;
- public static final long END_TIME = 523372036854775806L;
- final String PATH = "target/outputDevice.tsfile";
-
- @Before
- public void setUp() {
- }
-
- @After
- public void tearDown() {
- File file = new File(PATH);
- if (file.exists()) {
- file.delete();
- }
- }
-
- @Test
- public void testWriteIntoFile() throws IOException {
- TsDeviceMetadata metaData = TestHelper.createSimpleDeviceMetaData();
- File file = new File(PATH);
- if (file.exists()) {
- file.delete();
- }
- FileOutputStream fos = new FileOutputStream(file);
- metaData.serializeTo(fos);
- fos.close();
-
- FileInputStream fis = new FileInputStream(new File(PATH));
- Utils.isTsDeviceMetadataEqual(metaData, TsDeviceMetadata.deserializeFrom(fis));
- fis.close();
- }
-}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatisticsTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatisticsTest.java
index 41cb663..0db1a37 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatisticsTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatisticsTest.java
@@ -35,11 +35,8 @@ public class BooleanStatisticsTest {
assertFalse(booleanStatistics.isEmpty());
booleanStatistics.updateStats(false);
assertFalse(booleanStatistics.isEmpty());
- assertTrue(booleanStatistics.getMax());
- assertFalse(booleanStatistics.getMin());
- assertEquals(0, booleanStatistics.getSum(), maxError);
- assertTrue(booleanStatistics.getFirst());
- assertFalse(booleanStatistics.getLast());
+ assertTrue(booleanStatistics.getFirstValue());
+ assertFalse(booleanStatistics.getLastValue());
}
@Test
@@ -55,17 +52,11 @@ public class BooleanStatisticsTest {
Statistics<Boolean> booleanStats3 = new BooleanStatistics();
booleanStats3.mergeStatistics(booleanStats1);
assertFalse(booleanStats3.isEmpty());
- assertFalse(booleanStats3.getMax());
- assertFalse(booleanStats3.getMin());
- assertEquals(0,booleanStats3.getSum(), maxError);
- assertFalse(booleanStats3.getFirst());
- assertFalse(booleanStats3.getLast());
+ assertFalse(booleanStats3.getFirstValue());
+ assertFalse(booleanStats3.getLastValue());
booleanStats3.mergeStatistics(booleanStats2);
- assertTrue(booleanStats3.getMax());
- assertFalse(booleanStats3.getMin());
- assertEquals(0, booleanStats3.getSum(), maxError);
- assertFalse(booleanStats3.getFirst());
- assertTrue(booleanStats3.getLast());
+ assertFalse(booleanStats3.getFirstValue());
+ assertTrue(booleanStats3.getLastValue());
}
}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatisticsTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatisticsTest.java
index 1a967b7..a531071 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatisticsTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatisticsTest.java
@@ -34,11 +34,11 @@ public class DoubleStatisticsTest {
assertFalse(doubleStats.isEmpty());
doubleStats.updateStats(2.32d);
assertFalse(doubleStats.isEmpty());
- assertEquals(2.32d, doubleStats.getMax(), maxError);
- assertEquals(1.34d, doubleStats.getMin(), maxError);
- assertEquals(2.32d + 1.34d, doubleStats.getSum(), maxError);
- assertEquals(1.34d, doubleStats.getFirst(), maxError);
- assertEquals(2.32d, doubleStats.getLast(), maxError);
+ assertEquals(2.32d, doubleStats.getMaxValue(), maxError);
+ assertEquals(1.34d, doubleStats.getMinValue(), maxError);
+ assertEquals(2.32d + 1.34d, doubleStats.getSumValue(), maxError);
+ assertEquals(1.34d, doubleStats.getFirstValue(), maxError);
+ assertEquals(2.32d, doubleStats.getLastValue(), maxError);
}
@Test
@@ -54,18 +54,18 @@ public class DoubleStatisticsTest {
Statistics<Double> doubleStats3 = new DoubleStatistics();
doubleStats3.mergeStatistics(doubleStats1);
assertFalse(doubleStats3.isEmpty());
- assertEquals(100.13453d, doubleStats3.getMax(), maxError);
- assertEquals(1.34d, doubleStats3.getMin(), maxError);
- assertEquals(100.13453d + 1.34d, doubleStats3.getSum(), maxError);
- assertEquals(1.34d, doubleStats3.getFirst(), maxError);
- assertEquals(100.13453d, doubleStats3.getLast(), maxError);
+ assertEquals(100.13453d, doubleStats3.getMaxValue(), maxError);
+ assertEquals(1.34d, doubleStats3.getMinValue(), maxError);
+ assertEquals(100.13453d + 1.34d, doubleStats3.getSumValue(), maxError);
+ assertEquals(1.34d, doubleStats3.getFirstValue(), maxError);
+ assertEquals(100.13453d, doubleStats3.getLastValue(), maxError);
doubleStats3.mergeStatistics(doubleStats2);
- assertEquals(200.435d, doubleStats3.getMax(), maxError);
- assertEquals(1.34d, doubleStats3.getMin(), maxError);
- assertEquals(100.13453d + 1.34d + 200.435d, doubleStats3.getSum(), maxError);
- assertEquals(1.34d, doubleStats3.getFirst(), maxError);
- assertEquals(200.435d, doubleStats3.getLast(), maxError);
+ assertEquals(200.435d, doubleStats3.getMaxValue(), maxError);
+ assertEquals(1.34d, doubleStats3.getMinValue(), maxError);
+ assertEquals(100.13453d + 1.34d + 200.435d, doubleStats3.getSumValue(), maxError);
+ assertEquals(1.34d, doubleStats3.getFirstValue(), maxError);
+ assertEquals(200.435d, doubleStats3.getLastValue(), maxError);
}
}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatisticsTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatisticsTest.java
index c363dd1..590b462 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatisticsTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatisticsTest.java
@@ -34,11 +34,11 @@ public class FloatStatisticsTest {
assertFalse(floatStats.isEmpty());
floatStats.updateStats(2.32f);
assertFalse(floatStats.isEmpty());
- assertEquals(2.32f, (double) floatStats.getMax(), maxError);
- assertEquals(1.34f, (double) floatStats.getMin(), maxError);
- assertEquals(2.32f + 1.34f, (double) floatStats.getSum(), maxError);
- assertEquals(1.34f, (double) floatStats.getFirst(), maxError);
- assertEquals(2.32f, (double) floatStats.getLast(), maxError);
+ assertEquals(2.32f, (double) floatStats.getMaxValue(), maxError);
+ assertEquals(1.34f, (double) floatStats.getMinValue(), maxError);
+ assertEquals(2.32f + 1.34f, (double) floatStats.getSumValue(), maxError);
+ assertEquals(1.34f, (double) floatStats.getFirstValue(), maxError);
+ assertEquals(2.32f, (double) floatStats.getLastValue(), maxError);
}
@Test
@@ -54,18 +54,18 @@ public class FloatStatisticsTest {
Statistics<Float> floatStats3 = new FloatStatistics();
floatStats3.mergeStatistics(floatStats1);
assertFalse(floatStats3.isEmpty());
- assertEquals(100.13453f, floatStats3.getMax(), maxError);
- assertEquals(1.34f, floatStats3.getMin(), maxError);
- assertEquals(100.13453f + 1.34f, (float) floatStats3.getSum(), maxError);
- assertEquals(1.34f, floatStats3.getFirst(), maxError);
- assertEquals(100.13453f, floatStats3.getLast(), maxError);
+ assertEquals(100.13453f, floatStats3.getMaxValue(), maxError);
+ assertEquals(1.34f, floatStats3.getMinValue(), maxError);
+ assertEquals(100.13453f + 1.34f, (float) floatStats3.getSumValue(), maxError);
+ assertEquals(1.34f, floatStats3.getFirstValue(), maxError);
+ assertEquals(100.13453f, floatStats3.getLastValue(), maxError);
floatStats3.mergeStatistics(floatStats2);
- assertEquals(200.435d, floatStats3.getMax(), maxError);
- assertEquals(1.34d, floatStats3.getMin(), maxError);
- assertEquals(100.13453f + 1.34f + 200.435d, (float) floatStats3.getSum(), maxError);
- assertEquals(1.34f, floatStats3.getFirst(), maxError);
- assertEquals(200.435f, floatStats3.getLast(), maxError);
+ assertEquals(200.435d, floatStats3.getMaxValue(), maxError);
+ assertEquals(1.34d, floatStats3.getMinValue(), maxError);
+ assertEquals(100.13453f + 1.34f + 200.435d, (float) floatStats3.getSumValue(), maxError);
+ assertEquals(1.34f, floatStats3.getFirstValue(), maxError);
+ assertEquals(200.435f, floatStats3.getLastValue(), maxError);
}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatisticsTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatisticsTest.java
index ba710e6..b9552c6 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatisticsTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatisticsTest.java
@@ -32,11 +32,11 @@ public class IntegerStatisticsTest {
assertFalse(intStats.isEmpty());
intStats.updateStats(2);
assertFalse(intStats.isEmpty());
- assertEquals(2, (int) intStats.getMax());
- assertEquals(1, (int) intStats.getMin());
- assertEquals(1, (int) intStats.getFirst());
- assertEquals(3, (int) intStats.getSum());
- assertEquals(2, (int) intStats.getLast());
+ assertEquals(2, (int) intStats.getMaxValue());
+ assertEquals(1, (int) intStats.getMinValue());
+ assertEquals(1, (int) intStats.getFirstValue());
+ assertEquals(3, (int) intStats.getSumValue());
+ assertEquals(2, (int) intStats.getLastValue());
}
@Test
@@ -52,17 +52,17 @@ public class IntegerStatisticsTest {
Statistics<Integer> intStats3 = new IntegerStatistics();
intStats3.mergeStatistics(intStats1);
assertFalse(intStats3.isEmpty());
- assertEquals(100, (int) intStats3.getMax());
- assertEquals(1, (int) intStats3.getMin());
- assertEquals(1, (int) intStats3.getFirst());
- assertEquals(1 + 100, (int) intStats3.getSum());
- assertEquals(100, (int) intStats3.getLast());
+ assertEquals(100, (int) intStats3.getMaxValue());
+ assertEquals(1, (int) intStats3.getMinValue());
+ assertEquals(1, (int) intStats3.getFirstValue());
+ assertEquals(1 + 100, (int) intStats3.getSumValue());
+ assertEquals(100, (int) intStats3.getLastValue());
intStats3.mergeStatistics(intStats2);
- assertEquals(200, (int) intStats3.getMax());
- assertEquals(1, (int) intStats3.getMin());
- assertEquals(1, (int) intStats3.getFirst());
- assertEquals(101 + 200, (int) intStats3.getSum());
- assertEquals(200, (int) intStats3.getLast());
+ assertEquals(200, (int) intStats3.getMaxValue());
+ assertEquals(1, (int) intStats3.getMinValue());
+ assertEquals(1, (int) intStats3.getFirstValue());
+ assertEquals(101 + 200, (int) intStats3.getSumValue());
+ assertEquals(200, (int) intStats3.getLastValue());
}
}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatisticsTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatisticsTest.java
index b49de22..2696eb8 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatisticsTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatisticsTest.java
@@ -37,11 +37,11 @@ public class LongStatisticsTest {
assertFalse(longStats.isEmpty());
longStats.updateStats(secondValue);
assertFalse(longStats.isEmpty());
- assertEquals(secondValue, (long) longStats.getMax());
- assertEquals(firstValue, (long) longStats.getMin());
- assertEquals(firstValue, (long) longStats.getFirst());
- assertEquals(firstValue + secondValue, (long) longStats.getSum());
- assertEquals(secondValue, (long) longStats.getLast());
+ assertEquals(secondValue, (long) longStats.getMaxValue());
+ assertEquals(firstValue, (long) longStats.getMinValue());
+ assertEquals(firstValue, (long) longStats.getFirstValue());
+ assertEquals(firstValue + secondValue, (long) longStats.getSumValue());
+ assertEquals(secondValue, (long) longStats.getLastValue());
}
@Test
@@ -59,18 +59,18 @@ public class LongStatisticsTest {
Statistics<Long> longStats3 = new LongStatistics();
longStats3.mergeStatistics(longStats1);
assertFalse(longStats3.isEmpty());
- assertEquals(max1, (long) longStats3.getMax());
- assertEquals(1, (long) longStats3.getMin());
- assertEquals(max1 + 1, (long) longStats3.getSum());
- assertEquals(1, (long) longStats3.getFirst());
- assertEquals(max1, (long) longStats3.getLast());
+ assertEquals(max1, (long) longStats3.getMaxValue());
+ assertEquals(1, (long) longStats3.getMinValue());
+ assertEquals(max1 + 1, (long) longStats3.getSumValue());
+ assertEquals(1, (long) longStats3.getFirstValue());
+ assertEquals(max1, (long) longStats3.getLastValue());
longStats3.mergeStatistics(longStats2);
- assertEquals(max2, (long) longStats3.getMax());
- assertEquals(1, (long) longStats3.getMin());
- assertEquals(max2 + max1 + 1, (long) longStats3.getSum());
- assertEquals(1, (long) longStats3.getFirst());
- assertEquals(max2, (long) longStats3.getLast());
+ assertEquals(max2, (long) longStats3.getMaxValue());
+ assertEquals(1, (long) longStats3.getMinValue());
+ assertEquals(max2 + max1 + 1, (long) longStats3.getSumValue());
+ assertEquals(1, (long) longStats3.getFirstValue());
+ assertEquals(max2, (long) longStats3.getLastValue());
// Test mismatch
IntegerStatistics intStats5 = new IntegerStatistics();
@@ -83,13 +83,13 @@ public class LongStatisticsTest {
fail();
}
- assertEquals(max2, (long) longStats3.getMax());
+ assertEquals(max2, (long) longStats3.getMaxValue());
// if not merge, the min value will not be changed by smaller value in
// intStats5
- assertEquals(1, (long) longStats3.getMin());
- assertEquals(max2 + max1 + 1, (long) longStats3.getSum());
- assertEquals(1, (long) longStats3.getFirst());
- assertEquals(max2, (long) longStats3.getLast());
+ assertEquals(1, (long) longStats3.getMinValue());
+ assertEquals(max2 + max1 + 1, (long) longStats3.getSumValue());
+ assertEquals(1, (long) longStats3.getFirstValue());
+ assertEquals(max2, (long) longStats3.getLastValue());
}
}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/StringStatisticsTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/StringStatisticsTest.java
index 068e737..88094ae 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/StringStatisticsTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/StringStatisticsTest.java
@@ -26,8 +26,6 @@ import org.junit.Test;
public class StringStatisticsTest {
- private static final double maxError = 0.0001d;
-
@Test
public void testUpdate() {
Statistics<Binary> binaryStats = new BinaryStatistics();
@@ -35,11 +33,8 @@ public class StringStatisticsTest {
assertFalse(binaryStats.isEmpty());
binaryStats.updateStats(new Binary("bbb"));
assertFalse(binaryStats.isEmpty());
- assertEquals("bbb", binaryStats.getMax().getStringValue());
- assertEquals("aaa", binaryStats.getMin().getStringValue());
- assertEquals(0, binaryStats.getSum(), maxError);
- assertEquals("aaa", binaryStats.getFirst().getStringValue());
- assertEquals("bbb", binaryStats.getLast().getStringValue());
+ assertEquals("aaa", binaryStats.getFirstValue().getStringValue());
+ assertEquals("bbb", binaryStats.getLastValue().getStringValue());
}
@Test
@@ -55,17 +50,11 @@ public class StringStatisticsTest {
Statistics<Binary> stringStats3 = new BinaryStatistics();
stringStats3.mergeStatistics(stringStats1);
assertFalse(stringStats3.isEmpty());
- assertEquals("ccc", stringStats3.getMax().getStringValue());
- assertEquals("aaa", stringStats3.getMin().getStringValue());
- assertEquals(0, stringStats3.getSum(), maxError);
- assertEquals("aaa", stringStats3.getFirst().getStringValue());
- assertEquals("ccc", stringStats3.getLast().getStringValue());
+ assertEquals("aaa", stringStats3.getFirstValue().getStringValue());
+ assertEquals("ccc", stringStats3.getLastValue().getStringValue());
stringStats3.mergeStatistics(stringStats2);
- assertEquals("ddd", (String) stringStats3.getMax().getStringValue());
- assertEquals("aaa", (String) stringStats3.getMin().getStringValue());
- assertEquals(0, stringStats3.getSum(), maxError);
- assertEquals("aaa", (String) stringStats3.getFirst().getStringValue());
- assertEquals("ddd", stringStats3.getLast().getStringValue());
+ assertEquals("aaa", (String) stringStats3.getFirstValue().getStringValue());
+ assertEquals("ddd", stringStats3.getLastValue().getStringValue());
}
}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/TestHelper.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/TestHelper.java
index afba538..a81aa30 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/TestHelper.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/TestHelper.java
@@ -18,38 +18,21 @@
*/
package org.apache.iotdb.tsfile.file.metadata.utils;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
import org.apache.iotdb.tsfile.file.header.PageHeader;
import org.apache.iotdb.tsfile.file.header.PageHeaderTest;
-import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetaData;
-import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetaDataTest;
-import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
-import org.apache.iotdb.tsfile.file.metadata.ChunkMetaDataTest;
import org.apache.iotdb.tsfile.file.metadata.TimeSeriesMetadataTest;
-import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadata;
import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadataIndex;
-import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadataTest;
import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData;
import org.apache.iotdb.tsfile.file.metadata.TsFileMetaDataTest;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
-import org.apache.iotdb.tsfile.utils.BytesUtils;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
public class TestHelper {
- private static final String MAX_VALUE = "321";
- private static final String MIN_VALUE = "123";
- private static final String SUM_VALUE = "321123";
- private static final String FIRST_VALUE = "1";
- private static final String LAST_VALUE = "222";
-
public static TsFileMetaData createSimpleFileMetaData() {
TsFileMetaData metaData = new TsFileMetaData(generateDeviceIndexMetadataMap(), new HashMap<>());
metaData.addMeasurementSchema(TestHelper.createSimpleMeasurementSchema());
@@ -58,7 +41,7 @@ public class TestHelper {
return metaData;
}
- public static Map<String, TsDeviceMetadataIndex> generateDeviceIndexMetadataMap() {
+ private static Map<String, TsDeviceMetadataIndex> generateDeviceIndexMetadataMap() {
Map<String, TsDeviceMetadataIndex> indexMap = new HashMap<>();
for (int i = 0; i < 5; i++) {
indexMap.put("device_" + i, createSimpleDeviceIndexMetadata());
@@ -66,15 +49,7 @@ public class TestHelper {
return indexMap;
}
- public static Map<String, TsDeviceMetadata> generateDeviceMetadataMap() {
- Map<String, TsDeviceMetadata> deviceMetadataMap = new HashMap<>();
- for (int i = 0; i < 5; i++) {
- deviceMetadataMap.put("device_" + i, createSimpleDeviceMetaData());
- }
- return deviceMetadataMap;
- }
-
- public static TsDeviceMetadataIndex createSimpleDeviceIndexMetadata() {
+ private static TsDeviceMetadataIndex createSimpleDeviceIndexMetadata() {
TsDeviceMetadataIndex index = new TsDeviceMetadataIndex();
index.setOffset(0);
index.setLen(10);
@@ -83,92 +58,14 @@ public class TestHelper {
return index;
}
- public static TsDeviceMetadata createSimpleDeviceMetaData() {
- TsDeviceMetadata metaData = new TsDeviceMetadata();
- metaData.setStartTime(TsDeviceMetadataTest.START_TIME);
- metaData.setEndTime(TsDeviceMetadataTest.END_TIME);
- metaData.addChunkGroupMetaData(TestHelper.createSimpleChunkGroupMetaData());
- metaData.addChunkGroupMetaData(TestHelper.createSimpleChunkGroupMetaData());
- return metaData;
- }
-
- public static ChunkGroupMetaData createEmptySeriesChunkGroupMetaData() {
- return new ChunkGroupMetaData("d1", new ArrayList<>(), 0);
- }
-
- public static ChunkGroupMetaData createSimpleChunkGroupMetaData() {
- ChunkGroupMetaData metaData = new ChunkGroupMetaData(ChunkGroupMetaDataTest.DELTA_OBJECT_UID,
- new ArrayList<>(), 0);
- metaData.addTimeSeriesChunkMetaData(TestHelper.createSimpleTimeSeriesChunkMetaData());
- metaData.addTimeSeriesChunkMetaData(TestHelper.createSimpleTimeSeriesChunkMetaData());
- return metaData;
- }
-
- public static ChunkMetaData createSimpleTimeSeriesChunkMetaData() {
- ChunkMetaData metaData = new ChunkMetaData(ChunkMetaDataTest.MEASUREMENT_UID,
- ChunkMetaDataTest.DATA_TYPE,
- ChunkMetaDataTest.FILE_OFFSET, ChunkMetaDataTest.START_TIME, ChunkMetaDataTest.END_TIME// ,
- // ChunkMetaDataTest.ENCODING_TYPE
- );
- metaData.setNumOfPoints(ChunkMetaDataTest.NUM_OF_POINTS);
- metaData.setDigest(Statistics.getStatsByType(ChunkMetaDataTest.DATA_TYPE));
- return metaData;
- }
-
- public static ChunkMetaData createNotCompleteSimpleTimeSeriesChunkMetaData() throws IOException {
- ChunkMetaData metaData = new ChunkMetaData(ChunkMetaDataTest.MEASUREMENT_UID,
- ChunkMetaDataTest.DATA_TYPE,
- ChunkMetaDataTest.FILE_OFFSET, ChunkMetaDataTest.START_TIME, ChunkMetaDataTest.END_TIME
- );
- metaData.setNumOfPoints(ChunkMetaDataTest.NUM_OF_POINTS);
- metaData.setDigest(createNotCompleteSimpleTsDigest(ChunkMetaDataTest.DATA_TYPE));
- return metaData;
- }
-
public static MeasurementSchema createSimpleMeasurementSchema() {
return new MeasurementSchema(TimeSeriesMetadataTest.measurementUID,
TSDataType.INT64,
TSEncoding.RLE);
}
- public static Statistics createSimpleTsDigest(TSDataType dataType) throws IOException {
- Statistics digest = Statistics.getStatsByType(dataType);
- ByteBuffer[] statisticsArray = new ByteBuffer[Statistics.StatisticType.getTotalTypeNum()];
- statisticsArray[Statistics.StatisticType.min_value.ordinal()] = ByteBuffer
- .wrap(BytesUtils.stringToBytes(MIN_VALUE));
- statisticsArray[Statistics.StatisticType.max_value.ordinal()] = ByteBuffer
- .wrap(BytesUtils.stringToBytes(MAX_VALUE));
- statisticsArray[Statistics.StatisticType.first_value.ordinal()] = ByteBuffer
- .wrap(BytesUtils.stringToBytes(FIRST_VALUE));
- statisticsArray[Statistics.StatisticType.last_value.ordinal()] = ByteBuffer
- .wrap(BytesUtils.stringToBytes(LAST_VALUE));
- statisticsArray[Statistics.StatisticType.sum_value.ordinal()] = ByteBuffer
- .wrap(BytesUtils.stringToBytes(SUM_VALUE));
- digest.setStatistics(statisticsArray);
- return digest;
- }
-
- public static Statistics createNotCompleteSimpleTsDigest(TSDataType dataType) throws IOException {
- Statistics digest = Statistics.getStatsByType(dataType);
- ByteBuffer[] statisticsArray = new ByteBuffer[Statistics.StatisticType.getTotalTypeNum()];
- statisticsArray[Statistics.StatisticType.first_value.ordinal()] = ByteBuffer
- .wrap(BytesUtils.stringToBytes(FIRST_VALUE));
- statisticsArray[Statistics.StatisticType.last_value.ordinal()] = ByteBuffer
- .wrap(BytesUtils.stringToBytes(LAST_VALUE));
- statisticsArray[Statistics.StatisticType.sum_value.ordinal()] = ByteBuffer
- .wrap(BytesUtils.stringToBytes(SUM_VALUE));
- digest.setStatistics(statisticsArray);
- return digest;
- }
-
- public static List<String> getJSONArray() {
- List<String> jsonMetaData = new ArrayList<String>();
- jsonMetaData.add("fsdfsfsd");
- jsonMetaData.add("424fd");
- return jsonMetaData;
- }
- public static PageHeader createSimplePageHeader() {
+ public static PageHeader createTestPageHeader() {
Statistics<?> statistics = Statistics.getStatsByType(PageHeaderTest.DATA_TYPE);
statistics.setEmpty(false);
return new PageHeader(PageHeaderTest.UNCOMPRESSED_SIZE,
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/Utils.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/Utils.java
index e59d541..09ef7d8 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/Utils.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/Utils.java
@@ -71,13 +71,13 @@ public class Utils {
}
}
- public static void isTwoTsDigestEqual(Statistics digestA, Statistics digestB, String name) {
- if ((digestA == null) ^ (digestB == null)) {
+ public static void isTwoTsDigestEqual(Statistics statisticsA, Statistics statisticsB, String name) {
+ if ((statisticsA == null) ^ (statisticsB == null)) {
System.out.println("error");
fail(String.format("one of %s is null", name));
}
- if (digestA != null) {
- Assert.assertEquals(digestA, digestB);
+ if (statisticsA != null) {
+ Assert.assertEquals(statisticsA, statisticsB);
}
}
@@ -118,9 +118,9 @@ public class Utils {
assertTrue(metadata1.getNumOfPoints() == metadata2.getNumOfPoints());
assertTrue(metadata1.getStartTime() == metadata2.getStartTime());
assertTrue(metadata1.getEndTime() == metadata2.getEndTime());
- assertNotNull(metadata1.getDigest());
- assertNotNull(metadata2.getDigest());
- Utils.isTwoTsDigestEqual(metadata1.getDigest(), metadata2.getDigest(), "TsDigest");
+ assertNotNull(metadata1.getStatistics());
+ assertNotNull(metadata2.getStatistics());
+ Utils.isTwoTsDigestEqual(metadata1.getStatistics(), metadata2.getStatistics(), "TsDigest");
}
}
@@ -206,8 +206,8 @@ public class Utils {
assertTrue(header1.getUncompressedSize() == header2.getUncompressedSize());
assertTrue(header1.getCompressedSize() == header2.getCompressedSize());
assertTrue(header1.getNumOfValues() == header2.getNumOfValues());
- assertTrue(header1.getMaxTimestamp() == header2.getMaxTimestamp());
- assertTrue(header1.getMinTimestamp() == header2.getMinTimestamp());
+ assertTrue(header1.getEndTime() == header2.getEndTime());
+ assertTrue(header1.getStartTime() == header2.getStartTime());
if (Utils
.isTwoObjectsNotNULL(header1.getStatistics(), header2.getStatistics(), "statistics")) {
Utils.isStatisticsEqual(header1.getStatistics(), header2.getStatistics());
@@ -225,11 +225,11 @@ public class Utils {
fail("one of statistics is empty while the other one is not");
}
if (!statistics1.isEmpty() && !statistics2.isEmpty()) {
- assertEquals(statistics1.getMin(), statistics2.getMin());
- assertEquals(statistics1.getMax(), statistics2.getMax());
- assertEquals(statistics1.getFirst(), statistics2.getFirst());
- assertEquals(statistics1.getSum(), statistics2.getSum(), maxError);
- assertEquals(statistics1.getLast(), statistics2.getLast());
+ assertEquals(statistics1.getMinValue(), statistics2.getMinValue());
+ assertEquals(statistics1.getMaxValue(), statistics2.getMaxValue());
+ assertEquals(statistics1.getFirstValue(), statistics2.getFirstValue());
+ assertEquals(statistics1.getSumValue(), statistics2.getSumValue(), maxError);
+ assertEquals(statistics1.getLastValue(), statistics2.getLastValue());
}
}
}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/ReadInPartitionTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/ReadInPartitionTest.java
index 110687e..94b08f2 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/ReadInPartitionTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/ReadInPartitionTest.java
@@ -68,8 +68,8 @@ public class ReadInPartitionTest {
@Before
- public void before() throws InterruptedException, WriteProcessException, IOException {
- TsFileGeneratorForTest.generateFile(1000000, 1024 * 1024, 10000);
+ public void before() throws IOException {
+ TsFileGeneratorForTest.generateFile(10000, 1024, 100);
TsFileSequenceReader reader = new TsFileSequenceReader(FILE_PATH);
roTsFile = new ReadOnlyTsFile(reader);
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/ReadOnlyTsFileTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/ReadOnlyTsFileTest.java
index 7bf2010..7f64946 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/ReadOnlyTsFileTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/ReadOnlyTsFileTest.java
@@ -48,7 +48,7 @@ public class ReadOnlyTsFileTest {
private ReadOnlyTsFile tsFile;
@Test
- public void test1() throws InterruptedException, WriteProcessException, IOException {
+ public void test1() throws IOException {
TSFileDescriptor.getInstance().getConfig().setTimeEncoder("TS_2DIFF");
int rowCount = 1000;
TsFileGeneratorForTest.generateFile(rowCount, 16 * 1024 * 1024, 10000);
@@ -156,7 +156,5 @@ public class ReadOnlyTsFileTest {
}
}
-
-
}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/ReadTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/ReadTest.java
index b25acda..a514d74 100755
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/ReadTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/ReadTest.java
@@ -48,7 +48,7 @@ public class ReadTest {
private static ReadOnlyTsFile roTsFile = null;
@Before
- public void prepare() throws IOException, InterruptedException, WriteProcessException {
+ public void prepare() throws IOException {
FileGenerator.generateFile(1000, 100);
TsFileSequenceReader reader = new TsFileSequenceReader(fileName);
roTsFile = new ReadOnlyTsFile(reader);
@@ -275,13 +275,10 @@ public class ReadTest {
public void queryStringTest() throws IOException {
List<Path> pathList = new ArrayList<>();
pathList.add(new Path("d1.s4"));
- IExpression valFilter = new SingleSeriesExpression(new Path("d1.s4"),
- ValueFilter.gt(new Binary("dog97")));
IExpression tFilter = BinaryExpression
.and(new GlobalTimeExpression(TimeFilter.gtEq(1480562618970L)),
new GlobalTimeExpression(TimeFilter.ltEq(1480562618981L)));
- IExpression finalFilter = BinaryExpression.and(valFilter, tFilter);
- QueryExpression queryExpression = QueryExpression.create(pathList, finalFilter);
+ QueryExpression queryExpression = QueryExpression.create(pathList, tFilter);
QueryDataSet dataSet = roTsFile.query(queryExpression);
int cnt = 0;
@@ -292,18 +289,16 @@ public class ReadTest {
Field f1 = r.getFields().get(0);
assertEquals("dog976", f1.toString());
}
- // System.out.println(r);
+ System.out.println(r);
cnt++;
}
Assert.assertEquals(1, cnt);
pathList = new ArrayList<>();
pathList.add(new Path("d1.s4"));
- valFilter = new SingleSeriesExpression(new Path("d1.s4"), ValueFilter.lt(new Binary("dog97")));
tFilter = BinaryExpression.and(new GlobalTimeExpression(TimeFilter.gtEq(1480562618970L)),
new GlobalTimeExpression(TimeFilter.ltEq(1480562618981L)));
- finalFilter = BinaryExpression.and(valFilter, tFilter);
- queryExpression = QueryExpression.create(pathList, finalFilter);
+ queryExpression = QueryExpression.create(pathList, tFilter);
dataSet = roTsFile.query(queryExpression);
cnt = 0;
while (dataSet.hasNext()) {
@@ -313,10 +308,9 @@ public class ReadTest {
Field f1 = r.getFields().get(0);
assertEquals("dog976", f1.getBinaryV().getStringValue());
}
- // System.out.println(r);
cnt++;
}
- Assert.assertEquals(0, cnt);
+ Assert.assertEquals(1, cnt);
}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TsFileSequenceReaderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TsFileSequenceReaderTest.java
index c149aaa..62e1455 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TsFileSequenceReaderTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TsFileSequenceReaderTest.java
@@ -46,8 +46,8 @@ public class TsFileSequenceReaderTest {
private ReadOnlyTsFile tsFile;
@Before
- public void before() throws InterruptedException, WriteProcessException, IOException {
- int rowCount = 1000;
+ public void before() throws IOException {
+ int rowCount = 100;
FileGenerator.generateFile(rowCount, 10000);
TsFileSequenceReader fileReader = new TsFileSequenceReader(FILE_PATH);
tsFile = new ReadOnlyTsFile(fileReader);
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderTest.java
index 32ebba9..19a8416 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderTest.java
@@ -20,7 +20,6 @@ package org.apache.iotdb.tsfile.read.controller;
import java.io.IOException;
import java.util.List;
-import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
import org.apache.iotdb.tsfile.file.header.ChunkHeader;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
@@ -38,7 +37,7 @@ public class ChunkLoaderTest {
private TsFileSequenceReader fileReader;
@Before
- public void before() throws InterruptedException, WriteProcessException, IOException {
+ public void before() throws IOException {
TsFileGeneratorForTest.generateFile(1000000, 1024 * 1024, 10000);
}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/controller/IMetadataQuerierByFileImplTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/controller/IMetadataQuerierByFileImplTest.java
index cfb4a13..9b14567 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/controller/IMetadataQuerierByFileImplTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/controller/IMetadataQuerierByFileImplTest.java
@@ -46,8 +46,8 @@ public class IMetadataQuerierByFileImplTest {
private ArrayList<long[]> d2chunkGroupMetaDataOffsetList = new ArrayList<>();
@Before
- public void before() throws InterruptedException, WriteProcessException, IOException {
- TsFileGeneratorForTest.generateFile(1000000, 1024 * 1024, 10000);
+ public void before() throws IOException {
+ TsFileGeneratorForTest.generateFile(10000, 1024, 100);
reader = new TsFileSequenceReader(FILE_PATH);
// Because the size of the generated chunkGroupMetaData may differ under different test environments,
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/DigestFilterTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/DigestFilterTest.java
deleted file mode 100644
index 68b0f0e..0000000
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/DigestFilterTest.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.tsfile.read.filter;
-
-import java.nio.ByteBuffer;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.read.filter.basic.Filter;
-import org.apache.iotdb.tsfile.read.filter.factory.FilterFactory;
-import org.apache.iotdb.tsfile.utils.BytesUtils;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class DigestFilterTest {
-
- private DigestForFilter digest1 = new DigestForFilter(1L, 100L, BytesUtils.intToBytes(1),
- BytesUtils.intToBytes(100), TSDataType.INT32);
- private DigestForFilter digest2 = new DigestForFilter(101L, 200L, BytesUtils.intToBytes(101),
- BytesUtils.intToBytes(200), TSDataType.INT32);
- private DigestForFilter digest3 = new DigestForFilter(101L, 200L, (ByteBuffer) null, null,
- TSDataType.INT32);
-
- @Test
- public void testEq() {
- Filter timeEq = TimeFilter.eq(10L);
- Assert.assertTrue(timeEq.satisfy(digest1));
- Assert.assertFalse(timeEq.satisfy(digest2));
- Assert.assertFalse(timeEq.satisfy(digest3));
-
- Filter valueEq = ValueFilter.eq(100);
- Assert.assertTrue(valueEq.satisfy(digest1));
- Assert.assertFalse(valueEq.satisfy(digest2));
- Assert.assertTrue(valueEq.satisfy(digest3));
- }
-
- @Test
- public void testGt() {
- Filter timeGt = TimeFilter.gt(100L);
- Assert.assertFalse(timeGt.satisfy(digest1));
- Assert.assertTrue(timeGt.satisfy(digest2));
- Assert.assertTrue(timeGt.satisfy(digest3));
-
- Filter valueGt = ValueFilter.gt(100);
- Assert.assertFalse(valueGt.satisfy(digest1));
- Assert.assertTrue(valueGt.satisfy(digest2));
- Assert.assertTrue(valueGt.satisfy(digest3));
- }
-
- @Test
- public void testGtEq() {
- Filter timeGtEq = TimeFilter.gtEq(100L);
- Assert.assertTrue(timeGtEq.satisfy(digest1));
- Assert.assertTrue(timeGtEq.satisfy(digest2));
- Assert.assertTrue(timeGtEq.satisfy(digest3));
-
- Filter valueGtEq = ValueFilter.gtEq(100);
- Assert.assertTrue(valueGtEq.satisfy(digest1));
- Assert.assertTrue(valueGtEq.satisfy(digest3));
- Assert.assertTrue(valueGtEq.satisfy(digest3));
- }
-
- @Test
- public void testLt() {
- Filter timeLt = TimeFilter.lt(101L);
- Assert.assertTrue(timeLt.satisfy(digest1));
- Assert.assertFalse(timeLt.satisfy(digest2));
- Assert.assertFalse(timeLt.satisfy(digest3));
-
- Filter valueLt = ValueFilter.lt(101);
- Assert.assertTrue(valueLt.satisfy(digest1));
- Assert.assertFalse(valueLt.satisfy(digest2));
- Assert.assertTrue(valueLt.satisfy(digest3));
- }
-
- @Test
- public void testLtEq() {
- Filter timeLtEq = TimeFilter.ltEq(101L);
- Assert.assertTrue(timeLtEq.satisfy(digest1));
- Assert.assertTrue(timeLtEq.satisfy(digest2));
- Assert.assertTrue(timeLtEq.satisfy(digest3));
-
- Filter valueLtEq = ValueFilter.ltEq(101);
- Assert.assertTrue(valueLtEq.satisfy(digest1));
- Assert.assertTrue(valueLtEq.satisfy(digest2));
- Assert.assertTrue(valueLtEq.satisfy(digest3));
- }
-
- @Test
- public void testAndOr() {
- Filter andFilter = FilterFactory.and(TimeFilter.gt(10L), ValueFilter.lt(50));
- Assert.assertTrue(andFilter.satisfy(digest1));
- Assert.assertFalse(andFilter.satisfy(digest2));
- Assert.assertTrue(andFilter.satisfy(digest3));
-
- Filter orFilter = FilterFactory.or(andFilter, TimeFilter.eq(200L));
- Assert.assertTrue(orFilter.satisfy(digest1));
- Assert.assertTrue(orFilter.satisfy(digest2));
- Assert.assertTrue(orFilter.satisfy(digest3));
- }
-
-}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/StatisticsFilterTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/StatisticsFilterTest.java
new file mode 100644
index 0000000..ebfa73e
--- /dev/null
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/StatisticsFilterTest.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.tsfile.read.filter;
+
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
+import org.apache.iotdb.tsfile.read.filter.basic.Filter;
+import org.apache.iotdb.tsfile.read.filter.factory.FilterFactory;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class StatisticsFilterTest {
+
+ private Statistics statistics1 = Statistics.getStatsByType(TSDataType.INT64);
+ private Statistics statistics2 = Statistics.getStatsByType(TSDataType.INT64);
+
+ @Before
+ public void before() {
+ statistics1.update(1L, 1L);
+ statistics1.update(100L, 100L);
+ statistics2.update(101L, 101L);
+ statistics2.update(200L, 200L);
+ }
+
+ @Test
+ public void testEq() {
+ Filter timeEq = TimeFilter.eq(10L);
+ Assert.assertTrue(timeEq.satisfy(statistics1));
+ Assert.assertFalse(timeEq.satisfy(statistics2));
+
+ Filter valueEq = ValueFilter.eq(101L);
+ Assert.assertFalse(valueEq.satisfy(statistics1));
+ Assert.assertTrue(valueEq.satisfy(statistics2));
+ }
+
+ @Test
+ public void testGt() {
+ Filter timeGt = TimeFilter.gt(100L);
+ Assert.assertFalse(timeGt.satisfy(statistics1));
+ Assert.assertTrue(timeGt.satisfy(statistics2));
+
+ Filter valueGt = ValueFilter.gt(100L);
+ Assert.assertFalse(valueGt.satisfy(statistics1));
+ Assert.assertTrue(valueGt.satisfy(statistics2));
+ }
+
+ @Test
+ public void testGtEq() {
+ Filter timeGtEq = TimeFilter.gtEq(100L);
+ Assert.assertTrue(timeGtEq.satisfy(statistics1));
+ Assert.assertTrue(timeGtEq.satisfy(statistics2));
+
+ Filter valueGtEq = ValueFilter.gtEq(100L);
+ Assert.assertTrue(valueGtEq.satisfy(statistics1));
+ Assert.assertTrue(valueGtEq.satisfy(statistics2));
+ }
+
+ @Test
+ public void testLt() {
+ Filter timeLt = TimeFilter.lt(101L);
+ Assert.assertTrue(timeLt.satisfy(statistics1));
+ Assert.assertFalse(timeLt.satisfy(statistics2));
+
+ Filter valueLt = ValueFilter.lt(101L);
+ Assert.assertTrue(valueLt.satisfy(statistics1));
+ Assert.assertFalse(valueLt.satisfy(statistics2));
+ }
+
+ @Test
+ public void testLtEq() {
+ Filter timeLtEq = TimeFilter.ltEq(101L);
+ Assert.assertTrue(timeLtEq.satisfy(statistics1));
+ Assert.assertTrue(timeLtEq.satisfy(statistics2));
+
+ Filter valueLtEq = ValueFilter.ltEq(101L);
+ Assert.assertTrue(valueLtEq.satisfy(statistics1));
+ Assert.assertTrue(valueLtEq.satisfy(statistics2));
+ }
+
+ @Test
+ public void testAndOr() {
+ Filter andFilter = FilterFactory.and(TimeFilter.gt(10L), ValueFilter.lt(50L));
+ Assert.assertTrue(andFilter.satisfy(statistics1));
+ Assert.assertFalse(andFilter.satisfy(statistics2));
+
+ Filter orFilter = FilterFactory.or(andFilter, TimeFilter.eq(200L));
+ Assert.assertTrue(orFilter.satisfy(statistics1));
+ Assert.assertTrue(orFilter.satisfy(statistics2));
+ }
+
+}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/reader/ReaderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/reader/ReaderTest.java
index 8b2a2ce..551e63b 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/reader/ReaderTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/reader/ReaderTest.java
@@ -50,7 +50,7 @@ public class ReaderTest {
private int rowCount = 1000000;
@Before
- public void before() throws InterruptedException, WriteProcessException, IOException {
+ public void before() throws IOException {
TSFileDescriptor.getInstance().getConfig().setTimeEncoder("TS_2DIFF");
TsFileGeneratorForTest.generateFile(rowCount, 10 * 1024 * 1024, 10000);
fileReader = new TsFileSequenceReader(FILE_PATH);
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/FileGenerator.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/FileGenerator.java
index 0fca65f..5ef208d 100755
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/FileGenerator.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/FileGenerator.java
@@ -39,19 +39,18 @@ import org.slf4j.LoggerFactory;
public class FileGenerator {
private static final Logger LOG = LoggerFactory.getLogger(FileGenerator.class);
- public static int ROW_COUNT = 1000;
- public static TsFileWriter innerWriter;
- public static String inputDataFile;
+ private static int ROW_COUNT = 1000;
+ private static TsFileWriter innerWriter;
+ private static String inputDataFile;
public static String outputDataFile = "target/perTestOutputData.tsfile";
- public static String errorOutputDataFile;
+ private static String errorOutputDataFile;
public static Schema schema;
- public static int oldMaxNumberOfPointsInPage;
public static void generateFile(int rowCount, int maxNumberOfPointsInPage)
- throws IOException, InterruptedException, WriteProcessException {
+ throws IOException {
ROW_COUNT = rowCount;
TSFileConfig config = TSFileDescriptor.getInstance().getConfig();
- oldMaxNumberOfPointsInPage = config.getMaxNumberOfPointsInPage();
+ int oldMaxNumberOfPointsInPage = config.getMaxNumberOfPointsInPage();
config.setMaxNumberOfPointsInPage(maxNumberOfPointsInPage);
prepare();
@@ -60,14 +59,14 @@ public class FileGenerator {
}
public static void generateFile()
- throws IOException, InterruptedException, WriteProcessException {
+ throws IOException {
generateFile(1000, 10);
}
public static void prepare() throws IOException {
inputDataFile = "target/perTestInputData";
errorOutputDataFile = "target/perTestErrorOutputData.tsfile";
- generateTestData();
+ generateTestSchema();
generateSampleInputDataFile();
}
@@ -158,14 +157,14 @@ public class FileGenerator {
// write
try {
- writeToFile(schema);
+ writeToTsFile(schema);
} catch (WriteProcessException e) {
e.printStackTrace();
}
LOG.info("write to file successfully!!");
}
- private static void generateTestData() {
+ private static void generateTestSchema() {
schema = new Schema();
TSFileConfig conf = TSFileDescriptor.getInstance().getConfig();
schema.registerMeasurement(new MeasurementSchema("s1", TSDataType.INT32,
@@ -180,7 +179,7 @@ public class FileGenerator {
schema.registerMeasurement(new MeasurementSchema("s7", TSDataType.DOUBLE, TSEncoding.RLE));
}
- static public void writeToFile(Schema schema)
+ private static void writeToTsFile(Schema schema)
throws IOException, WriteProcessException {
Scanner in = getDataFile(inputDataFile);
long lineCount = 0;
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/TsFileGeneratorForTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/TsFileGeneratorForTest.java
index e695aa0..ae217e3 100755
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/TsFileGeneratorForTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/TsFileGeneratorForTest.java
@@ -52,22 +52,21 @@ public class TsFileGeneratorForTest {
public static final long START_TIMESTAMP = 1480562618000L;
private static final Logger LOG = LoggerFactory.getLogger(TsFileGeneratorForTest.class);
- public static TsFileWriter innerWriter;
- public static String inputDataFile;
+ private static String inputDataFile;
public static String outputDataFile = "target/testTsFile.tsfile";
- public static String errorOutputDataFile;
+ private static String errorOutputDataFile;
private static int rowCount;
private static int chunkGroupSize;
private static int pageSize;
private static FSFactory fsFactory = FSFactoryProducer.getFSFactory();
public static void generateFile(int rowCount, int chunkGroupSize, int pageSize)
- throws IOException, InterruptedException, WriteProcessException {
+ throws IOException {
generateFile(rowCount, rowCount, chunkGroupSize, pageSize);
}
public static void generateFile(int minRowCount, int maxRowCount,int chunkGroupSize, int pageSize)
- throws IOException, InterruptedException, WriteProcessException {
+ throws IOException {
TsFileGeneratorForTest.rowCount = maxRowCount;
TsFileGeneratorForTest.chunkGroupSize = chunkGroupSize;
TsFileGeneratorForTest.pageSize = pageSize;
@@ -163,7 +162,8 @@ public class TsFileGeneratorForTest {
TSFileDescriptor.getInstance().getConfig().setGroupSizeInByte(chunkGroupSize);
TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(pageSize);
- innerWriter = new TsFileWriter(file, schema, TSFileDescriptor.getInstance().getConfig());
+ TsFileWriter innerWriter = new TsFileWriter(file, schema,
+ TSFileDescriptor.getInstance().getConfig());
// write
try (Scanner in = new Scanner(fsFactory.getFile(inputDataFile))) {
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileIOWriterTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileIOWriterTest.java
index 6875e8d..cfde477 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileIOWriterTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileIOWriterTest.java
@@ -54,13 +54,13 @@ public class TsFileIOWriterTest {
// chunk statistics
Statistics statistics = Statistics.getStatsByType(measurementSchema.getType());
- statistics.updateStats(0L);
+ statistics.updateStats(0L, 0L);
// chunk group 1
writer.startChunkGroup(deviceId);
writer.startFlushChunk(measurementSchema, measurementSchema.getCompressor(),
- measurementSchema.getType(), measurementSchema.getEncodingType(), statistics, 0, 0, 0, 0);
- writer.endChunk(0);
+ measurementSchema.getType(), measurementSchema.getEncodingType(), statistics, 0, 0);
+ writer.endCurrentChunk();
writer.endChunkGroup(0);
// end file
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriterTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriterTest.java
index b76a2a5..83e01b4 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriterTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriterTest.java
@@ -120,7 +120,7 @@ public class RestorableTsFileIOWriterTest {
writer.getIOWriter()
.startFlushChunk(new MeasurementSchema("s1", TSDataType.FLOAT, TSEncoding.PLAIN),
CompressionType.SNAPPY, TSDataType.FLOAT,
- TSEncoding.PLAIN, new FloatStatistics(), 100, 50, 100, 10);
+ TSEncoding.PLAIN, new FloatStatistics(), 100, 10);
writer.getIOWriter().close();
RestorableTsFileIOWriter rWriter = new RestorableTsFileIOWriter(file);