You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2020/12/02 08:04:26 UTC
[iotdb] 02/11: change some
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch NewTsFile
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 314c38af5197aaf33079761021a28108e655cfbf
Author: JackieTien97 <Ja...@foxmail.com>
AuthorDate: Wed Nov 25 14:29:30 2020 +0800
change some
---
.../iotdb/tsfile/file/header/ChunkHeader.java | 4 ++--
.../iotdb/tsfile/file/metadata/TsFileMetadata.java | 27 ++--------------------
.../metadata/statistics/BooleanStatistics.java | 9 +++++++-
.../file/metadata/statistics/DoubleStatistics.java | 19 +++++++++------
.../file/metadata/statistics/FloatStatistics.java | 19 +++++++++------
.../file/metadata/statistics/LongStatistics.java | 27 ++++++++++++++--------
.../iotdb/tsfile/utils/ReadWriteIOUtils.java | 2 +-
.../write/writer/ForceAppendTsFileWriter.java | 4 +---
8 files changed, 55 insertions(+), 56 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java
index c318d44..6f9beac 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java
@@ -173,7 +173,7 @@ public class ChunkHeader {
*/
public int serializeTo(OutputStream outputStream) throws IOException {
int length = 0;
- length += ReadWriteIOUtils.write(MetaMarker.CHUNK_HEADER, outputStream);
+ length += ReadWriteIOUtils.write(chunkType, outputStream);
length += ReadWriteIOUtils.write(measurementID, outputStream);
length += ReadWriteIOUtils.write(dataSize, outputStream);
length += ReadWriteIOUtils.write(dataType, outputStream);
@@ -191,7 +191,7 @@ public class ChunkHeader {
*/
public int serializeTo(ByteBuffer buffer) {
int length = 0;
- length += ReadWriteIOUtils.write(MetaMarker.CHUNK_HEADER, buffer);
+ length += ReadWriteIOUtils.write(chunkType, buffer);
length += ReadWriteIOUtils.write(measurementID, buffer);
length += ReadWriteIOUtils.write(dataSize, buffer);
length += ReadWriteIOUtils.write(dataType, buffer);
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java
index 6098284..33c5f8d 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java
@@ -29,6 +29,7 @@ import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.utils.BloomFilter;
import org.apache.iotdb.tsfile.utils.Pair;
+import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
/**
@@ -36,13 +37,6 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
*/
public class TsFileMetadata {
- // fields below are IoTDB extensions and they does not affect TsFile's
- // stand-alone functionality
- private int totalChunkNum;
- // invalid means a chunk has been rewritten by merge and the chunk's data is in
- // another new chunk
- private int invalidChunkNum;
-
// bloom filter
private BloomFilter bloomFilter;
@@ -66,12 +60,10 @@ public class TsFileMetadata {
// metadataIndex
fileMetaData.metadataIndex = MetadataIndexNode.deserializeFrom(buffer);
- fileMetaData.totalChunkNum = ReadWriteIOUtils.readInt(buffer);
- fileMetaData.invalidChunkNum = ReadWriteIOUtils.readInt(buffer);
// versionInfo
List<Pair<Long, Long>> versionInfo = new ArrayList<>();
- int versionSize = ReadWriteIOUtils.readInt(buffer);
+ int versionSize = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
for (int i = 0; i < versionSize; i++) {
long versionPos = ReadWriteIOUtils.readLong(buffer);
long version = ReadWriteIOUtils.readLong(buffer);
@@ -166,21 +158,6 @@ public class TsFileMetadata {
return filter;
}
- public int getTotalChunkNum() {
- return totalChunkNum;
- }
-
- public void setTotalChunkNum(int totalChunkNum) {
- this.totalChunkNum = totalChunkNum;
- }
-
- public int getInvalidChunkNum() {
- return invalidChunkNum;
- }
-
- public void setInvalidChunkNum(int invalidChunkNum) {
- this.invalidChunkNum = invalidChunkNum;
- }
public long getMetaOffset() {
return metaOffset;
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatistics.java
index e24ef46..ddbdf34 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatistics.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatistics.java
@@ -200,6 +200,7 @@ public class BooleanStatistics extends Statistics<Boolean> {
int byteLen = 0;
byteLen += ReadWriteIOUtils.write(firstValue, outputStream);
byteLen += ReadWriteIOUtils.write(lastValue, outputStream);
+ byteLen += ReadWriteIOUtils.write(sumValue, outputStream);
return byteLen;
}
@@ -207,16 +208,22 @@ public class BooleanStatistics extends Statistics<Boolean> {
void deserialize(InputStream inputStream) throws IOException {
this.firstValue = ReadWriteIOUtils.readBool(inputStream);
this.lastValue = ReadWriteIOUtils.readBool(inputStream);
+ this.sumValue = ReadWriteIOUtils.readLong(inputStream);
}
@Override
void deserialize(ByteBuffer byteBuffer) {
this.firstValue = ReadWriteIOUtils.readBool(byteBuffer);
this.lastValue = ReadWriteIOUtils.readBool(byteBuffer);
+ this.sumValue = ReadWriteIOUtils.readLong(byteBuffer);
}
@Override
public String toString() {
- return super.toString() + " [firstValue:" + firstValue + ",lastValue:" + lastValue + "]";
+ return "BooleanStatistics{" +
+ "firstValue=" + firstValue +
+ ", lastValue=" + lastValue +
+ ", sumValue=" + sumValue +
+ '}';
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatistics.java
index ef7293c..aba11ba 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatistics.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatistics.java
@@ -18,14 +18,14 @@
*/
package org.apache.iotdb.tsfile.file.metadata.statistics;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.utils.BytesUtils;
-import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
-
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;
+import org.apache.iotdb.tsfile.exception.filter.StatisticsClassException;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.utils.BytesUtils;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
public class DoubleStatistics extends Statistics<Double> {
@@ -143,20 +143,25 @@ public class DoubleStatistics extends Statistics<Double> {
}
@Override
- public double getSumValue() {
+ public double getSumDoubleValue() {
return sumValue;
}
@Override
+ public long getSumLongValue() {
+ throw new StatisticsClassException("Double statistics does not support: long sum");
+ }
+
+ @Override
protected void mergeStatisticsValue(Statistics stats) {
DoubleStatistics doubleStats = (DoubleStatistics) stats;
if (this.isEmpty) {
initializeStats(doubleStats.getMinValue(), doubleStats.getMaxValue(), doubleStats.getFirstValue(),
- doubleStats.getLastValue(), doubleStats.getSumValue());
+ doubleStats.getLastValue(), doubleStats.sumValue);
isEmpty = false;
} else {
updateStats(doubleStats.getMinValue(), doubleStats.getMaxValue(), doubleStats.getFirstValue(),
- doubleStats.getLastValue(), doubleStats.getSumValue(), stats.getStartTime(), stats.getEndTime());
+ doubleStats.getLastValue(), doubleStats.sumValue, stats.getStartTime(), stats.getEndTime());
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatistics.java
index 3080fd7..106ff32 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatistics.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatistics.java
@@ -18,14 +18,14 @@
*/
package org.apache.iotdb.tsfile.file.metadata.statistics;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.utils.BytesUtils;
-import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
-
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;
+import org.apache.iotdb.tsfile.exception.filter.StatisticsClassException;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.utils.BytesUtils;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
/**
* Statistics for float type.
@@ -138,20 +138,25 @@ public class FloatStatistics extends Statistics<Float> {
}
@Override
- public double getSumValue() {
+ public double getSumDoubleValue() {
return sumValue;
}
@Override
+ public long getSumLongValue() {
+ throw new StatisticsClassException("Float statistics does not support: long sum");
+ }
+
+ @Override
protected void mergeStatisticsValue(Statistics stats) {
FloatStatistics floatStats = (FloatStatistics) stats;
if (isEmpty) {
initializeStats(floatStats.getMinValue(), floatStats.getMaxValue(), floatStats.getFirstValue(),
- floatStats.getLastValue(), floatStats.getSumValue());
+ floatStats.getLastValue(), floatStats.sumValue);
isEmpty = false;
} else {
updateStats(floatStats.getMinValue(), floatStats.getMaxValue(), floatStats.getFirstValue(),
- floatStats.getLastValue(), floatStats.getSumValue(), stats.getStartTime(), stats.getEndTime());
+ floatStats.getLastValue(), floatStats.sumValue, stats.getStartTime(), stats.getEndTime());
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatistics.java
index e4958d3..60cc881 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatistics.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatistics.java
@@ -18,14 +18,14 @@
*/
package org.apache.iotdb.tsfile.file.metadata.statistics;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.utils.BytesUtils;
-import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
-
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;
+import org.apache.iotdb.tsfile.exception.filter.StatisticsClassException;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.utils.BytesUtils;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
public class LongStatistics extends Statistics<Long> {
@@ -67,7 +67,8 @@ public class LongStatistics extends Statistics<Long> {
this.lastValue = lastValue;
}
- private void updateStats(long minValue, long maxValue, long firstValue, long lastValue, double sumValue, long startTime, long endTime) {
+ private void updateStats(long minValue, long maxValue, long firstValue, long lastValue,
+ double sumValue, long startTime, long endTime) {
if (minValue < this.minValue) {
this.minValue = minValue;
}
@@ -113,11 +114,16 @@ public class LongStatistics extends Statistics<Long> {
}
@Override
- public double getSumValue() {
+ public double getSumDoubleValue() {
return sumValue;
}
@Override
+ public long getSumLongValue() {
+ throw new StatisticsClassException("Long statistics does not support: long sum");
+ }
+
+ @Override
void updateStats(long value) {
if (isEmpty) {
initializeStats(value, value, value, value, value);
@@ -154,11 +160,11 @@ public class LongStatistics extends Statistics<Long> {
LongStatistics longStats = (LongStatistics) stats;
if (isEmpty) {
initializeStats(longStats.getMinValue(), longStats.getMaxValue(), longStats.getFirstValue(),
- longStats.getLastValue(), longStats.getSumValue());
+ longStats.getLastValue(), longStats.sumValue);
isEmpty = false;
} else {
- updateStats(longStats.getMinValue(), longStats.getMaxValue(), longStats.getFirstValue(), longStats.getLastValue(),
- longStats.getSumValue(), stats.getStartTime(), stats.getEndTime());
+ updateStats(longStats.getMinValue(), longStats.getMaxValue(), longStats.getFirstValue(),
+ longStats.getLastValue(), longStats.sumValue, stats.getStartTime(), stats.getEndTime());
}
}
@@ -244,7 +250,8 @@ public class LongStatistics extends Statistics<Long> {
@Override
public String toString() {
- return super.toString() + " [minValue:" + minValue + ",maxValue:" + maxValue + ",firstValue:" + firstValue +
+ return super.toString() + " [minValue:" + minValue + ",maxValue:" + maxValue + ",firstValue:"
+ + firstValue +
",lastValue:" + lastValue + ",sumValue:" + sumValue + "]";
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/ReadWriteIOUtils.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/ReadWriteIOUtils.java
index 45f00ae..524ee01 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/ReadWriteIOUtils.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/ReadWriteIOUtils.java
@@ -769,7 +769,7 @@ public class ReadWriteIOUtils {
* read a int + buffer
*/
public static ByteBuffer readByteBufferWithSelfDescriptionLength(ByteBuffer buffer) {
- int byteLength = readInt(buffer);
+ int byteLength = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
byte[] bytes = new byte[byteLength];
buffer.get(bytes);
ByteBuffer byteBuffer = ByteBuffer.allocate(byteLength);
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriter.java
index 9dfccf8..6625f68 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriter.java
@@ -66,9 +66,7 @@ public class ForceAppendTsFileWriter extends TsFileIOWriter {
canWrite = true;
versionInfo = tsFileMetadata.getVersionInfo();
- totalChunkNum = tsFileMetadata.getTotalChunkNum();
- invalidChunkNum = tsFileMetadata.getInvalidChunkNum();
-
+
List<String> devices = reader.getAllDevices();
for (String device : devices) {
List<ChunkMetadata> chunkMetadataList = new ArrayList<>();