You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2022/05/27 02:52:27 UTC
[iotdb] 02/02: add compression statistics
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch improvedAlign_for_expr
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 3c21eeff60977fe67576fc04b56e8a268b0b6b5d
Author: jt <jt...@163.com>
AuthorDate: Fri May 27 10:52:17 2022 +0800
add compression statistics
---
.../apache/iotdb/tsfile/write/page/PageWriter.java | 36 ++++++++++++++++++++++
.../iotdb/tsfile/write/page/TimePageWriter.java | 7 +++++
.../iotdb/tsfile/write/page/ValuePageWriter.java | 21 +++++++++++++
3 files changed, 64 insertions(+)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/PageWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/PageWriter.java
index 0401908477..097f81edbf 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/PageWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/PageWriter.java
@@ -36,6 +36,7 @@ import java.io.Serializable;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.WritableByteChannel;
+import java.util.concurrent.atomic.AtomicLong;
/**
* This writer is used to write time-value into a page. It consists of a time encoder, a value
@@ -43,6 +44,12 @@ import java.nio.channels.WritableByteChannel;
*/
public class PageWriter {
+ public static final AtomicLong timeRawSize = new AtomicLong();
+ public static final AtomicLong timeEncodedSize = new AtomicLong();
+ public static final AtomicLong valueRawSize = new AtomicLong();
+ public static final AtomicLong valueEncodedSize = new AtomicLong();
+ public static final AtomicLong compressedSize = new AtomicLong();
+
private static final Logger logger = LoggerFactory.getLogger(PageWriter.class);
private ICompressor compressor;
@@ -82,6 +89,8 @@ public class PageWriter {
timeEncoder.encode(time, timeOut);
valueEncoder.encode(value, valueOut);
statistics.update(time, value);
+ timeRawSize.addAndGet(Long.BYTES);
+ valueRawSize.addAndGet(1);
}
/** write a time value pair into encoder */
@@ -89,6 +98,8 @@ public class PageWriter {
timeEncoder.encode(time, timeOut);
valueEncoder.encode(value, valueOut);
statistics.update(time, value);
+ timeRawSize.addAndGet(Long.BYTES);
+ valueRawSize.addAndGet(Short.BYTES);
}
/** write a time value pair into encoder */
@@ -96,6 +107,8 @@ public class PageWriter {
timeEncoder.encode(time, timeOut);
valueEncoder.encode(value, valueOut);
statistics.update(time, value);
+ timeRawSize.addAndGet(Long.BYTES);
+ valueRawSize.addAndGet(Integer.BYTES);
}
/** write a time value pair into encoder */
@@ -103,6 +116,8 @@ public class PageWriter {
timeEncoder.encode(time, timeOut);
valueEncoder.encode(value, valueOut);
statistics.update(time, value);
+ timeRawSize.addAndGet(Long.BYTES);
+ valueRawSize.addAndGet(Long.BYTES);
}
/** write a time value pair into encoder */
@@ -110,6 +125,8 @@ public class PageWriter {
timeEncoder.encode(time, timeOut);
valueEncoder.encode(value, valueOut);
statistics.update(time, value);
+ timeRawSize.addAndGet(Long.BYTES);
+ valueRawSize.addAndGet(Float.BYTES);
}
/** write a time value pair into encoder */
@@ -117,6 +134,8 @@ public class PageWriter {
timeEncoder.encode(time, timeOut);
valueEncoder.encode(value, valueOut);
statistics.update(time, value);
+ timeRawSize.addAndGet(Long.BYTES);
+ valueRawSize.addAndGet(Double.BYTES);
}
/** write a time value pair into encoder */
@@ -124,6 +143,8 @@ public class PageWriter {
timeEncoder.encode(time, timeOut);
valueEncoder.encode(value, valueOut);
statistics.update(time, value);
+ timeRawSize.addAndGet(Long.BYTES);
+ valueRawSize.addAndGet(value.getLength());
}
/** write time series into encoder */
@@ -132,6 +153,8 @@ public class PageWriter {
timeEncoder.encode(timestamps[i], timeOut);
valueEncoder.encode(values[i], valueOut);
}
+ timeRawSize.addAndGet(Long.BYTES * batchSize);
+ valueRawSize.addAndGet(batchSize);
statistics.update(timestamps, values, batchSize);
}
@@ -141,6 +164,8 @@ public class PageWriter {
timeEncoder.encode(timestamps[i], timeOut);
valueEncoder.encode(values[i], valueOut);
}
+ timeRawSize.addAndGet(Long.BYTES * batchSize);
+ valueRawSize.addAndGet(Integer.BYTES * batchSize);
statistics.update(timestamps, values, batchSize);
}
@@ -150,6 +175,8 @@ public class PageWriter {
timeEncoder.encode(timestamps[i], timeOut);
valueEncoder.encode(values[i], valueOut);
}
+ timeRawSize.addAndGet(Long.BYTES * batchSize);
+ valueRawSize.addAndGet(Long.BYTES * batchSize);
statistics.update(timestamps, values, batchSize);
}
@@ -159,6 +186,8 @@ public class PageWriter {
timeEncoder.encode(timestamps[i], timeOut);
valueEncoder.encode(values[i], valueOut);
}
+ timeRawSize.addAndGet(Long.BYTES * batchSize);
+ valueRawSize.addAndGet(Float.BYTES * batchSize);
statistics.update(timestamps, values, batchSize);
}
@@ -168,6 +197,8 @@ public class PageWriter {
timeEncoder.encode(timestamps[i], timeOut);
valueEncoder.encode(values[i], valueOut);
}
+ timeRawSize.addAndGet(Long.BYTES * batchSize);
+ valueRawSize.addAndGet(Double.BYTES * batchSize);
statistics.update(timestamps, values, batchSize);
}
@@ -176,6 +207,8 @@ public class PageWriter {
for (int i = 0; i < batchSize; i++) {
timeEncoder.encode(timestamps[i], timeOut);
valueEncoder.encode(values[i], valueOut);
+ timeRawSize.addAndGet(Long.BYTES);
+ valueRawSize.addAndGet(values[i].getLength());
}
statistics.update(timestamps, values, batchSize);
}
@@ -199,6 +232,8 @@ public class PageWriter {
buffer.put(timeOut.getBuf(), 0, timeOut.size());
buffer.put(valueOut.getBuf(), 0, valueOut.size());
buffer.flip();
+ timeEncodedSize.addAndGet(timeOut.size());
+ valueEncodedSize.addAndGet(valueOut.size());
return buffer;
}
@@ -227,6 +262,7 @@ public class PageWriter {
compressor.compress(
pageData.array(), pageData.position(), uncompressedSize, compressedBytes);
}
+ PageWriter.compressedSize.addAndGet(compressedSize);
// write the page header to IOWriter
int sizeWithoutStatistic = 0;
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/TimePageWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/TimePageWriter.java
index 1c668fc881..1bb1d9bf1d 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/TimePageWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/TimePageWriter.java
@@ -32,6 +32,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.WritableByteChannel;
+import java.util.concurrent.atomic.AtomicLong;
/**
* This writer is used to write time into a page. It consists of a time encoder and respective
@@ -40,6 +41,9 @@ import java.nio.channels.WritableByteChannel;
public class TimePageWriter {
private static final Logger logger = LoggerFactory.getLogger(TimePageWriter.class);
+ public static final AtomicLong timeRawSize = new AtomicLong();
+ public static final AtomicLong timeEncodedSize = new AtomicLong();
+ public static final AtomicLong timeCompressedSize = new AtomicLong();
private final ICompressor compressor;
@@ -71,6 +75,7 @@ public class TimePageWriter {
for (int i = 0; i < batchSize; i++) {
timeEncoder.encode(timestamps[i], timeOut);
}
+ timeRawSize.addAndGet(batchSize * Long.BYTES);
statistics.update(timestamps, batchSize);
}
@@ -90,6 +95,7 @@ public class TimePageWriter {
ByteBuffer buffer = ByteBuffer.allocate(timeOut.size());
buffer.put(timeOut.getBuf(), 0, timeOut.size());
buffer.flip();
+ timeEncodedSize.addAndGet(timeOut.size());
return buffer;
}
@@ -118,6 +124,7 @@ public class TimePageWriter {
compressor.compress(
pageData.array(), pageData.position(), uncompressedSize, compressedBytes);
}
+ timeCompressedSize.addAndGet(compressedSize);
// write the page header to IOWriter
int sizeWithoutStatistic = 0;
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/ValuePageWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/ValuePageWriter.java
index 988575aa2c..aca4a450ad 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/ValuePageWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/ValuePageWriter.java
@@ -35,12 +35,18 @@ import java.io.Serializable;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.WritableByteChannel;
+import java.util.concurrent.atomic.AtomicLong;
/**
* This writer is used to write value into a page. It consists of a value encoder and respective
* OutputStream.
*/
public class ValuePageWriter {
+
+ public static final AtomicLong valueRawSize = new AtomicLong();
+ public static final AtomicLong valueEncodedSize = new AtomicLong();
+ public static final AtomicLong valueCompressedSize = new AtomicLong();
+
private static final Logger logger = LoggerFactory.getLogger(ValuePageWriter.class);
private final ICompressor compressor;
@@ -77,6 +83,7 @@ public class ValuePageWriter {
public void write(long time, boolean value, boolean isNull) {
setBit(isNull);
if (!isNull) {
+ valueRawSize.addAndGet(1);
valueEncoder.encode(value, valueOut);
statistics.update(time, value);
}
@@ -86,6 +93,7 @@ public class ValuePageWriter {
public void write(long time, short value, boolean isNull) {
setBit(isNull);
if (!isNull) {
+ valueRawSize.addAndGet(Short.BYTES);
valueEncoder.encode(value, valueOut);
statistics.update(time, value);
}
@@ -95,6 +103,7 @@ public class ValuePageWriter {
public void write(long time, int value, boolean isNull) {
setBit(isNull);
if (!isNull) {
+ valueRawSize.addAndGet(Integer.BYTES);
valueEncoder.encode(value, valueOut);
statistics.update(time, value);
}
@@ -104,6 +113,7 @@ public class ValuePageWriter {
public void write(long time, long value, boolean isNull) {
setBit(isNull);
if (!isNull) {
+ valueRawSize.addAndGet(Long.BYTES);
valueEncoder.encode(value, valueOut);
statistics.update(time, value);
}
@@ -113,6 +123,7 @@ public class ValuePageWriter {
public void write(long time, float value, boolean isNull) {
setBit(isNull);
if (!isNull) {
+ valueRawSize.addAndGet(Float.BYTES);
valueEncoder.encode(value, valueOut);
statistics.update(time, value);
}
@@ -122,6 +133,7 @@ public class ValuePageWriter {
public void write(long time, double value, boolean isNull) {
setBit(isNull);
if (!isNull) {
+ valueRawSize.addAndGet(Double.BYTES);
valueEncoder.encode(value, valueOut);
statistics.update(time, value);
}
@@ -131,6 +143,7 @@ public class ValuePageWriter {
public void write(long time, Binary value, boolean isNull) {
setBit(isNull);
if (!isNull) {
+ valueRawSize.addAndGet(value.getLength());
valueEncoder.encode(value, valueOut);
statistics.update(time, value);
}
@@ -152,6 +165,7 @@ public class ValuePageWriter {
for (int i = 0; i < batchSize; i++) {
valueEncoder.encode(values[i], valueOut);
}
+ valueRawSize.addAndGet(batchSize);
statistics.update(timestamps, values, batchSize);
}
@@ -160,6 +174,7 @@ public class ValuePageWriter {
for (int i = 0; i < batchSize; i++) {
valueEncoder.encode(values[i], valueOut);
}
+ valueRawSize.addAndGet(Integer.BYTES * batchSize);
statistics.update(timestamps, values, batchSize);
}
@@ -168,6 +183,7 @@ public class ValuePageWriter {
for (int i = 0; i < batchSize; i++) {
valueEncoder.encode(values[i], valueOut);
}
+ valueRawSize.addAndGet(Long.BYTES * batchSize);
statistics.update(timestamps, values, batchSize);
}
@@ -176,6 +192,7 @@ public class ValuePageWriter {
for (int i = 0; i < batchSize; i++) {
valueEncoder.encode(values[i], valueOut);
}
+ valueRawSize.addAndGet(Float.BYTES * batchSize);
statistics.update(timestamps, values, batchSize);
}
@@ -184,6 +201,7 @@ public class ValuePageWriter {
for (int i = 0; i < batchSize; i++) {
valueEncoder.encode(values[i], valueOut);
}
+ valueRawSize.addAndGet(Double.BYTES * batchSize);
statistics.update(timestamps, values, batchSize);
}
@@ -191,6 +209,7 @@ public class ValuePageWriter {
public void write(long[] timestamps, Binary[] values, int batchSize) {
for (int i = 0; i < batchSize; i++) {
valueEncoder.encode(values[i], valueOut);
+ valueRawSize.addAndGet(values[i].getLength());
}
statistics.update(timestamps, values, batchSize);
}
@@ -211,6 +230,7 @@ public class ValuePageWriter {
*/
public ByteBuffer getUncompressedBytes() throws IOException {
prepareEndWriteOnePage();
+ valueEncodedSize.addAndGet(Integer.BYTES + bitmapOut.size() + valueOut.size());
ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES + bitmapOut.size() + valueOut.size());
buffer.putInt(size);
buffer.put(bitmapOut.getBuf(), 0, bitmapOut.size());
@@ -251,6 +271,7 @@ public class ValuePageWriter {
compressor.compress(
pageData.array(), pageData.position(), uncompressedSize, compressedBytes);
}
+ valueCompressedSize.addAndGet(compressedSize);
// write the page header to IOWriter
int sizeWithoutStatistic = 0;