You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2022/04/26 01:13:41 UTC
[iotdb] branch research/encoding-exp updated: [To Research/encoding] Research/encoding exp (#5606)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch research/encoding-exp
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/research/encoding-exp by this push:
new 07163abf91 [To Research/encoding] Research/encoding exp (#5606)
07163abf91 is described below
commit 07163abf91fbe3130c3932381d2e1f42d7ae31c2
Author: huangyuxiang03 <57...@users.noreply.github.com>
AuthorDate: Tue Apr 26 09:13:33 2022 +0800
[To Research/encoding] Research/encoding exp (#5606)
---
.../org/apache/iotdb/db/qp/sql/IoTDBSqlLexer.g4 | 10 +-
client-cpp/src/main/Session.h | 7 +-
client-py/iotdb/utils/IoTDBConstants.py | 5 +
mvn | 0
.../org/apache/iotdb/db/utils/SchemaUtils.java | 2 +
tsfile/README.md | 2 +-
tsfile/pom.xml | 11 +
.../apache/iotdb/tsfile/compress/ICompressor.java | 39 +-
.../iotdb/tsfile/compress/IUnCompressor.java | 12 +-
.../tsfile/encoding/HuffmanTree/Frequency.java | 11 +
.../tsfile/encoding/HuffmanTree/HuffmanCode.java | 12 +
.../tsfile/encoding/HuffmanTree/HuffmanTree.java | 21 +
.../iotdb/tsfile/encoding/decoder/Decoder.java | 29 +
.../encoding/decoder/DeltaBinaryDecoder.java | 7 +-
.../encoding/decoder/DoublePrecisionDecoderV2.java | 3 +-
.../tsfile/encoding/decoder/DoubleRAKEDecoder.java | 10 +-
.../tsfile/encoding/decoder/FloatDecoder.java | 3 +
.../tsfile/encoding/decoder/FloatRAKEDecoder.java | 10 +-
.../encoding/decoder/FloatSprintzDecoder.java | 3 +-
.../tsfile/encoding/decoder/HuffmanDecoder.java | 164 ++++++
.../tsfile/encoding/decoder/IntRAKEDecoder.java | 8 +-
.../tsfile/encoding/decoder/IntRleDecoder.java | 4 +-
.../tsfile/encoding/decoder/LongRAKEDecoder.java | 10 +-
.../tsfile/encoding/decoder/PlainDecoder.java | 18 +-
.../iotdb/tsfile/encoding/decoder/RAKEDecoder.java | 4 +-
.../encoding/decoder/SinglePrecisionDecoderV2.java | 3 +-
.../tsfile/encoding/decoder/TextRleDecoder.java | 50 ++
.../iotdb/tsfile/encoding/encoder/Encoder.java | 4 +
.../tsfile/encoding/encoder/FloatEncoder.java | 16 +
.../tsfile/encoding/encoder/GorillaEncoderV2.java | 5 +
.../tsfile/encoding/encoder/HuffmanEncoder.java | 235 ++++++++
.../iotdb/tsfile/encoding/encoder/IntRLBE.java | 6 +
.../iotdb/tsfile/encoding/encoder/RAKEEncoder.java | 2 +-
.../iotdb/tsfile/encoding/encoder/RleEncoder.java | 7 +-
.../encoding/encoder/SinglePrecisionEncoderV2.java | 5 +
.../tsfile/encoding/encoder/TSEncodingBuilder.java | 48 ++
.../tsfile/encoding/encoder/TextRleEncoder.java | 90 +++
.../tsfile/file/metadata/enums/TSEncoding.java | 8 +-
.../iotdb/tsfile/encoding/decoder/EncodeTest.java | 548 ++++++++++++++++++
.../encoding/decoder/HuffmanDecoderTest.java | 101 ++++
.../tsfile/encoding/decoder/PLAINDecoderTest.java | 217 +++++++
.../tsfile/encoding/decoder/RAKEDecoderTest.java | 626 +++++++++++++++++++++
.../encoding/decoder/TextRleDecoderTest.java | 125 ++++
43 files changed, 2447 insertions(+), 54 deletions(-)
diff --git a/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlLexer.g4 b/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlLexer.g4
index 32c0b68fc4..603e17a84e 100644
--- a/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlLexer.g4
+++ b/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlLexer.g4
@@ -633,7 +633,7 @@ TEXT
// Encoding Type Keywords
ENCODING_VALUE
- : DICTIONARY | DIFF | GORILLA | PLAIN | REGULAR | RLE | TS_2DIFF | RLBE | RAKE | SPRINTZ
+ : DICTIONARY | DIFF | GORILLA | PLAIN | REGULAR | RLE | TS_2DIFF | RLBE | RAKE | SPRINTZ | HUFFMAN | TEXTRLE
;
DICTIONARY
@@ -676,6 +676,14 @@ SPRINTZ
: S P R I N T Z
;
+HUFFMAN
+ : H U F F M A N
+ ;
+
+TEXTRLE
+ : T E X T R L E
+ ;
+
// Compressor Type Keywords
COMPRESSOR_VALUE
diff --git a/client-cpp/src/main/Session.h b/client-cpp/src/main/Session.h
index 3a93a8e9af..b3c3ee140f 100644
--- a/client-cpp/src/main/Session.h
+++ b/client-cpp/src/main/Session.h
@@ -143,7 +143,12 @@ namespace TSEncoding {
BITMAP = (char) 5,
GORILLA_V1 = (char) 6,
REGULAR = (char) 7,
- GORILLA = (char) 8
+ GORILLA = (char) 8,
+ SPRINTZ = (char) 9,
+ RAKE = (char) 10,
+ RLBE = (char) 11,
+ TEXTRLE = (char) 12,
+ HUFFMAN = (char) 13
};
}
diff --git a/client-py/iotdb/utils/IoTDBConstants.py b/client-py/iotdb/utils/IoTDBConstants.py
index 50344d1480..bac922b781 100644
--- a/client-py/iotdb/utils/IoTDBConstants.py
+++ b/client-py/iotdb/utils/IoTDBConstants.py
@@ -48,6 +48,11 @@ class TSEncoding(Enum):
GORILLA_V1 = 6
REGULAR = 7
GORILLA = 8
+ SPRINTZ = 9,
+ RAKE = 10,
+ RLBE = 11,
+ TEXTRLE = 12,
+ HUFFMAN = 13
# this method is implemented to avoid the issue reported by:
# https://bugs.python.org/issue30545
diff --git a/mvn b/mvn
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java
index c953e2d4ab..a9a66764f3 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java
@@ -89,6 +89,8 @@ public class SchemaUtils {
Set<TSEncoding> textSet = new HashSet<>();
textSet.add(TSEncoding.PLAIN);
textSet.add(TSEncoding.DICTIONARY);
+ textSet.add(TSEncoding.HUFFMAN);
+ textSet.add(TSEncoding.RLE);
schemaChecker.put(TSDataType.TEXT, textSet);
}
diff --git a/tsfile/README.md b/tsfile/README.md
index fa5a31db3c..599d458c92 100644
--- a/tsfile/README.md
+++ b/tsfile/README.md
@@ -60,4 +60,4 @@ The features of TsFile is as follow:
* [Get Started](https://github.com/thulab/tsfile/wiki/Get-Started)
* [TsFile-Spark Connector](https://github.com/thulab/tsfile/wiki/TsFile-Spark-Connector)
-
+test0
\ No newline at end of file
diff --git a/tsfile/pom.xml b/tsfile/pom.xml
index 10a39adbe7..40a970281c 100644
--- a/tsfile/pom.xml
+++ b/tsfile/pom.xml
@@ -58,6 +58,17 @@
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
</dependency>
+ <dependency>
+ <groupId>net.sourceforge.javacsv</groupId>
+ <artifactId>javacsv</artifactId>
+ <version>2.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.jetbrains</groupId>
+ <artifactId>annotations</artifactId>
+ <version>RELEASE</version>
+ <scope>compile</scope>
+ </dependency>
</dependencies>
<build>
<plugins>
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/compress/ICompressor.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/compress/ICompressor.java
index 9243e53417..ae86c96864 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/compress/ICompressor.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/compress/ICompressor.java
@@ -25,6 +25,8 @@ import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import net.jpountz.lz4.LZ4Compressor;
import net.jpountz.lz4.LZ4Factory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.xerial.snappy.Snappy;
import java.io.ByteArrayInputStream;
@@ -42,6 +44,8 @@ import static org.apache.iotdb.tsfile.file.metadata.enums.CompressionType.SNAPPY
/** compress data according to type in schema. */
public interface ICompressor extends Serializable {
+ Logger logger = LoggerFactory.getLogger(ICompressor.class);
+
static ICompressor getCompressor(String name) {
return getCompressor(CompressionType.valueOf(name));
}
@@ -149,7 +153,8 @@ public interface ICompressor extends Serializable {
if (data == null) {
return new byte[0];
}
- return Snappy.compress(data);
+ byte[] r = Snappy.compress(data);
+ return r;
}
@Override
@@ -168,12 +173,14 @@ public interface ICompressor extends Serializable {
@Override
public int compress(byte[] data, int offset, int length, byte[] compressed) throws IOException {
- return Snappy.compress(data, offset, length, compressed, 0);
+ int r = Snappy.compress(data, offset, length, compressed, 0);
+ return r;
}
@Override
public int compress(ByteBuffer data, ByteBuffer compressed) throws IOException {
- return Snappy.compress(data, compressed);
+ int r = Snappy.compress(data, compressed);
+ return r;
}
@Override
@@ -201,23 +208,27 @@ public interface ICompressor extends Serializable {
if (data == null) {
return new byte[0];
}
- return compressor.compress(data);
+ byte[] r = compressor.compress(data);
+ return r;
}
@Override
public byte[] compress(byte[] data, int offset, int length) throws IOException {
- return compressor.compress(data, offset, length);
+ byte[] r = compressor.compress(data, offset, length);
+ return r;
}
@Override
public int compress(byte[] data, int offset, int length, byte[] compressed) {
- return compressor.compress(data, offset, length, compressed, 0);
+ int r = compressor.compress(data, offset, length, compressed, 0);
+ return r;
}
@Override
public int compress(ByteBuffer data, ByteBuffer compressed) {
compressor.compress(data, compressed);
- return data.limit();
+ int r = data.limit();
+ return r;
}
@Override
@@ -237,7 +248,8 @@ public interface ICompressor extends Serializable {
GZIPOutputStream gzip = new GZIPOutputStream(out);
gzip.write(data);
gzip.close();
- return out.toByteArray();
+ byte[] r = out.toByteArray();
+ return r;
}
public static byte[] uncompress(byte[] data) throws IOException {
@@ -251,8 +263,8 @@ public interface ICompressor extends Serializable {
out.write(buffer, 0, n);
}
in.close();
-
- return out.toByteArray();
+ byte[] r = out.toByteArray();
+ return r;
}
}
@@ -262,15 +274,16 @@ public interface ICompressor extends Serializable {
if (null == data) {
return new byte[0];
}
-
- return GZIPCompress.compress(data);
+ byte[] r = GZIPCompress.compress(data);
+ return r;
}
@Override
public byte[] compress(byte[] data, int offset, int length) throws IOException {
byte[] dataBefore = new byte[length];
System.arraycopy(data, offset, dataBefore, 0, length);
- return GZIPCompress.compress(dataBefore);
+ byte[] r = GZIPCompress.compress(dataBefore);
+ return r;
}
/** @exception GZIPCompressOverflowException if compressed byte array is too small. */
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/compress/IUnCompressor.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/compress/IUnCompressor.java
index b4b333e949..e97fe5f8fc 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/compress/IUnCompressor.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/compress/IUnCompressor.java
@@ -35,6 +35,7 @@ import java.nio.ByteBuffer;
/** uncompress data according to type in metadata. */
public interface IUnCompressor {
+ Logger logger = LoggerFactory.getLogger(IUnCompressor.class);
/**
* get the UnCompressor based on the CompressionType.
*
@@ -154,7 +155,8 @@ public interface IUnCompressor {
}
try {
- return Snappy.uncompress(bytes);
+ byte[] r = Snappy.uncompress(bytes);
+ return r;
} catch (IOException e) {
logger.error(
"tsfile-compression SnappyUnCompressor: errors occurs when uncompress input byte", e);
@@ -165,7 +167,8 @@ public interface IUnCompressor {
@Override
public int uncompress(byte[] byteArray, int offset, int length, byte[] output, int outOffset)
throws IOException {
- return Snappy.uncompress(byteArray, offset, length, output, outOffset);
+ int r = Snappy.uncompress(byteArray, offset, length, output, outOffset);
+ return r;
}
@Override
@@ -175,7 +178,8 @@ public interface IUnCompressor {
}
try {
- return Snappy.uncompress(compressed, uncompressed);
+ int r = Snappy.uncompress(compressed, uncompressed);
+ return r;
} catch (IOException e) {
logger.error(
"tsfile-compression SnappyUnCompressor: errors occurs when uncompress input byte", e);
@@ -281,7 +285,6 @@ public interface IUnCompressor {
if (null == byteArray) {
return new byte[0];
}
-
return ICompressor.GZIPCompress.uncompress(byteArray);
}
@@ -303,7 +306,6 @@ public interface IUnCompressor {
byte[] res = ICompressor.GZIPCompress.uncompress(dataBefore);
uncompressed.put(res);
-
return res.length;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/HuffmanTree/Frequency.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/HuffmanTree/Frequency.java
new file mode 100644
index 0000000000..b7fa6f6d60
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/HuffmanTree/Frequency.java
@@ -0,0 +1,11 @@
+package org.apache.iotdb.tsfile.encoding.HuffmanTree;
+
+public class Frequency {
+ public int index;
+ public int frequency;
+
+ Frequency(int i, int f) {
+ index = i;
+ frequency = f;
+ }
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/HuffmanTree/HuffmanCode.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/HuffmanTree/HuffmanCode.java
new file mode 100644
index 0000000000..b4c1fe3dd1
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/HuffmanTree/HuffmanCode.java
@@ -0,0 +1,12 @@
+package org.apache.iotdb.tsfile.encoding.HuffmanTree;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class HuffmanCode {
+ public List<Boolean> huffmanCode;
+
+ public HuffmanCode() {
+ huffmanCode = new ArrayList<>();
+ }
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/HuffmanTree/HuffmanTree.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/HuffmanTree/HuffmanTree.java
new file mode 100644
index 0000000000..20d514b42d
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/HuffmanTree/HuffmanTree.java
@@ -0,0 +1,21 @@
+package org.apache.iotdb.tsfile.encoding.HuffmanTree;
+
+public class HuffmanTree {
+ public boolean isLeaf = false;
+ public int frequency;
+ public byte originalbyte;
+ public boolean isRecordEnd = false;
+ public HuffmanTree leftNode;
+ public HuffmanTree rightNode;
+
+ public void HuffmanTree() {}
+
+ public void clear() {
+ isLeaf = false;
+ isRecordEnd = false;
+ frequency = 0;
+ originalbyte = 0;
+ leftNode = null;
+ rightNode = null;
+ }
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java
index e14c1bb464..4b0c150d1b 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java
@@ -24,12 +24,17 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.utils.Binary;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.io.IOException;
import java.math.BigDecimal;
import java.nio.ByteBuffer;
public abstract class Decoder {
+ protected static final Logger logger = LoggerFactory.getLogger(Decoder.class);
+
private static final String ERROR_MSG = "Decoder not found: %s , DataType is : %s";
private TSEncoding type;
@@ -61,6 +66,8 @@ public abstract class Decoder {
case FLOAT:
case DOUBLE:
return new FloatDecoder(TSEncoding.valueOf(encoding.toString()), dataType);
+ case TEXT:
+ return new TextRleDecoder();
default:
throw new TsFileDecodingException(String.format(ERROR_MSG, encoding, dataType));
}
@@ -151,6 +158,28 @@ public abstract class Decoder {
default:
throw new TsFileDecodingException(String.format(ERROR_MSG, encoding, dataType));
}
+ case TEXTRLE:
+ switch (dataType) {
+ case TEXT:
+ return new TextRleDecoder();
+ case INT32:
+ case INT64:
+ case FLOAT:
+ case DOUBLE:
+ default:
+ throw new TsFileDecodingException(String.format(ERROR_MSG, encoding, dataType));
+ }
+ case HUFFMAN:
+ switch (dataType) {
+ case TEXT:
+ return new HuffmanDecoder();
+ case INT32:
+ case INT64:
+ case FLOAT:
+ case DOUBLE:
+ default:
+ throw new TsFileDecodingException(String.format(ERROR_MSG, encoding, dataType));
+ }
default:
throw new TsFileDecodingException(String.format(ERROR_MSG, encoding, dataType));
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java
index 7f50cd4bb6..e918b24176 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java
@@ -103,7 +103,8 @@ public abstract class DeltaBinaryDecoder extends Decoder {
@Override
public int readInt(ByteBuffer buffer) {
- return readT(buffer);
+ int r = readT(buffer);
+ return r;
}
/**
@@ -218,8 +219,8 @@ public abstract class DeltaBinaryDecoder extends Decoder {
@Override
public long readLong(ByteBuffer buffer) {
-
- return readT(buffer);
+ long r = readT(buffer);
+ return r;
}
@Override
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DoublePrecisionDecoderV2.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DoublePrecisionDecoderV2.java
index 47b56f1c89..bae149769d 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DoublePrecisionDecoderV2.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DoublePrecisionDecoderV2.java
@@ -39,7 +39,8 @@ public class DoublePrecisionDecoderV2 extends LongGorillaDecoder {
@Override
public final double readDouble(ByteBuffer in) {
- return Double.longBitsToDouble(readLong(in));
+ double r = Double.longBitsToDouble(readLong(in));
+ return r;
}
@Override
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DoubleRAKEDecoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DoubleRAKEDecoder.java
index 3194b189b2..8c4d209add 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DoubleRAKEDecoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DoubleRAKEDecoder.java
@@ -27,14 +27,16 @@ public class DoubleRAKEDecoder extends RAKEDecoder {
parseBuffer(buffer, 64);
String subNumBuffer = numBuffer.substring(0, 64);
this.numBuffer = "";
- if (subNumBuffer.charAt(0) == '0')
- return Double.longBitsToDouble(Long.parseUnsignedLong(subNumBuffer, 2));
- else {
+ if (subNumBuffer.charAt(0) == '0') {
+ double r = Double.longBitsToDouble(Long.parseUnsignedLong(subNumBuffer, 2));
+ return r;
+ } else {
String tmpSubNumBuffer = "0";
for (int i = 1; i < subNumBuffer.length(); i++) {
tmpSubNumBuffer += subNumBuffer.charAt(i);
}
- return -Double.longBitsToDouble(Long.parseUnsignedLong(tmpSubNumBuffer, 2));
+ double r = -Double.longBitsToDouble(Long.parseUnsignedLong(tmpSubNumBuffer, 2));
+ return r;
}
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FloatDecoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FloatDecoder.java
index f5690acf97..4322852e9d 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FloatDecoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FloatDecoder.java
@@ -47,8 +47,11 @@ public class FloatDecoder extends Decoder {
/** flag that indicates whether we have read maxPointNumber and calculated maxPointValue. */
private boolean isMaxPointNumberRead;
+ private TSEncoding encoderType;
+
public FloatDecoder(TSEncoding encodingType, TSDataType dataType) {
super(encodingType);
+ encoderType = encodingType;
if (encodingType == TSEncoding.RLE) {
if (dataType == TSDataType.FLOAT) {
decoder = new IntRleDecoder();
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FloatRAKEDecoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FloatRAKEDecoder.java
index 95acd37a1e..431ec733a5 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FloatRAKEDecoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FloatRAKEDecoder.java
@@ -27,14 +27,16 @@ public class FloatRAKEDecoder extends RAKEDecoder {
parseBuffer(buffer, 32);
String subNumBuffer = numBuffer.substring(0, 32);
this.numBuffer = "";
- if (subNumBuffer.charAt(0) == '0')
- return Float.intBitsToFloat(Integer.parseUnsignedInt(subNumBuffer, 2));
- else {
+ if (subNumBuffer.charAt(0) == '0') {
+ float r = Float.intBitsToFloat(Integer.parseUnsignedInt(subNumBuffer, 2));
+ return r;
+ } else {
String tmpSubNumBuffer = "0";
for (int i = 1; i < subNumBuffer.length(); i++) {
tmpSubNumBuffer += subNumBuffer.charAt(i);
}
- return -Float.intBitsToFloat(Integer.parseUnsignedInt(tmpSubNumBuffer, 2));
+ float r = -Float.intBitsToFloat(Integer.parseUnsignedInt(tmpSubNumBuffer, 2));
+ return r;
}
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FloatSprintzDecoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FloatSprintzDecoder.java
index 40f523e91b..9fd732e7af 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FloatSprintzDecoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FloatSprintzDecoder.java
@@ -130,7 +130,8 @@ public class FloatSprintzDecoder extends SprintzDecoder {
logger.error("Error occured when readInt with Sprintz Decoder.", e);
}
}
- currentValue = currentBuffer[currentCount++];
+ currentValue = currentBuffer[currentCount];
+ currentCount++;
if (currentCount == decodeSize) {
isBlockReaded = false;
currentCount = 0;
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/HuffmanDecoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/HuffmanDecoder.java
new file mode 100644
index 0000000000..67e6b23fb0
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/HuffmanDecoder.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import org.apache.iotdb.tsfile.encoding.HuffmanTree.HuffmanTree;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.utils.Binary;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Queue;
+
+public class HuffmanDecoder extends Decoder {
+
+ private int recordnum;
+ private int numberLeftInBuffer;
+ private byte byteBuffer;
+ private Queue<Binary> records;
+ private HuffmanTree tree;
+
+ HuffmanDecoder() {
+ super(TSEncoding.HUFFMAN);
+ records = new LinkedList<>();
+ tree = new HuffmanTree();
+ reset();
+ }
+
+ @Override
+ public Binary readBinary(ByteBuffer buffer) {
+ if (records.isEmpty()) {
+ reset();
+ loadTree(buffer);
+ loadRecords(buffer);
+ clearBuffer(buffer);
+ }
+ return records.poll();
+ }
+
+ public boolean hasNext(ByteBuffer buffer) {
+ return ((!records.isEmpty()) || buffer.hasRemaining());
+ }
+
+ private void loadTree(ByteBuffer buffer) {
+ recordnum = getInt(buffer);
+ int endOfRecordLength = getInt(buffer);
+ HuffmanTree header = tree;
+ for (int i = 0; i < endOfRecordLength; i++) {
+ if (readbit(buffer) == 0) {
+ if (header.leftNode == null) header.leftNode = new HuffmanTree();
+ header = header.leftNode;
+ } else {
+ if (header.rightNode == null) header.rightNode = new HuffmanTree();
+ header = header.rightNode;
+ }
+ if (i == endOfRecordLength - 1) {
+ header.isLeaf = true;
+ header.isRecordEnd = true;
+ }
+ }
+
+ int usednum = getInt(buffer);
+ for (int i = 0; i < usednum; i++) {
+ byte cha = getByte(buffer);
+ int codeLength = getInt(buffer);
+ String s = new String();
+ HuffmanTree tempTree = tree;
+ for (int j = 0; j < codeLength; j++) {
+ int b = readbit(buffer);
+ if (b == 0) {
+ if (tempTree.leftNode == null) tempTree.leftNode = new HuffmanTree();
+ tempTree = tempTree.leftNode;
+ } else {
+ if (tempTree.rightNode == null) tempTree.rightNode = new HuffmanTree();
+ tempTree = tempTree.rightNode;
+ }
+ if (j == codeLength - 1) {
+ tempTree.isLeaf = true;
+ tempTree.originalbyte = cha;
+ }
+ }
+ }
+ }
+
+ private void loadRecords(ByteBuffer buffer) {
+ for (int i = 0; i < recordnum; i++) {
+ HuffmanTree tempTree = tree;
+ List<Byte> rec = new ArrayList<>();
+ while (true) {
+ tempTree = tree;
+ while (!tempTree.isLeaf) {
+ if (readbit(buffer) == 0) tempTree = tempTree.leftNode;
+ else tempTree = tempTree.rightNode;
+ }
+ if (tempTree.isRecordEnd) break;
+ rec.add(tempTree.originalbyte);
+ }
+ byte[] currec = new byte[rec.size()];
+ for (int j = 0; j < rec.size(); j++) currec[j] = rec.get(j);
+ records.add(new Binary(currec));
+ }
+ }
+
+ public void reset() {
+ recordnum = 0;
+ records.clear();
+ tree.clear();
+ }
+
+ private int getInt(ByteBuffer buffer) {
+ int val = 0;
+ for (int i = 31; i >= 0; i--) {
+ val |= (readbit(buffer) << i);
+ }
+ return val;
+ }
+
+ private byte getByte(ByteBuffer buffer) {
+ byte val = 0;
+ for (int i = 7; i >= 0; i--) {
+ val |= (readbit(buffer) << i);
+ }
+ return val;
+ }
+
+ private int readbit(ByteBuffer buffer) {
+ if (numberLeftInBuffer == 0) {
+ loadBuffer(buffer);
+ numberLeftInBuffer = 8;
+ }
+ int top = ((byteBuffer >> 7) & 1);
+ byteBuffer <<= 1;
+ numberLeftInBuffer--;
+ return top;
+ }
+
+ private void loadBuffer(ByteBuffer buffer) {
+ byteBuffer = buffer.get();
+ }
+
+ private void clearBuffer(ByteBuffer buffer) {
+ while (numberLeftInBuffer > 0) {
+ readbit(buffer);
+ }
+ }
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/IntRAKEDecoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/IntRAKEDecoder.java
index 6abc08b397..fc37e024ce 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/IntRAKEDecoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/IntRAKEDecoder.java
@@ -26,16 +26,16 @@ public class IntRAKEDecoder extends RAKEDecoder {
@Override
public int readInt(ByteBuffer buffer) {
parseBuffer(buffer, 32);
- String subNumBuffer = numBuffer.substring(0, 32);
+ String subNumBuffer = numBuffer.substring(0, Math.min(32, numBuffer.length()));
this.numBuffer = "";
- if (subNumBuffer.charAt(0) == '0') return Integer.parseInt(subNumBuffer, 2);
- else {
+ if (subNumBuffer.charAt(0) == '0') {
+ return Integer.parseInt(subNumBuffer, 2);
+ } else {
String tmpSubNumBuffer = "0";
for (int i = 1; i < subNumBuffer.length(); i++) {
if (subNumBuffer.charAt(i) == '1') tmpSubNumBuffer += "0";
else tmpSubNumBuffer += "1";
}
-
return -Integer.parseInt(tmpSubNumBuffer, 2) - 1;
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/IntRleDecoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/IntRleDecoder.java
index 219a35c23e..850bacdf6b 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/IntRleDecoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/IntRleDecoder.java
@@ -51,7 +51,8 @@ public class IntRleDecoder extends RleDecoder {
@Override
public boolean readBoolean(ByteBuffer buffer) {
- return this.readInt(buffer) == 0 ? false : true;
+ boolean r = this.readInt(buffer) == 0 ? false : true;
+ return r;
}
/**
@@ -96,6 +97,7 @@ public class IntRleDecoder extends RleDecoder {
if (!hasNextPackage()) {
isLengthAndBitWidthReaded = false;
}
+
return result;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/LongRAKEDecoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/LongRAKEDecoder.java
index 10167cfc43..cde491549b 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/LongRAKEDecoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/LongRAKEDecoder.java
@@ -28,15 +28,17 @@ public class LongRAKEDecoder extends RAKEDecoder {
parseBuffer(buffer, 64);
String subNumBuffer = numBuffer.substring(0, 64);
this.numBuffer = "";
- if (subNumBuffer.charAt(0) == '0') return Long.parseLong(subNumBuffer, 2);
- else {
+ if (subNumBuffer.charAt(0) == '0') {
+ long r = Long.parseLong(subNumBuffer, 2);
+ return r;
+ } else {
String tmpSubNumBuffer = "0";
for (int i = 1; i < subNumBuffer.length(); i++) {
if (subNumBuffer.charAt(i) == '1') tmpSubNumBuffer += "0";
else tmpSubNumBuffer += "1";
}
-
- return -Long.parseLong(tmpSubNumBuffer, 2) - 1;
+ long r = -Long.parseLong(tmpSubNumBuffer, 2) - 1;
+ return r;
}
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/PlainDecoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/PlainDecoder.java
index 30c3badb54..6bdf669d13 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/PlainDecoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/PlainDecoder.java
@@ -35,31 +35,37 @@ public class PlainDecoder extends Decoder {
@Override
public boolean readBoolean(ByteBuffer buffer) {
- return buffer.get() != 0;
+ byte r = buffer.get();
+ return r != 0;
}
@Override
public short readShort(ByteBuffer buffer) {
- return buffer.getShort();
+ short r = buffer.getShort();
+ return r;
}
@Override
public int readInt(ByteBuffer buffer) {
- return ReadWriteForEncodingUtils.readVarInt(buffer);
+ int r = ReadWriteForEncodingUtils.readVarInt(buffer);
+ return r;
}
@Override
public long readLong(ByteBuffer buffer) {
- return buffer.getLong();
+ long r = buffer.getLong();
+ return r;
}
@Override
public float readFloat(ByteBuffer buffer) {
- return buffer.getFloat();
+ float r = buffer.getFloat();
+ return r;
}
@Override
public double readDouble(ByteBuffer buffer) {
+ double r = buffer.getDouble();
return buffer.getDouble();
}
@@ -78,7 +84,7 @@ public class PlainDecoder extends Decoder {
@Override
public BigDecimal readBigDecimal(ByteBuffer buffer) {
- throw new TsFileDecodingException("Method readBigDecimal is not supproted by PlainDecoder");
+ throw new TsFileDecodingException("Method readBigDecimal is not supported by PlainDecoder");
}
@Override
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/RAKEDecoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/RAKEDecoder.java
index a0118ede10..296cab30fa 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/RAKEDecoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/RAKEDecoder.java
@@ -189,7 +189,9 @@ public abstract class RAKEDecoder extends Decoder {
@Override
public boolean hasNext(ByteBuffer buffer) {
- return buffer.remaining() > 0;
+ // System.out.println(buffer);
+
+ return buffer.remaining() > 0 && numBuffer != "";
}
@Override
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/SinglePrecisionDecoderV2.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/SinglePrecisionDecoderV2.java
index ea3d7bc67d..9dac3c26b8 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/SinglePrecisionDecoderV2.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/SinglePrecisionDecoderV2.java
@@ -39,7 +39,8 @@ public class SinglePrecisionDecoderV2 extends IntGorillaDecoder {
@Override
public final float readFloat(ByteBuffer in) {
- return Float.intBitsToFloat(readInt(in));
+ float r = Float.intBitsToFloat(readInt(in));
+ return r;
}
@Override
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/TextRleDecoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/TextRleDecoder.java
new file mode 100644
index 0000000000..ffdb87250a
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/TextRleDecoder.java
@@ -0,0 +1,50 @@
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import org.apache.iotdb.tsfile.encoding.encoder.TextRleEncoder;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.utils.Binary;
+import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.ByteBuffer;
+
+public class TextRleDecoder extends Decoder {
+ protected static final Logger logger = LoggerFactory.getLogger(TextRleEncoder.class);
+
+ public TextRleDecoder() {
+ super(TSEncoding.RLE);
+ }
+
+ @Override
+ public boolean hasNext(ByteBuffer buffer) {
+ return buffer.hasRemaining();
+ }
+
+ @Override
+ public Binary readBinary(ByteBuffer buffer) {
+ int length = ReadWriteForEncodingUtils.readVarInt(buffer);
+ byte[] values = new byte[length];
+ Decoder decoder = Decoder.getDecoderByType(TSEncoding.RLE, TSDataType.INT32);
+ int upper = length - length % 4;
+ for (int i = 0; i < upper; i += 4) {
+ int val = decoder.readInt(buffer);
+ values[i] = (byte) ((val >> 24) & 0xFF);
+ values[i + 1] = (byte) ((val >> 16) & 0xFF);
+ values[i + 2] = (byte) ((val >> 8) & 0xFF);
+ values[i + 3] = (byte) ((val) & 0xFF);
+ }
+ if (upper != length) {
+ int val = decoder.readInt(buffer);
+ for (int i = 0; i < length % 4; i++) {
+ values[i + upper] = (byte) ((val >> (3 - i) * 8) & 0xFF);
+ }
+ }
+ return new Binary(values);
+ }
+
+ @Override
+ public void reset() {}
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/Encoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/Encoder.java
index b201fb97da..da18875589 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/Encoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/Encoder.java
@@ -65,6 +65,10 @@ public abstract class Encoder {
throw new TsFileEncodingException("Method encode long is not supported by Encoder");
}
+ public void encode(String value, ByteArrayOutputStream out) {
+ throw new TsFileEncodingException("Method encode text is not supported by Encoder");
+ };
+
public void encode(float value, ByteArrayOutputStream out) {
throw new TsFileEncodingException("Method encode float is not supported by Encoder");
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/FloatEncoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/FloatEncoder.java
index 2d477cb936..ee0d4f5963 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/FloatEncoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/FloatEncoder.java
@@ -24,6 +24,9 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.io.ByteArrayOutputStream;
import java.io.IOException;
@@ -49,9 +52,16 @@ public class FloatEncoder extends Encoder {
/** flag to check whether maxPointNumber is saved in the stream. */
private boolean isMaxPointNumberSaved;
+ /** same as encodingType when constructing an object. */
+ private TSEncoding encoderType;
+
+ /** logger to save start and stop. */
+ protected static final Logger logger = LoggerFactory.getLogger(FloatEncoder.class);
+
public FloatEncoder(TSEncoding encodingType, TSDataType dataType, int maxPointNumber) {
super(encodingType);
this.maxPointNumber = maxPointNumber;
+ this.encoderType = encodingType;
calculateMaxPonitNum();
isMaxPointNumberSaved = false;
if (encodingType == TSEncoding.RLE) {
@@ -80,9 +90,15 @@ public class FloatEncoder extends Encoder {
@Override
public void encode(float value, ByteArrayOutputStream out) {
+ if (this.encoderType == TSEncoding.RLE) {
+ } else if (this.encoderType == TSEncoding.TS_2DIFF) {
+ }
saveMaxPointNumber(out);
int valueInt = convertFloatToInt(value);
encoder.encode(valueInt, out);
+ if (this.encoderType == TSEncoding.RLE) {
+ } else if (this.encoderType == TSEncoding.TS_2DIFF) {
+ }
}
@Override
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/GorillaEncoderV2.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/GorillaEncoderV2.java
index 00422e7e78..6a41745e32 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/GorillaEncoderV2.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/GorillaEncoderV2.java
@@ -21,6 +21,9 @@ package org.apache.iotdb.tsfile.encoding.encoder;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.io.ByteArrayOutputStream;
/**
@@ -41,6 +44,8 @@ public abstract class GorillaEncoderV2 extends Encoder {
private byte buffer = 0;
protected int bitsLeft = Byte.SIZE;
+ protected static final Logger logger = LoggerFactory.getLogger(GorillaEncoderV2.class);
+
protected GorillaEncoderV2() {
super(TSEncoding.GORILLA);
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/HuffmanEncoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/HuffmanEncoder.java
new file mode 100644
index 0000000000..81412a7931
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/HuffmanEncoder.java
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.encoder;
+
+import org.apache.iotdb.tsfile.encoding.HuffmanTree.HuffmanCode;
+import org.apache.iotdb.tsfile.encoding.HuffmanTree.HuffmanTree;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.utils.Binary;
+
+import java.io.ByteArrayOutputStream;
+import java.util.*;
+
+public class HuffmanEncoder extends Encoder {
+
+ private HuffmanTree[] byteFrequency;
+ private List<Binary> records;
+ private HuffmanCode[] huffmanCodes;
+ PriorityQueue<HuffmanTree> huffmanQueue;
+ private HuffmanTree treeTop;
+ private byte byteBuffer;
+ private int numberLeftInBuffer = 0;
+ private boolean[] used;
+ private int usednum;
+ private int maxRecordLength;
+ private int totLength;
+ private int recordnum;
+
+ public HuffmanEncoder() {
+ super(TSEncoding.HUFFMAN);
+ byteFrequency =
+ new HuffmanTree[257]; // byteFrequency[256] is used to save the frequency of end-of-records
+ for (int i = 0; i <= 256; i++) byteFrequency[i] = new HuffmanTree();
+ records = new ArrayList<Binary>();
+ huffmanQueue = new PriorityQueue<HuffmanTree>(huffmanTreeComparator);
+ huffmanCodes = new HuffmanCode[257];
+ for (int i = 0; i <= 256; i++) {
+ huffmanCodes[i] = new HuffmanCode();
+ }
+ used = new boolean[257];
+ treeTop = new HuffmanTree();
+ reset();
+ }
+
+ @Override
+ public void encode(Binary value, ByteArrayOutputStream out) {
+ recordnum++;
+ maxRecordLength = Math.max(maxRecordLength, value.getLength());
+ records.add(value);
+ for (int i = 0; i < value.getLength(); i++) {
+ byte cur = value.getValues()[i];
+ int curr = (int) cur;
+ if (curr < 0) curr += (1 << 8);
+ byteFrequency[curr].frequency++;
+ }
+ byteFrequency[256].frequency++;
+ }
+
+ @Override
+ public void flush(ByteArrayOutputStream out) {
+ buildHuffmanTree();
+ List<Boolean> code = new ArrayList<>();
+ getHuffmanCode(treeTop, code);
+ flushHeader(out);
+ for (Binary rec : records) flushRecord(rec, out);
+ reset();
+ clearBuffer(out);
+ }
+
+ @Override
+ public int getOneItemMaxSize() {
+ return maxRecordLength;
+ }
+
+ @Override
+ public long getMaxByteSize() {
+ return totLength;
+ }
+
+ private void buildHuffmanTree() {
+ for (int i = 0; i <= 256; i++) {
+ if (byteFrequency[i].frequency != 0) {
+ huffmanQueue.add(byteFrequency[i]);
+ used[i] = true;
+ usednum++;
+ }
+ }
+ usednum -= 1;
+ while (huffmanQueue.size() > 1) {
+ HuffmanTree cur = new HuffmanTree();
+ cur.leftNode = huffmanQueue.poll();
+ cur.rightNode = huffmanQueue.poll();
+ cur.frequency = cur.leftNode.frequency + cur.rightNode.frequency;
+ cur.isRecordEnd = false;
+ cur.isLeaf = false;
+ huffmanQueue.add(cur);
+ }
+ treeTop = huffmanQueue.poll();
+ }
+
+ private void getHuffmanCode(HuffmanTree cur, List<Boolean> code) {
+ if (cur.isLeaf) {
+ if (cur.isRecordEnd) {
+ for (int i = 0; i < code.size(); i++) huffmanCodes[256].huffmanCode.add(code.get(i));
+ } else {
+ for (int i = 0; i < code.size(); i++) {
+ int idx = (int) cur.originalbyte;
+ if (idx < 0) idx += (1 << 8);
+ huffmanCodes[idx].huffmanCode.add(code.get(i));
+ }
+ }
+ return;
+ }
+ code.add(false);
+ getHuffmanCode(cur.leftNode, code);
+ code.remove(code.size() - 1);
+ code.add(true);
+ getHuffmanCode(cur.rightNode, code);
+ code.remove(code.size() - 1);
+ }
+
+ private void flushHeader(ByteArrayOutputStream out) {
+ writeInt(recordnum, out); // write the number of records
+ totLength += 4;
+ writeInt(
+ huffmanCodes[256].huffmanCode.size(),
+ out); // Write the length of huffman code of end-of-record sign
+ totLength += 4;
+ for (boolean b : huffmanCodes[256].huffmanCode) { // Write the end-of-record sign
+ writeBit(b, out);
+ }
+ writeInt(usednum, out); // Write how many character have been used in this section
+ totLength += 4;
+ for (int i = 0; i < 256; i++) {
+ if (used[i]) {
+ writeByte((byte) i, out);
+ writeInt(
+ huffmanCodes[i].huffmanCode.size(), out); // First we store the length of huffman code
+ totLength += 8;
+ for (boolean b : huffmanCodes[i].huffmanCode) // Then we store the huffman code
+ writeBit(b, out);
+ }
+ }
+ }
+
+ private void flushRecord(Binary rec, ByteArrayOutputStream out) {
+ for (byte r : rec.getValues()) {
+ int idx = (int) r;
+ if (idx < 0) idx += (1 << 8);
+ for (boolean b : huffmanCodes[idx].huffmanCode) writeBit(b, out);
+ }
+ for (boolean b : huffmanCodes[256].huffmanCode) writeBit(b, out);
+ }
+
+ private void reset() {
+ for (int i = 0; i < 256; i++) {
+ byteFrequency[i].frequency = 0;
+ byteFrequency[i].originalbyte = (byte) i;
+ byteFrequency[i].isLeaf = true;
+ byteFrequency[i].isRecordEnd = false;
+ huffmanCodes[i].huffmanCode.clear();
+ used[i] = false;
+ }
+ byteFrequency[256].frequency = 0;
+ byteFrequency[256].isLeaf = true;
+ byteFrequency[256].isRecordEnd = true;
+ huffmanCodes[256].huffmanCode.clear();
+ records.clear();
+ huffmanQueue.clear();
+ usednum = 0;
+ maxRecordLength = 0;
+ totLength = 0;
+ recordnum = 0;
+ treeTop.clear();
+ }
+
+ public static Comparator<HuffmanTree> huffmanTreeComparator =
+ new Comparator<HuffmanTree>() {
+ @Override
+ public int compare(HuffmanTree o1, HuffmanTree o2) {
+ return o1.frequency - o2.frequency;
+ }
+ };
+
+ protected void writeBit(boolean b, ByteArrayOutputStream out) {
+ byteBuffer <<= 1;
+ if (b) {
+ byteBuffer |= 1;
+ }
+
+ numberLeftInBuffer++;
+ if (numberLeftInBuffer == 8) {
+ clearBuffer(out);
+ }
+ }
+
+ protected void clearBuffer(ByteArrayOutputStream out) {
+ if (numberLeftInBuffer == 0) return;
+ if (numberLeftInBuffer > 0) byteBuffer <<= (8 - numberLeftInBuffer);
+ out.write(byteBuffer);
+ totLength++;
+ numberLeftInBuffer = 0;
+ byteBuffer = 0;
+ }
+
+ private void writeInt(int val, ByteArrayOutputStream out) {
+ for (int i = 31; i >= 0; i--) {
+ if ((val & (1 << i)) > 0) writeBit(true, out);
+ else writeBit(false, out);
+ }
+ }
+
+ private void writeByte(byte val, ByteArrayOutputStream out) {
+ for (int i = 7; i >= 0; i--) {
+ if ((val & (1 << i)) > 0) writeBit(true, out);
+ else writeBit(false, out);
+ }
+ }
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/IntRLBE.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/IntRLBE.java
index 16868469ac..5bec5b4df8 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/IntRLBE.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/IntRLBE.java
@@ -19,6 +19,9 @@
package org.apache.iotdb.tsfile.encoding.encoder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.io.ByteArrayOutputStream;
import java.io.IOException;
@@ -32,6 +35,9 @@ public class IntRLBE extends RLBE {
/** previous value of original value */
private int previousvalue;
+ /** logger */
+ private static final Logger logger = LoggerFactory.getLogger(IntRLBE.class);
+
/** constructor of IntRLBE */
public IntRLBE() {
super();
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/RAKEEncoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/RAKEEncoder.java
index 137c789d3b..8f8bff951c 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/RAKEEncoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/RAKEEncoder.java
@@ -44,7 +44,7 @@ public abstract class RAKEEncoder extends Encoder {
private int T;
private int L;
protected ByteArrayOutputStream byteCache;
- private static final Logger logger = LoggerFactory.getLogger(RAKEEncoder.class);
+ protected static final Logger logger = LoggerFactory.getLogger(RAKEEncoder.class);
/** constructor. * */
public RAKEEncoder() {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/RleEncoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/RleEncoder.java
index 2b40e880b4..3edfd48d2a 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/RleEncoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/RleEncoder.java
@@ -61,7 +61,7 @@ import java.util.List;
*/
public abstract class RleEncoder<T extends Comparable<T>> extends Encoder {
- private static final Logger logger = LoggerFactory.getLogger(RleEncoder.class);
+ protected static final Logger logger = LoggerFactory.getLogger(RleEncoder.class);
/** we save all value in a list and calculate its bitwidth. */
protected List<T> values;
@@ -311,6 +311,11 @@ public abstract class RleEncoder<T extends Comparable<T>> extends Encoder {
throw new TsFileEncodingException(getClass().getName());
}
+ @Override
+ public void encode(String value, ByteArrayOutputStream out) {
+ throw new TsFileEncodingException(getClass().getName());
+ }
+
@Override
public void encode(float value, ByteArrayOutputStream out) {
throw new TsFileEncodingException(getClass().getName());
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/SinglePrecisionEncoderV2.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/SinglePrecisionEncoderV2.java
index 0be2f1f1c9..842232db5f 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/SinglePrecisionEncoderV2.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/SinglePrecisionEncoderV2.java
@@ -36,11 +36,14 @@ public class SinglePrecisionEncoderV2 extends IntGorillaEncoder {
@Override
public final void encode(float value, ByteArrayOutputStream out) {
+ // logger.error("Encode GORILLA start");
encode(Float.floatToRawIntBits(value), out);
+ // logger.error("Encode GORILLA stop");
}
@Override
public void flush(ByteArrayOutputStream out) {
+ // logger.error("Flush GORILLA start");
// ending stream
encode(GORILLA_ENCODING_ENDING_FLOAT, out);
@@ -51,5 +54,7 @@ public class SinglePrecisionEncoderV2 extends IntGorillaEncoder {
// the encoder may be reused, so let us reset it
reset();
+
+ // logger.error("Flush GORILLA stop");
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java
index cc16acd97e..8dd568f5c4 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java
@@ -76,6 +76,10 @@ public abstract class TSEncodingBuilder {
return new Sprintz();
case RLBE:
return new RLBE();
+ case TEXTRLE:
+ return new TEXTRLE();
+ case HUFFMAN:
+ return new HUFFMAN();
default:
throw new UnsupportedOperationException(type.toString());
}
@@ -147,6 +151,8 @@ public abstract class TSEncodingBuilder {
case FLOAT:
case DOUBLE:
return new FloatEncoder(TSEncoding.RLE, type, maxPointNumber);
+ case TEXT:
+ return new TextRleEncoder();
default:
throw new UnSupportedDataTypeException("RLE doesn't support data type: " + type);
}
@@ -390,4 +396,46 @@ public abstract class TSEncodingBuilder {
// do nothing
}
}
+
+ public static class HUFFMAN extends TSEncodingBuilder {
+ @Override
+ public Encoder getEncoder(TSDataType type) {
+ switch (type) {
+ case TEXT:
+ return new HuffmanEncoder();
+ case INT32:
+ case INT64:
+ case FLOAT:
+ case DOUBLE:
+ default:
+ throw new UnSupportedDataTypeException("HUFFMAN doesn't support data type: " + type);
+ }
+ }
+
+ @Override
+ public void initFromProps(Map<String, String> props) {
+ // do nothing
+ }
+ }
+
+ public static class TEXTRLE extends TSEncodingBuilder {
+ @Override
+ public Encoder getEncoder(TSDataType type) {
+ switch (type) {
+ case TEXT:
+ return new TextRleEncoder();
+ case INT32:
+ case INT64:
+ case FLOAT:
+ case DOUBLE:
+ default:
+ throw new UnSupportedDataTypeException("TEXTRLE doesn't support data type: " + type);
+ }
+ }
+
+ @Override
+ public void initFromProps(Map<String, String> props) {
+ // do nothing
+ }
+ }
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TextRleEncoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TextRleEncoder.java
new file mode 100644
index 0000000000..a9761170e9
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TextRleEncoder.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.encoder;
+
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.utils.Binary;
+import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+
+public class TextRleEncoder extends Encoder {
+ protected static final Logger logger = LoggerFactory.getLogger(TextRleEncoder.class);
+
+ public TextRleEncoder() {
+ super(TSEncoding.RLE);
+ }
+
+ @Override
+ public void encode(Binary value, ByteArrayOutputStream out) {
+ byte[] values = value.getValues();
+ int length = values.length;
+ ReadWriteForEncodingUtils.writeVarInt(length, out);
+ ArrayList<Integer> buffer = new ArrayList<>();
+ int idx = length - length % 4;
+ for (int i = 0; i < idx; i += 4) {
+ int tmp = 0;
+ tmp += (values[i] & 0xFF) << 24;
+ tmp += (values[i + 1] & 0xFF) << 16;
+ tmp += (values[i + 2] & 0xFF) << 8;
+ tmp += values[i + 3] & 0xFF;
+ buffer.add(tmp);
+ }
+ if (length % 4 != 0) {
+ int tmp = 0;
+ for (int i = 0; i < length % 4; i++) {
+ int shift = (3 - i) * 8;
+ tmp += (values[i + idx] & 0xFF) << shift;
+ }
+ buffer.add(tmp);
+ }
+ int size = buffer.size();
+ Encoder encoder =
+ TSEncodingBuilder.getEncodingBuilder(TSEncoding.RLE).getEncoder(TSDataType.INT32);
+ for (int val : buffer) {
+ encoder.encode(val, out);
+ }
+ try {
+ encoder.flush(out);
+ } catch (IOException e) {
+ logger.error("RLE encoding for text failed: flush()");
+ }
+ }
+
+ @Override
+ public void flush(ByteArrayOutputStream out) throws IOException {}
+
+ @Override
+ public int getOneItemMaxSize() {
+ // 256*2
+ return 512;
+ }
+
+ @Override
+ public long getMaxByteSize() {
+ return 512 * 1000000;
+ }
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/TSEncoding.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/TSEncoding.java
index f6f029d00e..8f7f3e54ae 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/TSEncoding.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/TSEncoding.java
@@ -30,7 +30,9 @@ public enum TSEncoding {
GORILLA((byte) 8),
SPRINTZ((byte) 9),
RAKE((byte) 10),
- RLBE((byte) 11);
+ RLBE((byte) 11),
+ TEXTRLE((byte) 12),
+ HUFFMAN((byte) 13);
private final byte type;
@@ -74,6 +76,10 @@ public enum TSEncoding {
return TSEncoding.RAKE;
case 11:
return TSEncoding.RLBE;
+ case 12:
+ return TSEncoding.TEXTRLE;
+ case 13:
+ return TSEncoding.HUFFMAN;
default:
throw new IllegalArgumentException("Invalid input: " + encoding);
}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/EncodeTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/EncodeTest.java
new file mode 100644
index 0000000000..2ac6b8d0f6
--- /dev/null
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/EncodeTest.java
@@ -0,0 +1,548 @@
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import org.apache.iotdb.tsfile.compress.ICompressor;
+import org.apache.iotdb.tsfile.compress.IUnCompressor;
+import org.apache.iotdb.tsfile.encoding.encoder.Encoder;
+import org.apache.iotdb.tsfile.encoding.encoder.TSEncodingBuilder;
+import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+import java.io.*;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+
+public class EncodeTest {
+
+ public static void main(@org.jetbrains.annotations.NotNull String[] args) throws IOException {
+ String inputPath = "C:\\Users\\xiaoj\\Desktop\\int",
+ Output = "C:\\Users\\xiaoj\\Desktop\\UncompressedSpeedResult_testint.csv";
+ if (args.length >= 2) inputPath = args[1];
+ if (args.length >= 3) Output = args[2];
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+ TSEncoding[] schemeList = {
+ TSEncoding.PLAIN,
+ TSEncoding.TS_2DIFF,
+ TSEncoding.RLE,
+ TSEncoding.SPRINTZ,
+ TSEncoding.GORILLA,
+ TSEncoding.RAKE
+ };
+ // TSEncoding[] schemeList = { TSEncoding.RLBE,TSEncoding.PLAIN};
+ // CompressionType[] compressList = {CompressionType.LZ4, CompressionType.GZIP,
+ // CompressionType.SNAPPY};
+ CompressionType[] compressList = {CompressionType.UNCOMPRESSED}; //
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Encoding",
+ "Compress",
+ "Encoding Time",
+ "Compress Time",
+ "Uncompress Time",
+ "Decoding Time",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+ int repeatTime = 10;
+
+ assert tempList != null;
+ int fileRepeat = 0;
+ for (File f : tempList) {
+ System.out.println(f);
+ fileRepeat += 1;
+ InputStream inputStream = new FileInputStream(f);
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ String fileName = f.getAbsolutePath();
+ ArrayList<String> data = new ArrayList<>();
+ TSDataType dataType = TSDataType.INT32;
+
+ loader.readHeaders();
+ while (loader.readRecord()) {
+ data.add(loader.getValues()[2]);
+ }
+ loader.close();
+ inputStream.close();
+
+ if (fileName.contains("int")) {
+ dataType = TSDataType.INT32;
+ ArrayList<Integer> tmp = new ArrayList<>();
+ for (String value : data) {
+ tmp.add(Integer.valueOf(value));
+ }
+ for (TSEncoding scheme : schemeList) {
+ Encoder encoder = TSEncodingBuilder.getEncodingBuilder(scheme).getEncoder(dataType);
+ Decoder decoder = Decoder.getDecoderByType(scheme, dataType);
+ long encodeTime = 0;
+ long decodeTime = 0;
+ for (CompressionType comp : compressList) {
+ ICompressor compressor = ICompressor.getCompressor(comp);
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(comp);
+ long compressTime = 0;
+ long uncompressTime = 0;
+ double ratio = 0;
+ for (int i = 0; i < repeatTime; i++) {
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+ long s = System.nanoTime();
+ for (int val : tmp) encoder.encode(val, buffer);
+ long e = System.nanoTime();
+ encodeTime += (e - s);
+ System.out.println("encodeTime:");
+ System.out.println(e - s);
+ encoder.flush(buffer);
+
+ byte[] elems = buffer.toByteArray();
+ s = System.nanoTime();
+ byte[] compressed = compressor.compress(elems);
+ e = System.nanoTime();
+ compressTime += (e - s);
+
+ int ratioTmp = compressed.length / (tmp.size() * Long.BYTES);
+ ratio += ratioTmp;
+
+ s = System.nanoTime();
+ byte[] x = unCompressor.uncompress(compressed);
+ e = System.nanoTime();
+ uncompressTime += (e - s);
+
+ ByteBuffer ebuffer = ByteBuffer.wrap(buffer.toByteArray());
+ s = System.nanoTime();
+ while (decoder.hasNext(ebuffer)) {
+ decoder.readInt(ebuffer);
+ }
+ e = System.nanoTime();
+ decodeTime += (e - s);
+ buffer.close();
+ }
+ ratio /= repeatTime;
+ encodeTime = encodeTime / ((long) repeatTime);
+ decodeTime = decodeTime / ((long) repeatTime);
+ // compressTime = compressTime / ((long) repeatTime);
+ // uncompressTime = uncompressTime / ((long) repeatTime);
+ System.out.println(scheme.toString());
+ // System.out.println(String.valueOf(encodeTime));
+ String[] record = {
+ scheme.toString(),
+ comp.toString(),
+ String.valueOf(encodeTime),
+ "",
+ "",
+ String.valueOf(decodeTime),
+ String.valueOf(ratio)
+ };
+
+ // String[] record = {scheme.toString(), comp.toString(),
+ // String.valueOf(encodeTime), String.valueOf(compressTime),
+ // String.valueOf(uncompressTime),
+ // String.valueOf(decodeTime), String.valueOf(ratio)};
+ writer.writeRecord(record);
+ }
+ }
+ } else if (fileName.contains("long")) {
+ dataType = TSDataType.INT64;
+ ArrayList<Long> tmp = new ArrayList<>();
+ for (String value : data) {
+ tmp.add(Long.valueOf(value));
+ }
+ for (TSEncoding scheme : schemeList) {
+ Encoder encoder = TSEncodingBuilder.getEncodingBuilder(scheme).getEncoder(dataType);
+ Decoder decoder = Decoder.getDecoderByType(scheme, dataType);
+ long encodeTime = 0;
+ long decodeTime = 0;
+ for (CompressionType comp : compressList) {
+ ICompressor compressor = ICompressor.getCompressor(comp);
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(comp);
+ long compressTime = 0;
+ long uncompressTime = 0;
+ double ratio = 0;
+ for (int i = 0; i < repeatTime; i++) {
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+ long s = System.nanoTime();
+ for (long val : tmp) encoder.encode(val, buffer);
+ long e = System.nanoTime();
+ encodeTime += (e - s);
+ System.out.println("encodeTime:");
+ System.out.println(e - s);
+ encoder.flush(buffer);
+
+ byte[] elems = buffer.toByteArray();
+ s = System.nanoTime();
+ byte[] compressed = compressor.compress(elems);
+ e = System.nanoTime();
+ compressTime += (e - s);
+
+ int ratioTmp = compressed.length / (tmp.size() * Long.BYTES);
+ ratio += ratioTmp;
+
+ s = System.nanoTime();
+ byte[] x = unCompressor.uncompress(compressed);
+ e = System.nanoTime();
+ uncompressTime += (e - s);
+
+ ByteBuffer ebuffer = ByteBuffer.wrap(buffer.toByteArray());
+ s = System.nanoTime();
+ while (decoder.hasNext(ebuffer)) {
+ decoder.readLong(ebuffer);
+ }
+ e = System.nanoTime();
+ decodeTime += (e - s);
+ buffer.close();
+ }
+ ratio /= repeatTime;
+ encodeTime = encodeTime / ((long) repeatTime);
+ decodeTime = decodeTime / ((long) repeatTime);
+ // compressTime = compressTime / ((long) repeatTime);
+ // uncompressTime = uncompressTime / ((long) repeatTime);
+ System.out.println(scheme.toString());
+ // System.out.println(String.valueOf(encodeTime));
+ String[] record = {
+ scheme.toString(),
+ comp.toString(),
+ String.valueOf(encodeTime),
+ "",
+ "",
+ String.valueOf(decodeTime),
+ String.valueOf(ratio)
+ };
+
+ // String[] record = {scheme.toString(), comp.toString(),
+ // String.valueOf(encodeTime), String.valueOf(compressTime),
+ // String.valueOf(uncompressTime),
+ // String.valueOf(decodeTime), String.valueOf(ratio)};
+ writer.writeRecord(record);
+ }
+ }
+ } else if (fileName.contains("double")) {
+ dataType = TSDataType.DOUBLE;
+ ArrayList<Double> tmp = new ArrayList<>();
+ for (String value : data) {
+ tmp.add(Double.valueOf(value));
+ }
+ for (TSEncoding scheme : schemeList) {
+ Encoder encoder = TSEncodingBuilder.getEncodingBuilder(scheme).getEncoder(dataType);
+ Decoder decoder = Decoder.getDecoderByType(scheme, dataType);
+ long encodeTime = 0;
+ long decodeTime = 0;
+ for (CompressionType comp : compressList) {
+ ICompressor compressor = ICompressor.getCompressor(comp);
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(comp);
+ long compressTime = 0;
+ long uncompressTime = 0;
+ double ratio = 0;
+ for (int i = 0; i < repeatTime; i++) {
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+ long s = System.nanoTime();
+ for (double val : tmp) encoder.encode(val, buffer);
+ long e = System.nanoTime();
+ encodeTime += (e - s);
+ System.out.println("encodeTime:");
+ System.out.println(e - s);
+ encoder.flush(buffer);
+
+ byte[] elems = buffer.toByteArray();
+ s = System.nanoTime();
+ byte[] compressed = compressor.compress(elems);
+ e = System.nanoTime();
+ compressTime += (e - s);
+
+ int ratioTmp = compressed.length / (tmp.size() * Long.BYTES);
+ ratio += ratioTmp;
+
+ s = System.nanoTime();
+ byte[] x = unCompressor.uncompress(compressed);
+ e = System.nanoTime();
+ uncompressTime += (e - s);
+
+ ByteBuffer ebuffer = ByteBuffer.wrap(buffer.toByteArray());
+ s = System.nanoTime();
+ while (decoder.hasNext(ebuffer)) {
+ decoder.readDouble(ebuffer);
+ }
+ e = System.nanoTime();
+ decodeTime += (e - s);
+ buffer.close();
+ }
+ ratio /= repeatTime;
+ encodeTime = encodeTime / ((long) repeatTime);
+ decodeTime = decodeTime / ((long) repeatTime);
+ // compressTime = compressTime / ((long) repeatTime);
+ // uncompressTime = uncompressTime / ((long) repeatTime);
+ System.out.println(scheme.toString());
+ // System.out.println(String.valueOf(encodeTime));
+ String[] record = {
+ scheme.toString(),
+ comp.toString(),
+ String.valueOf(encodeTime),
+ "",
+ "",
+ String.valueOf(decodeTime),
+ String.valueOf(ratio)
+ };
+
+ // String[] record = {scheme.toString(), comp.toString(),
+ // String.valueOf(encodeTime), String.valueOf(compressTime),
+ // String.valueOf(uncompressTime),
+ // String.valueOf(decodeTime), String.valueOf(ratio)};
+ writer.writeRecord(record);
+ }
+ }
+ } else if (fileName.contains("float")) {
+ dataType = TSDataType.FLOAT;
+ ArrayList<Float> tmp = new ArrayList<>();
+ for (String value : data) {
+ tmp.add(Float.valueOf(value));
+ }
+ for (TSEncoding scheme : schemeList) {
+ Encoder encoder = TSEncodingBuilder.getEncodingBuilder(scheme).getEncoder(dataType);
+ Decoder decoder = Decoder.getDecoderByType(scheme, dataType);
+ long encodeTime = 0;
+ long decodeTime = 0;
+ for (CompressionType comp : compressList) {
+ ICompressor compressor = ICompressor.getCompressor(comp);
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(comp);
+ long compressTime = 0;
+ long uncompressTime = 0;
+ double ratio = 0;
+ for (int i = 0; i < repeatTime; i++) {
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+ long s = System.nanoTime();
+ for (float val : tmp) encoder.encode(val, buffer);
+ long e = System.nanoTime();
+ encodeTime += (e - s);
+ System.out.println("encodeTime:");
+ System.out.println(e - s);
+ encoder.flush(buffer);
+
+ byte[] elems = buffer.toByteArray();
+ s = System.nanoTime();
+ byte[] compressed = compressor.compress(elems);
+ e = System.nanoTime();
+ compressTime += (e - s);
+
+ int ratioTmp = compressed.length / (tmp.size() * Long.BYTES);
+ ratio += ratioTmp;
+
+ s = System.nanoTime();
+ byte[] x = unCompressor.uncompress(compressed);
+ e = System.nanoTime();
+ uncompressTime += (e - s);
+
+ ByteBuffer ebuffer = ByteBuffer.wrap(buffer.toByteArray());
+ s = System.nanoTime();
+ while (decoder.hasNext(ebuffer)) {
+ decoder.readFloat(ebuffer);
+ }
+ e = System.nanoTime();
+ decodeTime += (e - s);
+ buffer.close();
+ }
+ ratio /= repeatTime;
+ encodeTime = encodeTime / ((long) repeatTime);
+ decodeTime = decodeTime / ((long) repeatTime);
+ // compressTime = compressTime / ((long) repeatTime);
+ // uncompressTime = uncompressTime / ((long) repeatTime);
+ System.out.println(scheme.toString());
+ // System.out.println(String.valueOf(encodeTime));
+ String[] record = {
+ scheme.toString(),
+ comp.toString(),
+ String.valueOf(encodeTime),
+ "",
+ "",
+ String.valueOf(decodeTime),
+ String.valueOf(ratio)
+ };
+
+ // String[] record = {scheme.toString(), comp.toString(),
+ // String.valueOf(encodeTime), String.valueOf(compressTime),
+ // String.valueOf(uncompressTime),
+ // String.valueOf(decodeTime), String.valueOf(ratio)};
+ writer.writeRecord(record);
+ }
+ }
+ } else if (fileName.contains("text")) {
+ dataType = TSDataType.FLOAT;
+ }
+
+ if (fileRepeat > 2) break;
+ }
+ writer.close();
+ }
+}
+// for (String value : data) {
+// tmp.add(Integer.valueOf(value));
+// }
+// for (TSEncoding scheme : schemeList) {
+// Encoder encoder =
+// TSEncodingBuilder.getEncodingBuilder(scheme).getEncoder(dataType);
+// Decoder decoder = Decoder.getDecoderByType(scheme, dataType);
+// long encodeTime = 0;
+// long decodeTime = 0;
+// for (CompressionType comp : compressList) {
+// ICompressor compressor = ICompressor.getCompressor(comp);
+// IUnCompressor unCompressor = IUnCompressor.getUnCompressor(comp);
+// long compressTime = 0;
+// long uncompressTime = 0;
+// double ratio = tmp.size() * Integer.BYTES;
+//
+// for(int i=0;i<repeatTime;i++){
+//
+// ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+//
+// long s = System.nanoTime();
+// for (int val : tmp) encoder.encode(val, buffer);
+// long e = System.nanoTime();
+// System.out.println("e-s:");
+// System.out.println(e-s);
+// encodeTime += (e - s);
+// encoder.flush(buffer);
+//
+// byte[] elems = buffer.toByteArray();
+// s = System.nanoTime();
+// byte[] compressed = compressor.compress(elems);
+// e = System.nanoTime();
+// compressTime += (e - s);
+//
+// int ratioTmp = compressed.length / (tmp.size() * Long.BYTES);
+// ratio += ratioTmp;
+//
+// s = System.nanoTime();
+// byte[] x = unCompressor.uncompress(compressed);
+// e = System.nanoTime();
+// uncompressTime += (e - s);
+//
+// ByteBuffer ebuffer = ByteBuffer.wrap(x);
+// s = System.nanoTime();
+// while(decoder.hasNext(ebuffer)){
+// decoder.readInt(ebuffer);
+// }
+// e = System.nanoTime();
+// decodeTime += (e - s);
+// buffer.close();
+// }
+//// System.out.println(encodeTime);
+// ratio /= repeatTime;
+// encodeTime = encodeTime * 2 / ((long) repeatTime);
+// decodeTime = decodeTime * 2 / ((long) repeatTime);
+// compressTime = compressTime * 2 / ((long) repeatTime);
+// uncompressTime = uncompressTime * 2 / ((long) repeatTime);
+// System.out.println(scheme.toString());
+//// System.out.println(String.valueOf(encodeTime));
+//// String[] record = {scheme.toString(), comp.toString(),
+// String.valueOf(encodeTime), "",
+//// "", String.valueOf(decodeTime), String.valueOf(ratio)};
+//
+// String[] record = {scheme.toString(), comp.toString(),
+// String.valueOf(encodeTime), String.valueOf(compressTime),
+// String.valueOf(uncompressTime), String.valueOf(decodeTime),
+// String.valueOf(ratio)};
+// writer.writeRecord(record);
+// }
+// }
+// else if (fileName.contains("double")) {
+// ArrayList<Double> tmp = new ArrayList<>();
+// dataType = TSDataType.DOUBLE;
+// for (String value : data) {
+// tmp.add(Double.valueOf(value));
+// }
+// for (TSEncoding scheme : schemeList) {
+// Encoder encoder =
+// TSEncodingBuilder.getEncodingBuilder(scheme).getEncoder(dataType);
+// Decoder decoder = Decoder.getDecoderByType(scheme, dataType);
+// long encodeTime = 0;
+// long decodeTime = 0;
+// for (int i = 0; i < repeatTime; i++) {
+// ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+// long s = System.nanoTime();
+// for (double val : tmp) encoder.encode(val, buffer);
+// long e = System.nanoTime();
+// encodeTime += (e - s);
+//
+// ByteBuffer ebuffer = ByteBuffer.wrap(buffer.toByteArray());
+// s = System.nanoTime();
+// for (int c = 0; c < tmp.size(); c++) decoder.readDouble(ebuffer);
+// e = System.nanoTime();
+// decodeTime += (e - s);
+// }
+//
+// encodeTime /= tmp.size();
+// decodeTime /= tmp.size();
+// String[] record = {scheme.toString(), "DOUBLE", String.valueOf(encodeTime),
+// String.valueOf(decodeTime)};
+// writer.writeRecord(record);
+// }
+// }
+// else if (fileName.contains("float")) {
+// ArrayList<Float> tmp = new ArrayList<>();
+// dataType = TSDataType.FLOAT;
+// for (String value : data) {
+// tmp.add(Float.valueOf(value));
+// }
+// System.out.println(tmp);
+// for (TSEncoding scheme : schemeList) {
+// Encoder encoder =
+// TSEncodingBuilder.getEncodingBuilder(scheme).getEncoder(dataType);
+// Decoder decoder = Decoder.getDecoderByType(scheme, dataType);
+// long encodeTime = 0;
+// long decodeTime = 0;
+// for (int i = 0; i < repeatTime; i++) {
+// ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+// long s = System.nanoTime();
+// for (float val : tmp) encoder.encode(val, buffer);
+// long e = System.nanoTime();
+// encodeTime += (e - s);
+//
+// ByteBuffer ebuffer = ByteBuffer.wrap(buffer.toByteArray());
+// s = System.nanoTime();
+// for (int c = 0; c < tmp.size(); c++) decoder.readFloat(ebuffer);
+// e = System.nanoTime();
+// decodeTime += (e - s);
+// }
+//
+// encodeTime /= tmp.size();
+// decodeTime /= tmp.size();
+// String[] record = {scheme.toString(), "FLOAT", String.valueOf(encodeTime),
+// String.valueOf(decodeTime)};
+// writer.writeRecord(record);
+// }
+// }
+// else if (fileName.contains("text")) {
+// ArrayList<Byte> tmp = new ArrayList<>();
+// dataType = TSDataType.FLOAT;
+// for (String value : data) {
+// tmp.add(Byte.valueOf(value));
+// }
+// for (TSEncoding scheme : schemeList) {
+// Encoder encoder =
+// TSEncodingBuilder.getEncodingBuilder(scheme).getEncoder(dataType);
+// Decoder decoder = Decoder.getDecoderByType(scheme, dataType);
+// long encodeTime = 0;
+// long decodeTime = 0;
+// for (int i = 0; i < repeatTime; i++) {
+// ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+// long s = System.nanoTime();
+// for (byte val : tmp) encoder.encode(val, buffer);
+// long e = System.nanoTime();
+// encodeTime += (e - s);
+//
+// ByteBuffer ebuffer = ByteBuffer.wrap(buffer.toByteArray());
+// s = System.nanoTime();
+// for (int c = 0; c < tmp.size(); c++) decoder.readBinary(ebuffer);
+// e = System.nanoTime();
+// decodeTime += (e - s);
+// }
+//
+// encodeTime /= tmp.size();
+// decodeTime /= tmp.size();
+// String[] record = {scheme.toString(), "TEXT", String.valueOf(encodeTime),
+// String.valueOf(decodeTime)};
+// writer.writeRecord(record);
+// }
+// } //else throw new NotImplementedException();
+// //System.gc();
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/HuffmanDecoderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/HuffmanDecoderTest.java
new file mode 100644
index 0000000000..cad6711d2a
--- /dev/null
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/HuffmanDecoderTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import org.apache.iotdb.tsfile.encoding.encoder.HuffmanEncoder;
+import org.apache.iotdb.tsfile.utils.Binary;
+
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.nio.ByteBuffer;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class HuffmanDecoderTest {
+ private HuffmanEncoder encoder = new HuffmanEncoder();
+ private HuffmanDecoder decoder = new HuffmanDecoder();
+ private ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ @Test
+ public void testSingle() {
+ testAll("a");
+ testAll("b");
+ testAll("c");
+ }
+
+ @Test
+ public void testAllUnique() {
+ testAll("a", "b", "c");
+ testAll("x", "o", "q");
+ testAll(",", ".", "c", "b", "e");
+ }
+
+ @Test
+ public void testAllSame() {
+ testAll("a", "a", "a");
+ testAll("b", "b", "b");
+ }
+
+ @Test
+ public void testConcatenated() {
+ testAll("aaa", "bbbb", "ccaeffsrhha");
+ }
+
+ @Test
+ public void testMinus() {
+ // all characters
+ String[] allChars = new String[1];
+ allChars[0] = "" + (char) ('a' + 1);
+ allChars[0] = "" + (char) (213);
+ testAll(allChars);
+ }
+
+ @Test
+ public void testMixed() {
+ // all characters
+ String[] allChars = new String[256];
+ allChars[0] = "" + (char) ('a' + 1);
+ for (int i = 0; i < 256; i++) {
+ allChars[i] = "" + (char) (i) + (char) (i) + (char) (i);
+ }
+ testAll(allChars);
+ }
+
+ private void testAll(String... all) {
+ for (String s : all) {
+ Binary temp = new Binary(s);
+ encoder.encode(temp, baos);
+ }
+ encoder.flush(baos);
+
+ ByteBuffer out = ByteBuffer.wrap(baos.toByteArray());
+
+ for (String s : all) {
+ assertTrue(decoder.hasNext(out));
+ Binary b = decoder.readBinary(out);
+ assertEquals(s, b.getStringValue());
+ }
+
+ decoder.reset();
+ baos.reset();
+ }
+}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/PLAINDecoderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/PLAINDecoderTest.java
new file mode 100644
index 0000000000..3ad6b56d23
--- /dev/null
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/PLAINDecoderTest.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import org.apache.iotdb.tsfile.encoding.encoder.IntRleEncoder;
+import org.apache.iotdb.tsfile.encoding.encoder.RleEncoder;
+import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+public class PLAINDecoderTest {
+
+ private List<Integer> rleList;
+ private List<Integer> bpList;
+ private List<Integer> hybridList;
+
+ @Before
+ public void setUp() {
+ rleList = new ArrayList<>();
+ int rleCount = 11;
+ int rleNum = 18;
+ int rleStart = 11;
+ for (int i = 0; i < rleNum; i++) {
+ for (int j = 0; j < rleCount; j++) {
+ rleList.add(rleStart);
+ }
+ for (int j = 0; j < rleCount; j++) {
+ rleList.add(rleStart - 1);
+ }
+ rleCount += 2;
+ rleStart *= -3;
+ }
+
+ bpList = new ArrayList<>(100000);
+ int bpCount = 100000;
+ int bpStart = 11;
+ for (int i = 0; i < bpCount; i++) {
+ bpStart += 3;
+ if (i % 2 == 1) {
+ bpList.add(bpStart * -1);
+ } else {
+ bpList.add(bpStart);
+ }
+ }
+
+ int hybridCount = 11;
+ int hybridNum = 1000;
+ int hybridStart = 20;
+ hybridList = new ArrayList<>(hybridCount * 2 * hybridNum);
+ for (int i = 0; i < hybridNum; i++) {
+ for (int j = 0; j < hybridCount; j++) {
+ hybridStart += 3;
+ if (j % 2 == 1) {
+ hybridList.add(hybridStart * -1);
+ } else {
+ hybridList.add(hybridStart);
+ }
+ }
+ for (int j = 0; j < hybridCount; j++) {
+ if (i % 2 == 1) {
+ hybridList.add(hybridStart * -1);
+ } else {
+ hybridList.add(hybridStart);
+ }
+ }
+ hybridCount += 2;
+ }
+ }
+
+ @After
+ public void tearDown() {}
+
+ @Test
+ public void testPLAINReadBigInt() throws IOException {
+ List<Integer> list = new ArrayList<>(3000000);
+ for (int i = 7000000; i < 10000000; i++) {
+ list.add(i);
+ }
+ testLength(list, false, 1);
+ for (int i = 1; i < 10; i++) {
+ testLength(list, false, i);
+ }
+ }
+
+ @Test
+ public void testPLAINReadInt() throws IOException {
+ for (int i = 1; i < 10; i++) {
+ testLength(rleList, false, i);
+ }
+ }
+
+ @Test
+ public void testMaxPLAINRepeatNUM() throws IOException {
+ List<Integer> repeatList = new ArrayList<>();
+ int rleCount = 17;
+ int rleNum = 5;
+ int rleStart = 11;
+ for (int i = 0; i < rleNum; i++) {
+ for (int j = 0; j < rleCount; j++) {
+ repeatList.add(rleStart);
+ }
+ for (int j = 0; j < rleCount; j++) {
+ repeatList.add(rleStart / 3);
+ }
+ rleCount *= 7;
+ rleStart *= -3;
+ }
+ for (int i = 1; i < 10; i++) {
+ testLength(repeatList, false, i);
+ }
+ }
+
+ @Test
+ public void testBitPackingReadInt() throws IOException {
+ for (int i = 1; i < 10; i++) {
+ testLength(bpList, false, i);
+ }
+ }
+
+ @Test
+ public void testHybridReadInt() throws IOException {
+ for (int i = 1; i < 3; i++) {
+ testLength(hybridList, false, i);
+ }
+ }
+
+ @Test
+ public void testHybridReadBoolean() throws IOException {
+ for (int i = 1; i < 10; i++) {
+ testLength(hybridList, false, i);
+ }
+ }
+
+ @Test
+ public void testBitPackingReadHeader() throws IOException {
+ for (int i = 1; i < 505; i++) {
+ testBitPackedReadHeader(i);
+ }
+ }
+
+ public void testLength(List<Integer> list, boolean isDebug, int repeatCount) throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ RleEncoder<Integer> encoder = new IntRleEncoder();
+ for (int i = 0; i < repeatCount; i++) {
+ for (int value : list) {
+ encoder.encode(value, baos);
+ }
+ encoder.flush(baos);
+ }
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+ RleDecoder decoder = new IntRleDecoder();
+ for (int i = 0; i < repeatCount; i++) {
+ for (int value : list) {
+ int value_ = decoder.readInt(buffer);
+ if (isDebug) {
+ System.out.println(value_ + "/" + value);
+ }
+ assertEquals(value, value_);
+ }
+ }
+ }
+
+ private void testBitPackedReadHeader(int num) throws IOException {
+ List<Integer> list = new ArrayList<>();
+
+ for (int i = 0; i < num; i++) {
+ list.add(i);
+ }
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ int bitWidth = ReadWriteForEncodingUtils.getIntMaxBitWidth(list);
+ RleEncoder<Integer> encoder = new IntRleEncoder();
+ for (int value : list) {
+ encoder.encode(value, baos);
+ }
+ encoder.flush(baos);
+ ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+ ReadWriteForEncodingUtils.readUnsignedVarInt(bais);
+ assertEquals(bitWidth, bais.read());
+ int header = ReadWriteForEncodingUtils.readUnsignedVarInt(bais);
+ int group = header >> 1;
+ assertEquals(group, (num + 7) / 8);
+ int lastBitPackedNum = bais.read();
+ if (num % 8 == 0) {
+ assertEquals(lastBitPackedNum, 8);
+ } else {
+ assertEquals(lastBitPackedNum, num % 8);
+ }
+ }
+}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/RAKEDecoderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/RAKEDecoderTest.java
new file mode 100644
index 0000000000..24f34fccf2
--- /dev/null
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/RAKEDecoderTest.java
@@ -0,0 +1,626 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import org.apache.iotdb.tsfile.encoding.encoder.*;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public class RAKEDecoderTest {
+ private static final double DELTA = 0;
+ private static final int FLOAT_MAX_POINT_VALUE = 10000;
+ private static final long DOUBLE_MAX_POINT_VALUE = 1000000000000000L;
+ private static final List<Float> floatList = new ArrayList<>();
+ private static final List<Double> doubleList = new ArrayList<>();
+ private static final List<Integer> intList = new ArrayList<>();
+ private static final List<Long> longList = new ArrayList<>();
+
+ private static final List<Integer> iterations = new ArrayList<>();
+
+ @BeforeClass
+ public static void setUp() {
+ int hybridCount = 11;
+ int hybridNum = 50;
+ int hybridStart = 2000;
+ for (int i = 0; i < hybridNum; i++) {
+ for (int j = 0; j < hybridCount; j++) {
+ floatList.add((float) hybridStart / FLOAT_MAX_POINT_VALUE);
+ doubleList.add((double) hybridStart / DOUBLE_MAX_POINT_VALUE);
+ intList.add(hybridStart);
+ longList.add((long) hybridStart);
+ }
+ for (int j = 0; j < hybridCount; j++) {
+ floatList.add((float) hybridStart / FLOAT_MAX_POINT_VALUE);
+ doubleList.add((double) hybridStart / DOUBLE_MAX_POINT_VALUE);
+ intList.add(hybridStart);
+ longList.add((long) hybridStart);
+ hybridStart += 3;
+ }
+ hybridCount += 2;
+ }
+
+ iterations.add(1);
+ iterations.add(3);
+ iterations.add(8);
+ iterations.add(16);
+ iterations.add(1000);
+ iterations.add(10000);
+ }
+
+ @Test
+ public void testIntSingleValue() throws IOException {
+ Encoder encoder = new IntRAKEEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ encoder.encode(777, baos);
+ encoder.flush(baos);
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ Decoder decoder = new IntRAKEDecoder();
+ if (decoder.hasNext(buffer)) {
+ assertEquals(777, decoder.readInt(buffer));
+ }
+ if (decoder.hasNext(buffer)) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testFloatSingleValue() throws IOException {
+ Encoder encoder = new FloatRAKEEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ encoder.encode(Float.MAX_VALUE, baos);
+ encoder.flush(baos);
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ Decoder decoder = new FloatRAKEDecoder();
+ if (decoder.hasNext(buffer)) {
+ assertEquals(Float.MAX_VALUE, decoder.readFloat(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testLongSingleValue() throws IOException {
+ Encoder encoder = new LongRAKEEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ encoder.encode((long) Integer.MAX_VALUE + 10, baos);
+ encoder.flush(baos);
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ Decoder decoder = new LongRAKEDecoder();
+ if (decoder.hasNext(buffer)) {
+ assertEquals((long) Integer.MAX_VALUE + 10, decoder.readLong(buffer));
+ }
+ if (decoder.hasNext(buffer)) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testDoubleSingleValue() throws IOException {
+ Encoder encoder = new DoubleRAKEEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ encoder.encode(Double.MAX_VALUE, baos);
+ encoder.flush(baos);
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ Decoder decoder = new DoubleRAKEDecoder();
+ if (decoder.hasNext(buffer)) {
+ assertEquals(Double.MAX_VALUE, decoder.readDouble(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testIntZeroNumber() throws IOException {
+ Encoder encoder = new IntRAKEEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ int value = 0;
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.flush(baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.flush(baos);
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+ for (int i = 0; i < 2; i++) {
+ Decoder decoder = new IntRAKEDecoder();
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readInt(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readInt(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readInt(buffer), DELTA);
+ }
+ }
+ }
+
+ @Test
+ public void testFloatZeroNumber() throws IOException {
+ Encoder encoder = new FloatRAKEEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ float value = 0f;
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.flush(baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.flush(baos);
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+ for (int i = 0; i < 2; i++) {
+ Decoder decoder = new FloatRAKEDecoder();
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readFloat(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readFloat(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readFloat(buffer), DELTA);
+ }
+ }
+ }
+
+ @Test
+ public void testLongZeroNumber() throws IOException {
+ Encoder encoder = new LongRAKEEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ long value = 0;
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.flush(baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.flush(baos);
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+ for (int i = 0; i < 2; i++) {
+ Decoder decoder = new LongRAKEDecoder();
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readLong(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readLong(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readLong(buffer), DELTA);
+ }
+ }
+ }
+
+ @Test
+ public void testDoubleZeroNumber() throws IOException {
+ Encoder encoder = new DoubleRAKEEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ double value = 0f;
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.flush(baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.flush(baos);
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+ for (int i = 0; i < 2; i++) {
+ Decoder decoder = new DoubleRAKEDecoder();
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readDouble(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readDouble(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readDouble(buffer), DELTA);
+ }
+ }
+ }
+
+ @Test
+ public void testInteger() throws IOException {
+
+ for (Integer num : iterations) {
+
+ Encoder encoder = new IntRAKEEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ int value = 7;
+ for (int i = 0; i < num; i++) {
+ encoder.encode(value + 2 * i, baos);
+ }
+ encoder.flush(baos);
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ Decoder decoder = new IntRAKEDecoder();
+ for (int i = 0; i < num; i++) {
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value + 2 * i, decoder.readInt(buffer));
+ continue;
+ }
+ fail();
+ }
+ if (decoder.hasNext(buffer)) {
+ fail();
+ }
+ }
+ }
+
+ @Test
+ public void testIntegerTime() throws IOException {
+ int num = 100000;
+ long encode_time = 0;
+ for (int k = 0; k < 10; k++) {
+ Encoder encoder = new DeltaBinaryEncoder.IntDeltaEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ int value = 7;
+ long start_time = System.currentTimeMillis();
+ for (int i = 0; i < num; i++) {
+ encoder.encode(value + 2 * i, baos);
+ }
+ encoder.flush(baos);
+ long end_time = System.currentTimeMillis();
+ System.out.println(end_time - start_time + "ms");
+ encode_time += end_time - start_time;
+ }
+ System.out.println(encode_time / 10);
+ //
+ //
+ // ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+ //
+ // Decoder decoder = new IntRAKEDecoder();
+ // for (int i = 0; i < num; i++) {
+ // if (decoder.hasNext(buffer)) {
+ // assertEquals(value + 2 * i, decoder.readInt(buffer));
+ // continue;
+ // }
+ // fail();
+ // }
+ // if (decoder.hasNext(buffer)) {
+ // fail();
+ // }
+ // }
+ }
+
+ @Test
+ public void testFloat() throws IOException {
+ for (Integer num : iterations) {
+ Encoder encoder = new FloatRAKEEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ float value = 7.101f;
+ for (int i = 0; i < num; i++) {
+ encoder.encode(value + 2 * i, baos);
+ }
+ encoder.flush(baos);
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ Decoder decoder = new FloatRAKEDecoder();
+ for (int i = 0; i < num; i++) {
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value + 2 * i, decoder.readFloat(buffer), DELTA);
+ continue;
+ }
+ fail();
+ }
+ if (decoder.hasNext(buffer)) {
+ fail();
+ }
+ }
+ }
+
+ @Test
+ public void testLong() throws IOException {
+ for (Integer num : iterations) {
+ Encoder encoder = new LongRAKEEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ long value = 7;
+ for (int i = 0; i < num; i++) {
+ encoder.encode(value + 2 * i, baos);
+ }
+ encoder.flush(baos);
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ Decoder decoder = new LongRAKEDecoder();
+ for (int i = 0; i < num; i++) {
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value + 2 * i, decoder.readLong(buffer));
+ continue;
+ }
+ fail();
+ }
+ if (decoder.hasNext(buffer)) {
+ fail();
+ }
+ }
+ }
+
+ @Test
+ public void testDouble() throws IOException {
+ for (Integer num : iterations) {
+ Encoder encoder = new DoubleRAKEEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ double value = 7.101f;
+ for (int i = 0; i < num; i++) {
+ encoder.encode(value + 2 * i, baos);
+ }
+ encoder.flush(baos);
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ Decoder decoder = new DoubleRAKEDecoder();
+ for (int i = 0; i < num; i++) {
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value + 2 * i, decoder.readDouble(buffer), DELTA);
+ continue;
+ }
+ fail();
+ }
+ if (decoder.hasNext(buffer)) {
+ fail();
+ }
+ }
+ }
+
+ @Test
+ public void testIntegerRepeat() throws Exception {
+ for (int i = 1; i <= 10; i++) {
+ testInteger(i);
+ }
+ }
+
+ private void testInteger(int repeatCount) throws Exception {
+ Encoder encoder = new IntRAKEEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ for (int i = 0; i < repeatCount; i++) {
+ for (int value : RAKEDecoderTest.intList) {
+ encoder.encode(value, baos);
+ }
+ encoder.flush(baos);
+ }
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ for (int i = 0; i < repeatCount; i++) {
+ Decoder decoder = new IntRAKEDecoder();
+ for (int expected : RAKEDecoderTest.intList) {
+ if (decoder.hasNext(buffer)) {
+ int actual = decoder.readInt(buffer);
+ assertEquals(expected, actual);
+ continue;
+ }
+ fail();
+ }
+ }
+
+ encoder = new IntRAKEEncoder();
+ baos = new ByteArrayOutputStream();
+ for (int i = 0; i < repeatCount; i++) {
+ for (int value : RAKEDecoderTest.intList) {
+ encoder.encode(-value, baos);
+ }
+ encoder.flush(baos);
+ }
+ encoder.flush(baos);
+ buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ for (int i = 0; i < repeatCount; i++) {
+ IntRAKEDecoder decoder = new IntRAKEDecoder();
+ for (int expected : RAKEDecoderTest.intList) {
+ if (decoder.hasNext(buffer)) {
+ int actual = decoder.readInt(buffer);
+ assertEquals(expected, -actual);
+ continue;
+ }
+ fail();
+ }
+ }
+ }
+
+ @Test
+ public void testFloatRepeat() throws Exception {
+ for (int i = 1; i <= 10; i++) {
+ testFloat(i);
+ }
+ }
+
+ private void testFloat(int repeatCount) throws Exception {
+ Encoder encoder = new FloatRAKEEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ for (int i = 0; i < repeatCount; i++) {
+ for (float value : RAKEDecoderTest.floatList) {
+ encoder.encode(value, baos);
+ }
+ encoder.flush(baos);
+ }
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ for (int i = 0; i < repeatCount; i++) {
+ Decoder decoder = new FloatRAKEDecoder();
+ for (float expected : RAKEDecoderTest.floatList) {
+ if (decoder.hasNext(buffer)) {
+ float actual = decoder.readFloat(buffer);
+ assertEquals(expected, actual, DELTA);
+ continue;
+ }
+ fail();
+ }
+ }
+
+ encoder = new FloatRAKEEncoder();
+ baos = new ByteArrayOutputStream();
+ for (int i = 0; i < repeatCount; i++) {
+ for (float value : RAKEDecoderTest.floatList) {
+ encoder.encode(-value, baos);
+ }
+ encoder.flush(baos);
+ }
+
+ buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ for (int i = 0; i < repeatCount; i++) {
+ Decoder decoder = new FloatRAKEDecoder();
+ for (float expected : RAKEDecoderTest.floatList) {
+ if (decoder.hasNext(buffer)) {
+ float actual = decoder.readFloat(buffer);
+ assertEquals(expected, -actual, DELTA);
+ continue;
+ }
+ fail();
+ }
+ }
+ }
+
+ @Test
+ public void testLongRepeat() throws Exception {
+ for (int i = 1; i <= 10; i++) {
+ testLong(i);
+ }
+ }
+
+ private void testLong(int repeatCount) throws Exception {
+ Encoder encoder = new LongRAKEEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ for (int i = 0; i < repeatCount; i++) {
+ for (long value : RAKEDecoderTest.longList) {
+ encoder.encode(value, baos);
+ }
+ encoder.flush(baos);
+ }
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+ for (int i = 0; i < repeatCount; i++) {
+ Decoder decoder = new LongRAKEDecoder();
+
+ for (long expected : RAKEDecoderTest.longList) {
+ if (decoder.hasNext(buffer)) {
+ long actual = decoder.readLong(buffer);
+ assertEquals(expected, actual);
+ continue;
+ }
+ fail();
+ }
+ }
+
+ encoder = new LongRAKEEncoder();
+ baos = new ByteArrayOutputStream();
+ for (int i = 0; i < repeatCount; i++) {
+ for (long value : RAKEDecoderTest.longList) {
+ encoder.encode(-value, baos);
+ }
+ encoder.flush(baos);
+ }
+
+ buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ for (int i = 0; i < repeatCount; i++) {
+ Decoder decoder = new LongRAKEDecoder();
+ for (long expected : RAKEDecoderTest.longList) {
+ if (decoder.hasNext(buffer)) {
+ long actual = decoder.readLong(buffer);
+ assertEquals(expected, -actual);
+ continue;
+ }
+ fail();
+ }
+ }
+ }
+
+ @Test
+ public void testDoubleRepeat() throws Exception {
+ for (int i = 1; i <= 10; i++) {
+ testDouble(i);
+ }
+ }
+
+ private void testDouble(int repeatCount) throws Exception {
+ Encoder encoder = new DoubleRAKEEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ for (int i = 0; i < repeatCount; i++) {
+ for (double value : RAKEDecoderTest.doubleList) {
+ encoder.encode(value, baos);
+ }
+ encoder.flush(baos);
+ }
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ for (int i = 0; i < repeatCount; i++) {
+ Decoder decoder = new DoubleRAKEDecoder();
+ for (double expected : RAKEDecoderTest.doubleList) {
+ if (decoder.hasNext(buffer)) {
+ double actual = decoder.readDouble(buffer);
+ assertEquals(expected, actual, DELTA);
+ continue;
+ }
+ fail();
+ }
+ }
+
+ encoder = new DoubleRAKEEncoder();
+ baos = new ByteArrayOutputStream();
+ for (int i = 0; i < repeatCount; i++) {
+ for (double value : RAKEDecoderTest.doubleList) {
+ encoder.encode(-value, baos);
+ }
+ encoder.flush(baos);
+ }
+
+ buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ for (int i = 0; i < repeatCount; i++) {
+ Decoder decoder = new DoubleRAKEDecoder();
+ for (double expected : RAKEDecoderTest.doubleList) {
+ if (decoder.hasNext(buffer)) {
+ double actual = decoder.readDouble(buffer);
+ assertEquals(expected, -actual, DELTA);
+ continue;
+ }
+ fail();
+ }
+ }
+ }
+}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/TextRleDecoderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/TextRleDecoderTest.java
new file mode 100644
index 0000000000..6db212a7e2
--- /dev/null
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/TextRleDecoderTest.java
@@ -0,0 +1,125 @@
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import org.apache.iotdb.tsfile.encoding.encoder.TextRleEncoder;
+import org.apache.iotdb.tsfile.utils.Binary;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+public class TextRleDecoderTest {
+ private List<Binary> rleList;
+ private List<Binary> bpList;
+
+ @Before
+ public void setUp() {
+ rleList = new ArrayList<>();
+ int rleCount = 11;
+ int rleNum = 38;
+ String textStr1 = "This is a string";
+ Binary rleStart = new Binary(textStr1);
+ for (int i = 0; i < rleNum; i++) {
+ for (int j = 0; j < rleCount; j++) {
+ rleList.add(rleStart);
+ }
+ for (int j = 0; j < rleCount; j++) {
+ rleList.add(new Binary(textStr1 + 1));
+ }
+ rleCount += 2;
+ }
+ bpList = new ArrayList<>();
+ int bpCount = 15;
+ String textStr2 = "Thatisateststring2.";
+ Binary bpStart = new Binary(textStr2);
+ for (int i = 0; i < bpCount; i++) {
+ textStr2 += 3;
+ if (i % 2 == 1) {
+ bpList.add(new Binary(textStr2));
+ } else {
+ bpList.add(bpStart);
+ }
+ }
+ }
+
+ @After
+ public void tearDown() {}
+
+ @Test
+ public void testSingleText() throws IOException {
+ Binary text = new Binary("123456789101");
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ TextRleEncoder encoder = new TextRleEncoder();
+ encoder.encode(text, baos);
+ encoder.flush(baos);
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+ TextRleDecoder decoder = new TextRleDecoder();
+ Binary text2 = decoder.readBinary(buffer);
+ assertEquals(text, text2);
+ }
+
+ @Test
+ public void testRleReadLong() throws IOException {
+ for (int i = 1; i < 2; i++) {
+ testLength(rleList, false, i);
+ }
+ }
+
+ @Test
+ public void testMaxRLERepeatNUM() throws IOException {
+ List<Binary> repeatList = new ArrayList<>();
+ int rleCount = 17;
+ int rleNum = 5;
+ long rleStart = 11;
+ for (int i = 0; i < rleNum; i++) {
+ for (int j = 0; j < rleCount; j++) {
+ repeatList.add(new Binary(String.valueOf(rleStart)));
+ }
+ for (int j = 0; j < rleCount; j++) {
+ repeatList.add(new Binary(String.valueOf(rleStart / 3)));
+ }
+ rleCount *= 7;
+ rleStart *= -3;
+ }
+ for (int i = 1; i < 10; i++) {
+ testLength(repeatList, false, i);
+ }
+ }
+
+ @Test
+ public void testBitPackingReadLong() throws IOException {
+ for (int i = 1; i < 10; i++) {
+ testLength(bpList, false, i);
+ }
+ }
+
+ public void testLength(List<Binary> list, boolean isDebug, int repeatCount) throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ TextRleEncoder encoder = new TextRleEncoder();
+ for (int i = 0; i < repeatCount; i++) {
+ for (Binary value : list) {
+ encoder.encode(value, baos);
+ }
+ encoder.flush(baos);
+ }
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+ TextRleDecoder decoder = new TextRleDecoder();
+ for (int i = 0; i < repeatCount; i++) {
+ for (Binary value : list) {
+ Binary value_ = decoder.readBinary(buffer);
+ if (isDebug) {
+ System.out.println(value_ + "/" + value);
+ }
+ assertEquals(value, value_);
+ }
+ }
+ }
+}