You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hu...@apache.org on 2022/11/01 15:04:30 UTC
[iotdb] branch research/encoding-exp created (now 8b2f277d15)
This is an automated email from the ASF dual-hosted git repository.
hui pushed a change to branch research/encoding-exp
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at 8b2f277d15 update
This branch includes the following new commits:
new 8b2f277d15 update
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: update
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch research/encoding-exp
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 8b2f277d15e4b577e89f69c4a9d237576c2bfa4f
Author: xjz17 <67...@users.noreply.github.com>
AuthorDate: Tue Nov 1 22:55:18 2022 +0800
update
---
docs/UserGuide/Data-Concept/Encoding.md | 37 +-
.../resources/conf/iotdb-engine.properties | 2 +-
tsfile/pom.xml | 33 ++
.../iotdb/tsfile/common/conf/TSFileConfig.java | 3 +-
.../apache/iotdb/tsfile/compress/ICompressor.java | 99 ++++
.../iotdb/tsfile/compress/IUnCompressor.java | 46 ++
.../iotdb/tsfile/encoding/BitConstructor.java | 94 ++++
.../apache/iotdb/tsfile/encoding/BitReader.java | 67 +++
.../org/apache/iotdb/tsfile/encoding/DST/DST.java | 237 +++++++++
.../tsfile/encoding/HuffmanTree/HuffmanTree.java | 21 +
.../tsfile/encoding/decoder/BucketDecoder.java | 109 ++++
.../tsfile/encoding/decoder/HuffmanDecoder.java | 164 ++++++
.../tsfile/encoding/decoder/HuffmanDecoderV2.java | 138 +++++
.../tsfile/encoding/encoder/BucketEncoder.java | 145 +++++
.../encoding/encoder/DeltaBinaryEncoder.java | 3 +-
.../iotdb/tsfile/encoding/encoder/Encoder.java | 85 +++
.../tsfile/encoding/encoder/FloatEncoder.java | 5 +
.../tsfile/encoding/encoder/HuffmanEncoderV2.java | 257 +++++++++
.../tsfile/encoding/encoder/TSEncodingBuilder.java | 163 +++++-
.../tsfile/encoding/encoderbuff/BuffEncoder.java | 260 +++++++++
.../iotdb/tsfile/encoding/encoderbuff/Encoder.java | 17 +
.../file/metadata/enums/CompressionType.java | 7 +-
.../tsfile/file/metadata/enums/TSEncoding.java | 28 +
.../apache/iotdb/tsfile/compress/CompressTest.java | 12 +
.../tsfile/encoding/decoder/BucketDecoderTest.java | 92 ++++
.../tsfile/encoding/decoder/ContestEncodeTest.java | 328 ++++++++++++
.../iotdb/tsfile/encoding/decoder/EncodeTest.java | 453 ++++++++++++++++
.../tsfile/encoding/decoder/EncodeTestBuff.java | 72 +++
.../tsfile/encoding/decoder/FloatDecoderTest.java | 4 +-
.../encoding/decoder/HuffmanV2DecoderTest.java | 93 ++++
.../tsfile/encoding/decoder/RLBEDecoderTest.java | 257 +++++++++
.../encoding/decoder/SprintzDecoderTest.java | 589 +++++++++++++++++++++
.../delta/DeltaBinaryEncoderIntegerTest.java | 13 +-
33 files changed, 3917 insertions(+), 16 deletions(-)
diff --git a/docs/UserGuide/Data-Concept/Encoding.md b/docs/UserGuide/Data-Concept/Encoding.md
index 0bcafdd6ef..e277fb113c 100644
--- a/docs/UserGuide/Data-Concept/Encoding.md
+++ b/docs/UserGuide/Data-Concept/Encoding.md
@@ -50,10 +50,31 @@ Currently, there are two versions of GORILLA encoding implementation, it is reco
Usage restrictions: When using GORILLA to encode INT32 data, you need to ensure that there is no data point with the value `Integer.MIN_VALUE` in the sequence. When using GORILLA to encode INT64 data, you need to ensure that there is no data point with the value `Long.MIN_VALUE` in the sequence.
+* RAKE
+
+The RAKE encoding is based only on bits counting operations. It is more suitable for the ‘1’s of binary numbers to be more sparsely.
+
+* RLBE
+
+The RLBE encoding proposes to combine delta, run-length and Fibonacci based encoding ideas. It has five steps: differential coding, binary encoding, run-length, Fibonacci coding and concatenation.
+It is more suitable for the differential value of time series is positive and small.
+
+* SPRINTZ
+
+The SPRINTZ encoding combines encodings in four steps: predicting, bit-packing, run-length encoding and entropy encoding. SPRINTZ algorithm is suitable for predictable time series. For delta function, the vast repeats or linearly increasing time series is the best target.
+
* DICTIONARY
DICTIONARY encoding is lossless. It is suitable for TEXT data with low cardinality (i.e. low number of distinct values). It is not recommended to use it for high-cardinality data.
+* TEXTRLE
+
+TEXT Run-Length Encoding (TEXTRLE) performs especially for data with strings of repeated characters (the length of the string is called a run).
+
+* HUFFMAN
+
+It is more suitable for data with many high frequency values in skewed data distribution and many repeated characters.
+
## Correspondence between data type and encoding
@@ -63,13 +84,13 @@ The five encodings described in the previous sections are applicable to differen
**The correspondence between the data type and its supported encodings**
-|Data Type |Supported Encoding|
-|:---:|:---:|
-|BOOLEAN| PLAIN, RLE|
-|INT32 |PLAIN, RLE, TS_2DIFF, GORILLA|
-|INT64 |PLAIN, RLE, TS_2DIFF, GORILLA|
-|FLOAT |PLAIN, RLE, TS_2DIFF, GORILLA|
-|DOUBLE |PLAIN, RLE, TS_2DIFF, GORILLA|
-|TEXT |PLAIN, DICTIONARY|
+|Data Type | Supported Encoding |
+|:---:|:--------------------------------------------------:|
+|BOOLEAN| PLAIN, RLE |
+|INT32 | PLAIN, RLE, TS_2DIFF, GORILLA, RAKE, RLBE, SPRINTZ |
+|INT64 | PLAIN, RLE, TS_2DIFF, GORILLA, RAKE, RLBE, SPRINTZ |
+|FLOAT | PLAIN, RLE, TS_2DIFF, GORILLA, RAKE, RLBE, SPRINTZ |
+|DOUBLE | PLAIN, RLE, TS_2DIFF, GORILLA, RAKE, RLBE, SPRINTZ |
+|TEXT | PLAIN, DICTIONARY, TEXTRLE, HUFFMAN |
</div>
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index 33e9a767c4..b29a2074df 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -697,7 +697,7 @@ timestamp_precision=ms
# Floating-point precision
# Datatype: int
-# float_precision=2
+#float_precision=5
# Encoder configuration
# Encoder of time series, supports TS_2DIFF, PLAIN and RLE(run-length encoding), REGULAR and default value is TS_2DIFF
diff --git a/tsfile/pom.xml b/tsfile/pom.xml
index 10a39adbe7..7ca6e4d1db 100644
--- a/tsfile/pom.xml
+++ b/tsfile/pom.xml
@@ -58,6 +58,39 @@
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
</dependency>
+<<<<<<< Updated upstream
+=======
+ <dependency>
+ <groupId>net.sourceforge.javacsv</groupId>
+ <artifactId>javacsv</artifactId>
+ <version>2.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.jetbrains</groupId>
+ <artifactId>annotations</artifactId>
+ <version>RELEASE</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.carrotsearch</groupId>
+ <artifactId>hppc</artifactId>
+ <version>0.7.2</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.parquet</groupId>
+ <artifactId>parquet-hadoop-bundle</artifactId>
+ <version>1.8.1</version>
+ <groupId>org.tukaani</groupId>
+ <artifactId>xz</artifactId>
+ <version>1.9</version>
+ <scope>compile</scope>
+ </dependency>
+>>>>>>> Stashed changes
</dependencies>
<build>
<plugins>
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
index d13f817fd1..e76d8bafe6 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
@@ -82,7 +82,7 @@ public class TSFileConfig implements Serializable {
/** Max length limitation of input string. */
private int maxStringLength = 128;
/** Floating-point precision. */
- private int floatPrecision = 2;
+ private int floatPrecision = 5;
/**
* Encoder of time column, TsFile supports TS_2DIFF, PLAIN and RLE(run-length encoding) Default
* value is TS_2DIFF.
@@ -201,6 +201,7 @@ public class TSFileConfig implements Serializable {
}
public int getFloatPrecision() {
+ System.out.println(floatPrecision);
return floatPrecision;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/compress/ICompressor.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/compress/ICompressor.java
index 9243e53417..3ced75d0a7 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/compress/ICompressor.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/compress/ICompressor.java
@@ -25,7 +25,15 @@ import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import net.jpountz.lz4.LZ4Compressor;
import net.jpountz.lz4.LZ4Factory;
+<<<<<<< Updated upstream
+=======
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.tukaani.xz.XZInputStream;
+import org.tukaani.xz.XZOutputStream;
+>>>>>>> Stashed changes
import org.xerial.snappy.Snappy;
+import org.tukaani.xz.LZMA2Options;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
@@ -38,6 +46,7 @@ import java.util.zip.GZIPOutputStream;
import static org.apache.iotdb.tsfile.file.metadata.enums.CompressionType.GZIP;
import static org.apache.iotdb.tsfile.file.metadata.enums.CompressionType.LZ4;
import static org.apache.iotdb.tsfile.file.metadata.enums.CompressionType.SNAPPY;
+import static org.apache.iotdb.tsfile.file.metadata.enums.CompressionType.LZMA2;
/** compress data according to type in schema. */
public interface ICompressor extends Serializable {
@@ -65,6 +74,8 @@ public interface ICompressor extends Serializable {
return new IOTDBLZ4Compressor();
case GZIP:
return new GZIPCompressor();
+ case LZMA2:
+ return new LZMA2Compressor();
default:
throw new CompressionTypeNotSupportedException(name.toString());
}
@@ -311,4 +322,92 @@ public interface ICompressor extends Serializable {
return GZIP;
}
}
+ class LZMA2Compress {
+ private static LZMA2Options options;
+
+ public LZMA2Compress() {
+ options = new LZMA2Options();
+ }
+
+ public static byte[] compress(byte[] data) throws IOException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ XZOutputStream lzma2 = new XZOutputStream(out, options);
+ lzma2.write(data);
+ lzma2.close();
+ byte[] r = out.toByteArray();
+ return r;
+ }
+
+ public static byte[] uncompress(byte[] data) throws IOException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ByteArrayInputStream in = new ByteArrayInputStream(data);
+
+ XZInputStream unlzma2 = new XZInputStream(in);
+
+ byte[] buffer = new byte[256];
+ int n;
+ while ((n = unlzma2.read(buffer)) > 0) {
+ out.write(buffer, 0, n);
+ }
+ in.close();
+ byte[] r = out.toByteArray();
+ return r;
+ }
+ }
+
+ class LZMA2Compressor implements ICompressor {
+
+ private static LZMA2Compress Compress;
+ public LZMA2Compressor() {
+ Compress = new LZMA2Compress();
+ }
+
+ @Override
+ public byte[] compress(byte[] data) throws IOException {
+ if (null == data) {
+ return new byte[0];
+ }
+ byte[] r = Compress.compress(data);
+ return r;
+ }
+
+ @Override
+ public byte[] compress(byte[] data, int offset, int length) throws IOException {
+ byte[] dataBefore = new byte[length];
+ System.arraycopy(data, offset, dataBefore, 0, length);
+ byte[] r = Compress.compress(dataBefore);
+ return r;
+ }
+
+ @Override
+ public int compress(byte[] data, int offset, int length, byte[] compressed) throws IOException {
+ byte[] dataBefore = new byte[length];
+ System.arraycopy(data, offset, dataBefore, 0, length);
+ byte[] res = Compress.compress(dataBefore);
+ System.arraycopy(res, 0, compressed, 0, res.length);
+ return res.length;
+ }
+
+ @Override
+ public int compress(ByteBuffer data, ByteBuffer compressed) throws IOException {
+ int length = data.remaining();
+ byte[] dataBefore = new byte[length];
+ data.get(dataBefore, 0, length);
+ byte[] res = LZMA2Compress.compress(dataBefore);
+ compressed.put(res);
+ return res.length;
+ }
+
+ @Override
+ public int getMaxBytesForCompression(int uncompressedDataSize) {
+ // hard to estimate
+ return 40 + uncompressedDataSize;
+ //return Math.max(40 + uncompressedDataSize / 2, uncompressedDataSize);
+ }
+
+ @Override
+ public CompressionType getType() {
+ return LZMA2;
+ }
+ }
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/compress/IUnCompressor.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/compress/IUnCompressor.java
index b4b333e949..6c31b658ac 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/compress/IUnCompressor.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/compress/IUnCompressor.java
@@ -312,4 +312,50 @@ public interface IUnCompressor {
return CompressionType.GZIP;
}
}
+ class LZMA2UnCompressor implements IUnCompressor {
+
+ @Override
+ public int getUncompressedLength(byte[] array, int offset, int length) {
+ throw new UnsupportedOperationException("unsupported get uncompress length");
+ }
+
+ @Override
+ public int getUncompressedLength(ByteBuffer buffer) {
+ throw new UnsupportedOperationException("unsupported get uncompress length");
+ }
+
+ @Override
+ public byte[] uncompress(byte[] byteArray) throws IOException {
+ if (null == byteArray) {
+ return new byte[0];
+ }
+ return ICompressor.LZMA2Compress.uncompress(byteArray);
+ }
+
+ @Override
+ public int uncompress(byte[] byteArray, int offset, int length, byte[] output, int outOffset)
+ throws IOException {
+ byte[] dataBefore = new byte[length];
+ System.arraycopy(byteArray, offset, dataBefore, 0, length);
+ byte[] res = ICompressor.LZMA2Compress.uncompress(dataBefore);
+ System.arraycopy(res, 0, output, outOffset, res.length);
+ return res.length;
+ }
+
+ @Override
+ public int uncompress(ByteBuffer compressed, ByteBuffer uncompressed) throws IOException {
+ int length = compressed.remaining();
+ byte[] dataBefore = new byte[length];
+ compressed.get(dataBefore, 0, length);
+
+ byte[] res = ICompressor.LZMA2Compress.uncompress(dataBefore);
+ uncompressed.put(res);
+ return res.length;
+ }
+
+ @Override
+ public CompressionType getCodecName() {
+ return CompressionType.LZMA2;
+ }
+ }
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/BitConstructor.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/BitConstructor.java
new file mode 100644
index 0000000000..7c66bef620
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/BitConstructor.java
@@ -0,0 +1,94 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.apache.iotdb.tsfile.encoding;
+
+//import org.eclipse.collections.impl.list.mutable.primitive.ByteArrayList;
+
+import com.carrotsearch.hppc.ByteArrayList;
+
+public class BitConstructor {
+
+// public static long time = 0;
+ private static final int BITS_IN_A_BYTE = 8;
+ private static final long ALL_MASK = -1;
+ private final ByteArrayList data;
+ private byte cache = 0;
+ private int cnt = 0;
+
+ public BitConstructor() {
+ this.data = new ByteArrayList();
+ }
+
+ public BitConstructor(int initialCapacity) {
+ this.data = new ByteArrayList(initialCapacity);
+ }
+
+ public void add(long x, int len) {
+// long before = System.nanoTime();
+ x = x & ~(ALL_MASK << len);//保证x除最低的len位之外都是0
+ while (len > 0) {
+ int m = len + cnt >= BITS_IN_A_BYTE ? BITS_IN_A_BYTE - cnt : len;//向cache中插入的bit数
+ len -= m;
+ cnt += m;
+ byte y = (byte) (x >> len);
+ y = (byte) (y << (BITS_IN_A_BYTE - cnt));
+ cache = (byte) (cache | y);
+ x = x & ~(ALL_MASK << len);
+ if (cnt == BITS_IN_A_BYTE) {
+ pad();
+ }
+ }
+// long after = System.nanoTime();
+// time += (after - before);
+ }
+
+ public byte[] toByteArray() {
+ byte[] ret;
+ if (cnt > 0) {
+ data.add(cache);
+ ret = data.toArray();
+ data.remove(data.size() - 1);
+ } else {
+ ret = data.toArray();
+ }
+ return ret;
+ }
+
+ public void clear() {
+ data.clear();
+ cache = 0x00;
+ cnt = 0;
+ }
+
+ /**
+ * 如果当前字节存在剩余位，则全部填充0
+ */
+ public void pad() {
+ if (cnt > 0) {
+ data.add(cache);
+ cache = 0x00;
+ cnt = 0;
+ }
+ }
+
+ public void add(byte[] bytes) {
+ if (cnt == 0) {
+ data.add(bytes);
+ } else {
+ for (int i = 0; i < bytes.length; i++) {
+ add(bytes[i], 8);
+ }
+ }
+ }
+
+ public int length() {
+ return data.size();
+ }
+
+ public int lengthInBits() {
+ return data.size() * 8 + cnt;
+ }
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/BitReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/BitReader.java
new file mode 100644
index 0000000000..47568ed04e
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/BitReader.java
@@ -0,0 +1,67 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.apache.iotdb.tsfile.encoding;
+
+import java.util.Arrays;
+
+/**
+ *
+ * @author Wang Haoyu
+ */
+public class BitReader {
+
+ private static final int BITS_IN_A_BYTE = 8;
+ private static final byte MASKS[] = {(byte) 0xff, 0x7f, 0x3f, 0x1f, 0x0f, 0x07, 0x03, 0x01};
+ private final byte[] data;
+ private int byteCnt = 0, bitCnt = 0;
+
+ public BitReader(byte[] data) {
+ this.data = data;
+ }
+
+ public long next(int len) {
+ long ret = 0;
+ while (len > 0) {
+ int m = len + bitCnt >= BITS_IN_A_BYTE ? BITS_IN_A_BYTE - bitCnt : len;//从当前byte中读取的bit数
+ len -= m;
+ ret = ret << m;
+ byte y = (byte) (data[byteCnt] & MASKS[bitCnt]);//运算时byte自动转化为int，需要&截取低位
+ y = (byte) ((y & 0xff) >>> (BITS_IN_A_BYTE - bitCnt - m));//逻辑右移，高位补0
+ ret = ret | (y & 0xff);
+ bitCnt += m;
+ if (bitCnt == BITS_IN_A_BYTE) {
+ skip();
+ }
+ }
+ return ret;
+ }
+
+ public byte[] nextBytes(int len) {
+ byte[] ret;
+ if (bitCnt == 0) {
+ ret = Arrays.copyOfRange(data, byteCnt, byteCnt + len);
+ byteCnt += len;
+ } else {
+ ret = new byte[len];
+ for (int i = 0; i < len; i++) {
+ ret[i] = (byte) next(8);
+ }
+ }
+ return ret;
+ }
+
+ public void skip() {
+ if (bitCnt > 0) {
+ bitCnt = 0;
+ byteCnt++;
+ }
+ }
+
+ public boolean hasNext() {
+ return byteCnt < data.length;
+ }
+
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/DST/DST.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/DST/DST.java
new file mode 100644
index 0000000000..6a6222ce3e
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/DST/DST.java
@@ -0,0 +1,237 @@
+package org.apache.iotdb.tsfile.encoding.DST;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+
+public class DST {
+
+ private List<Integer> factor(int k, boolean[] isprime) {
+ List<Integer> ret = new ArrayList<>();
+ if(k == 1) {
+ ret.add(1);
+ return ret;
+ }
+ int p = 2;
+ while (k > 1) {
+ while(!isprime[p] && p <= k) {
+ p++;
+ }
+ while(k % p == 0) {
+ ret.add(p);
+ k /= p;
+ }
+ p++;
+ }
+ return ret;
+ }
+
+ private int gcd(int p, int q)
+ {
+ if(q==0)
+ return p;
+ int r = p%q;
+ return gcd(q, r);
+ }
+
+ public int[][] diffSeq(int N) {
+
+ boolean isprime[] = new boolean[N+1];
+ for(int i = 1; i <=N; i++) {
+ isprime[i] = true;
+ }
+ for(int i=2;i*i<=N;i++)
+ {
+ if(isprime[i])
+ {
+ int tmp = i+i;
+ while(tmp<=N)
+ {
+ isprime[tmp] = false;
+ tmp+=i;
+ }
+ }
+ }
+ isprime[1] = false;
+ //prime selector------------------
+
+ int W2[][] = new int[N+1][N+1];
+ for (int i = 1; i <= N; i++) {
+ for (int j = 1; j <= N; j++) {
+ W2[i][j] = 0;
+ if (j == 1) {
+ W2[i][j] = 1;
+ }
+ }
+ }
+ int a[] = {0, 1, -1};
+
+ for (int k = 1; k <= N; k++) {
+ if (k <= 2) {
+ W2[k][2] = a[k];
+ } else {
+ if (k % 2 == 0) {
+ W2[k][2] = a[2];
+ } else {
+ W2[k][2] = a[k%2];
+ }
+ }
+ }
+
+ for (int i = 3; i <= N; i++) {
+ if(isprime[i]) {
+ int b[] = new int[i - 1];
+ int c[] = new int[i + 1];
+ c[i - 1] = 1;
+ c[i] = -1;
+ for (int k = 1; k <= N; k++) {
+ if (k <= i)
+ W2[k][i] = c[k];
+ else {
+ if ((k) % (i) == 0)
+ W2[k][i] = c[i];
+ else
+ W2[k][i] = c[(k) % (i)];
+ }
+ }
+ } else {
+ List<Integer> d = factor(i, isprime);
+ d.add(0, 0);
+ HashSet<Integer> unidt = new HashSet<>(d);
+// for(int dd : d) {
+// System.out.print(dd);
+// System.out.print(" ");
+// }
+// System.out.println("");
+// for(int dd : unidt) {
+// System.out.print(dd);
+// System.out.print(" ");
+// }
+// System.out.println("");
+ List<Integer> unid = new ArrayList<>(unidt);
+// for(int dd : unid) {
+// System.out.print(dd);
+// System.out.print(" ");
+// }
+// System.out.println("");
+ int lenu = unid.size() - 1;
+ if (lenu == 1) {
+ int e[] = new int[i+1];
+
+ for (int k = 1; k <= d.get(1); k++) {
+ e[k+((k-1)*(i/d.get(1)-1))] = W2[k][d.get(1)];
+ }
+// for(int t = 1; t <= i; t++) {
+// System.out.print(e[t]);
+// System.out.print(" ");
+// }
+// System.out.println("");
+
+ for (int k = 1; k <= N; k++) {
+ if(k <= i)
+ W2[k][i] = e[k];
+ else {
+ if (k%i == 0)
+ W2[k][i] = e[i];
+ else
+ W2[k][i] = e[k%i];
+ }
+ }
+ }
+ else {
+ int muld[] = new int[lenu+1];
+ for (int k = 1; k <= lenu; k++) {
+ int tempk = unid.get(k);
+ int lend = 0;
+ for (int t = 1; t < d.size(); t++)
+ if (d.get(t) == tempk)
+ lend++;
+ muld[k] = (int)Math.pow(tempk, lend);
+ }
+ int f[] = new int[i+1];
+ for (int k = 1; k <= i; k++)
+ f[k] = 1;
+ for (int k = 1; k <= lenu; k++) {
+ for (int t = 1; t <= i; t++) {
+ f[t] *= W2[t][muld[k]];
+ }
+ }
+ for (int k = 1; k <= N; k++) {
+ if(k <= i)
+ W2[k][i] = f[k];
+ else {
+ if (k%i == 0)
+ W2[k][i] = f[i];
+ else
+ W2[k][i] = f[k%i];
+ }
+ }
+ }
+ }
+ }
+
+ return W2;
+ }
+
+ public int[][] DSTmatrix(int N) {
+ int z[][] = diffSeq(N);
+ int F[][] = new int[N+1][N+1];
+ int k = 1;
+ for (int i = 1; i <= N; i++) {
+ if(N%i == 0) {
+ int X[] = new int[N + 1];
+ for (int j = 1; j <= N; j++) {
+ F[j][k] = z[j][i];
+ X[j] = F[j][k];
+ }
+ int rp = 0;
+ for (int j = 1; j <= i; j++) {
+ if (gcd(j, i) == 1)
+ rp++;
+ }
+ for (int j = 1; j <= rp - 1; j++) {
+ F[1][k + j] = X[N];
+ for (int t = 2; t <= N; t++) {
+ F[t][k + j] = X[t - 1];
+ }
+ for (int t = 1; t <= N; t++) {
+ X[t] = F[t][k + j];
+ }
+ }
+ k += rp;
+ }
+ }
+ int FT[][] = new int[N+1][N+1];
+ for (int i = 1; i <= N; i++) {
+ for (int j = 1; j <= N; j++) {
+ FT[i][j] = F[j][i];
+ }
+ }
+ return FT;
+ }
+
+ public List<Integer> fit(List<Integer> org, int N) {
+ int MT[][] = DSTmatrix(N);
+ List<Integer> ret = new ArrayList<>();
+ int s = org.size() / N + 1;
+ for (int k = 0; k < s - 1; k++) {
+ for (int i = 0; i < N; i++) {
+ int tmp = 0;
+ for (int j = 0; j < N; j++) {
+ tmp += MT[i+1][j+1]*org.get(k*N+j);
+ }
+ ret.add(tmp);
+ }
+ }
+ for (int i = 0; i < org.size()-s*N; i++) {
+ int tmp = 0;
+ for (int j = 0; j < org.size()-s*N; j++) {
+ tmp += MT[i+1][j+1]*org.get(s*N+j);
+ }
+ ret.add(tmp);
+ }
+ return ret;
+ }
+
+}
+
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/HuffmanTree/HuffmanTree.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/HuffmanTree/HuffmanTree.java
new file mode 100644
index 0000000000..fd3193b7c1
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/HuffmanTree/HuffmanTree.java
@@ -0,0 +1,21 @@
+package org.apache.iotdb.tsfile.encoding.HuffmanTree;
+
+public class HuffmanTree {
+ public boolean isLeaf = false;
+ public int frequency;
+ public int originalbyte;
+ public boolean isRecordEnd = false;
+ public HuffmanTree leftNode;
+ public HuffmanTree rightNode;
+
+ public void HuffmanTree() {}
+
+ public void clear() {
+ isLeaf = false;
+ isRecordEnd = false;
+ frequency = 0;
+ originalbyte = 0;
+ leftNode = null;
+ rightNode = null;
+ }
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/BucketDecoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/BucketDecoder.java
new file mode 100644
index 0000000000..40efdfaf3b
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/BucketDecoder.java
@@ -0,0 +1,109 @@
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import org.apache.iotdb.tsfile.encoding.HuffmanTree.HuffmanCode;
+import org.apache.iotdb.tsfile.encoding.HuffmanTree.HuffmanTree;
+import org.apache.iotdb.tsfile.encoding.encoder.IntRleEncoder;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.PriorityQueue;
+
+public class BucketDecoder extends Decoder {
+ private int recordNum = -1;
+ private int minNum;
+ private List<Integer> level;
+ private List<Integer> rec;
+ private int curPos;
+ private int segmentLength;
+ public BucketDecoder() {
+ super(TSEncoding.BUCKET);
+ recordNum = -1;
+ level = new ArrayList<>();
+ rec = new ArrayList<>();
+ curPos = 0;
+ }
+
+ @Override
+ public void reset() {
+ recordNum = -1;
+ level.clear();
+ rec.clear();
+ curPos = 0;
+ }
+
+ public int readInt(ByteBuffer buffer) {
+ if(recordNum == -1) {
+ reset();
+ minNum = buffer.getInt();
+ segmentLength = buffer.getInt();
+ HuffmanDecoderV2 decoder = new HuffmanDecoderV2();
+ int tmp = decoder.readInt(buffer);
+ level.add(tmp);
+ recordNum = decoder.recordnum;
+ for(int i = 0; i < recordNum-1; i++) {
+ tmp = decoder.readInt(buffer);
+ level.add(tmp);
+ }
+ int bitwidth = 32-Integer.numberOfLeadingZeros(segmentLength) - 1;
+ for(int i = 0; i < recordNum; i++) {
+ tmp = 0;
+ for(int j = bitwidth-1; j >=0; j--) {
+ int b = readbit(buffer);
+ tmp |= (b<<j);
+ }
+ int lv = level.get(i);
+ rec.add(tmp + minNum + lv * segmentLength);
+ }
+ }
+ int ret = rec.get(curPos++);
+ if(curPos == recordNum) {
+ reset();
+ clearBuffer(buffer);
+ }
+ return ret;
+ }
+
+ @Override
+ public boolean hasNext(ByteBuffer buffer) {
+ if(recordNum == -1) {
+ if (buffer.hasRemaining())
+ return true;
+ else
+ return false;
+ }
+ else if(curPos < recordNum)
+ return true;
+ else
+ return false;
+ }
+
+ private int numberLeftInBuffer = 0;
+ private int byteBuffer = 0;
+
+
+ private int readbit(ByteBuffer buffer) {
+ if (numberLeftInBuffer == 0) {
+ loadBuffer(buffer);
+ numberLeftInBuffer = 8;
+ }
+ int top = ((byteBuffer >> 7) & 1);
+ byteBuffer <<= 1;
+ numberLeftInBuffer--;
+ return top;
+ }
+
+ private void loadBuffer(ByteBuffer buffer) {
+ byteBuffer = buffer.get();
+ }
+
+ private void clearBuffer(ByteBuffer buffer) {
+ while (numberLeftInBuffer > 0) {
+ readbit(buffer);
+ }
+ }
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/HuffmanDecoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/HuffmanDecoder.java
new file mode 100644
index 0000000000..4f5b8d7935
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/HuffmanDecoder.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import org.apache.iotdb.tsfile.encoding.HuffmanTree.HuffmanTree;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.utils.Binary;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Queue;
+
+public class HuffmanDecoder extends Decoder {
+
+ private int recordnum;
+ private int numberLeftInBuffer;
+ private byte byteBuffer;
+ private Queue<Binary> records;
+ private HuffmanTree tree;
+
+ HuffmanDecoder() {
+ super(TSEncoding.HUFFMAN);
+ records = new LinkedList<>();
+ tree = new HuffmanTree();
+ reset();
+ }
+
+ @Override
+ public Binary readBinary(ByteBuffer buffer) {
+ if (records.isEmpty()) {
+ reset();
+ loadTree(buffer);
+ loadRecords(buffer);
+ clearBuffer(buffer);
+ }
+ return records.poll();
+ }
+
+ public boolean hasNext(ByteBuffer buffer) {
+ return ((!records.isEmpty()) || buffer.hasRemaining());
+ }
+
+ private void loadTree(ByteBuffer buffer) {
+ recordnum = getInt(buffer);
+ int endOfRecordLength = getInt(buffer);
+ HuffmanTree header = tree;
+ for (int i = 0; i < endOfRecordLength; i++) {
+ if (readbit(buffer) == 0) {
+ if (header.leftNode == null) header.leftNode = new HuffmanTree();
+ header = header.leftNode;
+ } else {
+ if (header.rightNode == null) header.rightNode = new HuffmanTree();
+ header = header.rightNode;
+ }
+ if (i == endOfRecordLength - 1) {
+ header.isLeaf = true;
+ header.isRecordEnd = true;
+ }
+ }
+
+ int usednum = getInt(buffer);
+ for (int i = 0; i < usednum; i++) {
+ byte cha = getByte(buffer);
+ int codeLength = getInt(buffer);
+ String s = new String();
+ HuffmanTree tempTree = tree;
+ for (int j = 0; j < codeLength; j++) {
+ int b = readbit(buffer);
+ if (b == 0) {
+ if (tempTree.leftNode == null) tempTree.leftNode = new HuffmanTree();
+ tempTree = tempTree.leftNode;
+ } else {
+ if (tempTree.rightNode == null) tempTree.rightNode = new HuffmanTree();
+ tempTree = tempTree.rightNode;
+ }
+ if (j == codeLength - 1) {
+ tempTree.isLeaf = true;
+ tempTree.originalbyte = cha;
+ }
+ }
+ }
+ }
+
+ private void loadRecords(ByteBuffer buffer) {
+ for (int i = 0; i < recordnum; i++) {
+ HuffmanTree tempTree = tree;
+ List<Byte> rec = new ArrayList<>();
+ while (true) {
+ tempTree = tree;
+ while (!tempTree.isLeaf) {
+ if (readbit(buffer) == 0) tempTree = tempTree.leftNode;
+ else tempTree = tempTree.rightNode;
+ }
+ if (tempTree.isRecordEnd) break;
+ rec.add((byte) tempTree.originalbyte);
+ }
+ byte[] currec = new byte[rec.size()];
+ for (int j = 0; j < rec.size(); j++) currec[j] = rec.get(j);
+ records.add(new Binary(currec));
+ }
+ }
+
+ public void reset() {
+ recordnum = 0;
+ records.clear();
+ tree.clear();
+ }
+
+ private int getInt(ByteBuffer buffer) {
+ int val = 0;
+ for (int i = 31; i >= 0; i--) {
+ val |= (readbit(buffer) << i);
+ }
+ return val;
+ }
+
+ private byte getByte(ByteBuffer buffer) {
+ byte val = 0;
+ for (int i = 7; i >= 0; i--) {
+ val |= (readbit(buffer) << i);
+ }
+ return val;
+ }
+
+ private int readbit(ByteBuffer buffer) {
+ if (numberLeftInBuffer == 0) {
+ loadBuffer(buffer);
+ numberLeftInBuffer = 8;
+ }
+ int top = ((byteBuffer >> 7) & 1);
+ byteBuffer <<= 1;
+ numberLeftInBuffer--;
+ return top;
+ }
+
+ private void loadBuffer(ByteBuffer buffer) {
+ byteBuffer = buffer.get();
+ }
+
+ private void clearBuffer(ByteBuffer buffer) {
+ while (numberLeftInBuffer > 0) {
+ readbit(buffer);
+ }
+ }
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/HuffmanDecoderV2.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/HuffmanDecoderV2.java
new file mode 100644
index 0000000000..2da75ed03f
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/HuffmanDecoderV2.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import org.apache.iotdb.tsfile.encoding.HuffmanTree.HuffmanTree;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.utils.Binary;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Queue;
+
+public class HuffmanDecoderV2 extends Decoder {
+
+ public int recordnum;
+ private int numberLeftInBuffer;
+ private byte byteBuffer;
+ private Queue<Integer> records;
+ private HuffmanTree tree;
+
+ HuffmanDecoderV2() {
+ super(TSEncoding.HUFFMAN);
+ records = new LinkedList<>();
+ tree = new HuffmanTree();
+ reset();
+ }
+
+ @Override
+ public int readInt(ByteBuffer buffer) {
+ if (records.isEmpty()) {
+ reset();
+ loadTree(buffer);
+ loadRecords(buffer);
+ clearBuffer(buffer);
+ }
+ return records.poll();
+ }
+
+ public boolean hasNext(ByteBuffer buffer) {
+ return ((!records.isEmpty()) || buffer.hasRemaining());
+ }
+
+ private void loadTree(ByteBuffer buffer) {
+ recordnum = getInt(buffer);
+ int usednum = getInt(buffer);
+ for (int i = 0; i < usednum; i++) {
+ int cha = getInt(buffer);
+ int codeLength = getInt(buffer);
+ HuffmanTree tempTree = tree;
+ for (int j = 0; j < codeLength; j++) {
+ int b = readbit(buffer);
+ if (b == 0) {
+ if (tempTree.leftNode == null) tempTree.leftNode = new HuffmanTree();
+ tempTree = tempTree.leftNode;
+ } else {
+ if (tempTree.rightNode == null) tempTree.rightNode = new HuffmanTree();
+ tempTree = tempTree.rightNode;
+ }
+ }
+ tempTree.isLeaf = true;
+ tempTree.originalbyte = cha;
+ }
+ }
+
+ private void loadRecords(ByteBuffer buffer) {
+ HuffmanTree tempTree = tree;
+ for(int i = 0; i < recordnum; i++) {
+ tempTree = tree;
+ while (!tempTree.isLeaf) {
+ if (readbit(buffer) == 0) tempTree = tempTree.leftNode;
+ else tempTree = tempTree.rightNode;
+ }
+ records.add(tempTree.originalbyte);
+ }
+ }
+
+ public void reset() {
+ recordnum = 0;
+ records.clear();
+ tree.clear();
+ }
+
+ private int getInt(ByteBuffer buffer) {
+ int val = 0;
+ for (int i = 31; i >= 0; i--) {
+ val |= (readbit(buffer) << i);
+ }
+ return val;
+ }
+
+ private byte getByte(ByteBuffer buffer) {
+ byte val = 0;
+ for (int i = 7; i >= 0; i--) {
+ val |= (readbit(buffer) << i);
+ }
+ return val;
+ }
+
+ private int readbit(ByteBuffer buffer) {
+ if (numberLeftInBuffer == 0) {
+ loadBuffer(buffer);
+ numberLeftInBuffer = 8;
+ }
+ int top = ((byteBuffer >> 7) & 1);
+ byteBuffer <<= 1;
+ numberLeftInBuffer--;
+ return top;
+ }
+
+ private void loadBuffer(ByteBuffer buffer) {
+ byteBuffer = buffer.get();
+ }
+
+ private void clearBuffer(ByteBuffer buffer) {
+ while (numberLeftInBuffer > 0) {
+ readbit(buffer);
+ }
+ }
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/BucketEncoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/BucketEncoder.java
new file mode 100644
index 0000000000..1c99307cec
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/BucketEncoder.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.encoder;
+
+import org.apache.iotdb.tsfile.encoding.DST.DST;
+import org.apache.iotdb.tsfile.encoding.HuffmanTree.HuffmanCode;
+import org.apache.iotdb.tsfile.encoding.HuffmanTree.HuffmanTree;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.read.filter.operator.In;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.PriorityQueue;
+
+public class BucketEncoder extends Encoder {
+
+ private byte byteBuffer;
+ private int numberLeftInBuffer = 0;
+ private int totLength;
+
+ private int minNum = 0x7FFFFFFF;
+ private int maxNum = -0x7FFFFFFF;
+ private int segmentLength =64;//2048; //65536 / 128;
+ private List<Integer> valueBuffer;
+
+ public BucketEncoder() {
+ super(TSEncoding.BUCKET);
+ valueBuffer = new ArrayList<>();
+ }
+
+ public BucketEncoder(int _segmentLength) {
+ super(TSEncoding.HUFFMANV2);
+ segmentLength = _segmentLength;
+ valueBuffer = new ArrayList<>();
+ }
+
+ private void reset() {
+ minNum = 0x7FFFFFFF;
+ maxNum = -0x7FFFFFFF;
+ valueBuffer.clear();
+ }
+
+ @Override
+ public void encode(int value, ByteArrayOutputStream out) {
+ valueBuffer.add(value);
+ minNum = Math.min(minNum, value);
+ maxNum = Math.max(maxNum, value);
+ }
+
+ @Override
+ public void encode(float value, ByteArrayOutputStream out) {
+ int _value = convertFloatToInt(value);
+ encode(_value, out);
+ }
+
+ private int convertFloatToInt(float value) {
+ return (int) Math.round(value * 100000);
+ }
+
+ @Override
+ public void flush(ByteArrayOutputStream out) {
+ writeInt(minNum, out);
+ writeInt(segmentLength, out);
+ int segmentNum = (maxNum - minNum) / segmentLength + 1;
+ HuffmanEncoderV2 huffmanEncoder = new HuffmanEncoderV2(segmentNum);
+ for (int e : valueBuffer) {
+ huffmanEncoder.encode((e - minNum) / segmentLength, out);
+ }
+ huffmanEncoder.flush(out);
+// System.out.println(" huffman length: ");
+// System.out.print(" ");
+// System.out.println(out.size());
+ int temp = out.size();
+
+ int bitwidth = 32 - Integer.numberOfLeadingZeros(segmentLength) - 1;
+ for (int e : valueBuffer) {
+ int encodedValue = (e - minNum) % segmentLength;
+ for (int i = bitwidth - 1; i >= 0; i--) {
+ writeBit((encodedValue & (1 << i)) > 0, out);
+ }
+ }
+ clearBuffer(out);
+
+// System.out.println(" total length: ");
+// System.out.print(" ");
+// System.out.println(out.size() - temp);
+ reset();
+ }
+
+
+ protected void writeBit(boolean b, ByteArrayOutputStream out) {
+ byteBuffer <<= 1;
+ if (b) {
+ byteBuffer |= 1;
+ }
+
+ numberLeftInBuffer++;
+ if (numberLeftInBuffer == 8) {
+ clearBuffer(out);
+ }
+ }
+
+ protected void clearBuffer(ByteArrayOutputStream out) {
+ if (numberLeftInBuffer == 0) return;
+ if (numberLeftInBuffer > 0) byteBuffer <<= (8 - numberLeftInBuffer);
+ out.write(byteBuffer);
+ totLength++;
+ numberLeftInBuffer = 0;
+ byteBuffer = 0;
+ }
+
+ private void writeInt(int val, ByteArrayOutputStream out) {
+ for (int i = 31; i >= 0; i--) {
+ if ((val & (1 << i)) > 0) writeBit(true, out);
+ else writeBit(false, out);
+ }
+ }
+
+ private void writeByte(byte val, ByteArrayOutputStream out) {
+ for (int i = 7; i >= 0; i--) {
+ if ((val & (1 << i)) > 0) writeBit(true, out);
+ else writeBit(false, out);
+ }
+ }
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DeltaBinaryEncoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DeltaBinaryEncoder.java
index b95846085e..767d74dd76 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DeltaBinaryEncoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DeltaBinaryEncoder.java
@@ -45,7 +45,7 @@ import java.io.IOException;
*/
public abstract class DeltaBinaryEncoder extends Encoder {
- protected static final int BLOCK_DEFAULT_SIZE = 128;
+ protected static final int BLOCK_DEFAULT_SIZE = 256;
private static final Logger logger = LoggerFactory.getLogger(DeltaBinaryEncoder.class);
protected ByteArrayOutputStream out;
protected int blockSize;
@@ -211,6 +211,7 @@ public abstract class DeltaBinaryEncoder extends Encoder {
@Override
public void encode(int value, ByteArrayOutputStream out) {
+// System.out.println(value);
encodeValue(value, out);
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/Encoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/Encoder.java
index b201fb97da..56350cbdcb 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/Encoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/Encoder.java
@@ -32,6 +32,7 @@ import java.math.BigDecimal;
*/
public abstract class Encoder {
+<<<<<<< Updated upstream
public static final String MAX_STRING_LENGTH = "max_string_length";
public static final String MAX_POINT_NUMBER = "max_point_number";
@@ -108,4 +109,88 @@ public abstract class Encoder {
public long getMaxByteSize() {
throw new UnsupportedOperationException();
}
+=======
+ public static final String MAX_STRING_LENGTH = "max_string_length";
+ public static final String MAX_POINT_NUMBER = "max_point_number";
+
+ private TSEncoding type;
+
+ public void setType(TSEncoding type) {
+ this.type = type;
+ }
+
+ public TSEncoding getType() {
+ return type;
+ }
+
+ public Encoder(TSEncoding type) {
+ this.type = type;
+ }
+
+ public void encode(boolean value, ByteArrayOutputStream out) {
+ throw new TsFileEncodingException("Method encode boolean is not supported by Encoder");
+ }
+
+ public void encode(short value, ByteArrayOutputStream out) {
+ throw new TsFileEncodingException("Method encode short is not supported by Encoder");
+ }
+
+ public void encode(int value, ByteArrayOutputStream out) {
+ throw new TsFileEncodingException("Method encode int is not supported by Encoder");
+ }
+
+ public void encode(long value, ByteArrayOutputStream out) {
+ throw new TsFileEncodingException("Method encode long is not supported by Encoder");
+ }
+
+ public void encode(String value, ByteArrayOutputStream out) {
+ throw new TsFileEncodingException("Method encode text is not supported by Encoder");
+ }
+
+ ;
+
+ public void encode(float value, ByteArrayOutputStream out) {
+ throw new TsFileEncodingException("Method encode float is not supported by Encoder");
+ }
+
+ public void encode(double value, ByteArrayOutputStream out) {
+ throw new TsFileEncodingException("Method encode double is not supported by Encoder");
+ }
+
+ public void encode(Binary value, ByteArrayOutputStream out) {
+ throw new TsFileEncodingException("Method encode Binary is not supported by Encoder");
+ }
+
+ public void encode(BigDecimal value, ByteArrayOutputStream out) {
+ throw new TsFileEncodingException("Method encode BigDecimal is not supported by Encoder");
+ }
+
+ /**
+ * Write all values buffered in memory cache to OutputStream.
+ *
+ * @param out - ByteArrayOutputStream
+ * @throws IOException cannot flush to OutputStream
+ */
+ public abstract void flush(ByteArrayOutputStream out) throws IOException;
+
+ /**
+ * When encoder accepts a new incoming data point, the maximal possible size in byte it takes to
+ * store in memory.
+ *
+ * @return the maximal possible size of one data item encoded by this encoder
+ */
+ public int getOneItemMaxSize() {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * The maximal possible memory size occupied by current Encoder. This statistic value doesn't
+ * involve OutputStream.
+ *
+ * @return the maximal size of possible memory occupied by current encoder
+ */
+ public long getMaxByteSize() {
+ throw new UnsupportedOperationException();
+ }
+>>>>>>> Stashed changes
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/FloatEncoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/FloatEncoder.java
index 2d477cb936..4f5546601c 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/FloatEncoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/FloatEncoder.java
@@ -51,6 +51,7 @@ public class FloatEncoder extends Encoder {
public FloatEncoder(TSEncoding encodingType, TSDataType dataType, int maxPointNumber) {
super(encodingType);
+// System.out.println(maxPointNumber);
this.maxPointNumber = maxPointNumber;
calculateMaxPonitNum();
isMaxPointNumberSaved = false;
@@ -81,7 +82,9 @@ public class FloatEncoder extends Encoder {
@Override
public void encode(float value, ByteArrayOutputStream out) {
saveMaxPointNumber(out);
+// System.out.println(value);
int valueInt = convertFloatToInt(value);
+// System.out.println(valueInt);
encoder.encode(valueInt, out);
}
@@ -98,10 +101,12 @@ public class FloatEncoder extends Encoder {
maxPointValue = 1;
} else {
maxPointValue = Math.pow(10, maxPointNumber);
+ System.out.println(maxPointNumber);
}
}
private int convertFloatToInt(float value) {
+// Math.round(value * maxPointValue);
return (int) Math.round(value * maxPointValue);
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/HuffmanEncoderV2.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/HuffmanEncoderV2.java
new file mode 100644
index 0000000000..ad75f6359a
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/HuffmanEncoderV2.java
@@ -0,0 +1,257 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.encoder;
+
+import org.apache.iotdb.tsfile.encoding.HuffmanTree.HuffmanCode;
+import org.apache.iotdb.tsfile.encoding.HuffmanTree.HuffmanTree;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.utils.Binary;
+
+import java.io.ByteArrayOutputStream;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.PriorityQueue;
+
+public class HuffmanEncoderV2 extends Encoder {
+
+ private HuffmanTree[] byteFrequency;
+ private List<Integer> records;
+ private HuffmanCode[] huffmanCodes;
+ PriorityQueue<HuffmanTree> huffmanQueue;
+ private HuffmanTree treeTop;
+ private byte byteBuffer;
+ private int numberLeftInBuffer = 0;
+ private boolean[] used;
+ private int usednum;
+ private int maxRecordLength;
+ private int totLength;
+ public int recordnum;
+
+ private int symbolSetSize;
+
+ public HuffmanEncoderV2(int _symbolSetSize) {
+ super(TSEncoding.HUFFMANV2);
+ symbolSetSize = _symbolSetSize;
+ byteFrequency =
+ new HuffmanTree[_symbolSetSize]; // byteFrequency[256] is used to save the frequency of end-of-records
+ for (int i = 0; i < _symbolSetSize; i++)
+ byteFrequency[i] = new HuffmanTree();
+ records = new ArrayList<Integer>();
+ huffmanQueue = new PriorityQueue<HuffmanTree>(huffmanTreeComparator);
+ huffmanCodes = new HuffmanCode[_symbolSetSize];
+ for (int i = 0; i < _symbolSetSize; i++) {
+ huffmanCodes[i] = new HuffmanCode();
+ }
+ used = new boolean[_symbolSetSize];
+ treeTop = new HuffmanTree();
+ reset();
+ }
+
+ @Override
+ public void encode(int value, ByteArrayOutputStream out) {
+ recordnum++;
+ records.add(value);
+ byteFrequency[value].frequency++;
+ }
+
+ @Override
+ public void flush(ByteArrayOutputStream out) {
+ buildHuffmanTree();
+ double cnt = 0;
+ double cnt2 = 0;
+ int tot = 0;
+ for (int i = 0; i < symbolSetSize; i++) {
+ if (byteFrequency[i].frequency != 0)
+ tot += 1;
+ }
+ for (int i = 0; i < symbolSetSize; i++) {
+ if (byteFrequency[i].frequency != 0) {
+ double shang = - (Math.log((double) byteFrequency[i].frequency / recordnum) / Math.log(2));
+ cnt += Math.round(shang) * byteFrequency[i].frequency;
+ }
+ }
+ cnt /= 8;
+ // System.out.println(cnt);
+ List<Boolean> code = new ArrayList<>();
+ getHuffmanCode(treeTop, code);
+
+// for (int i = 0; i < symbolSetSize; i++) {
+// if (byteFrequency[i].frequency != 0) {
+// System.out.print(i);
+// System.out.print(": ");
+// for (Boolean b : huffmanCodes[i].huffmanCode)
+// System.out.print((b.toString()));
+// System.out.println("");
+// }
+// }
+
+ flushHeader(out);
+// System.out.println(" huffman heaeder:");
+// System.out.print(" ");
+// System.out.println(out.size());
+ int temp = out.size();
+ for (int i = 0; i < records.size(); i++) {
+ flushRecord(records.get(i), out);
+ }
+ reset();
+ clearBuffer(out);
+
+// System.out.println(" huffman body:");
+// System.out.print(" ");
+// System.out.println(out.size() - temp);
+ }
+
+ @Override
+ public int getOneItemMaxSize() {
+ return maxRecordLength;
+ }
+
+ @Override
+ public long getMaxByteSize() {
+ return totLength;
+ }
+
+ private void buildHuffmanTree() {
+ for (int i = 0; i < symbolSetSize; i++) {
+ if (byteFrequency[i].frequency != 0) {
+ huffmanQueue.add(byteFrequency[i]);
+ used[i] = true;
+ usednum++;
+ }
+ }
+ while (huffmanQueue.size() > 1) {
+ HuffmanTree cur = new HuffmanTree();
+ cur.leftNode = huffmanQueue.poll();
+ cur.rightNode = huffmanQueue.poll();
+ cur.frequency = cur.leftNode.frequency + cur.rightNode.frequency;
+ cur.isRecordEnd = false;
+ cur.isLeaf = false;
+ huffmanQueue.add(cur);
+ }
+ treeTop = huffmanQueue.poll();
+ }
+
+ private void getHuffmanCode(HuffmanTree cur, List<Boolean> code) {
+ if (cur.isLeaf) {
+ for (int i = 0; i < code.size(); i++) {
+ int idx = (int) cur.originalbyte;
+ if (idx < 0) idx += (1 << 8);
+ huffmanCodes[idx].huffmanCode.add(code.get(i));
+ }
+ return;
+ }
+ code.add(false);
+ getHuffmanCode(cur.leftNode, code);
+ code.remove(code.size() - 1);
+ code.add(true);
+ getHuffmanCode(cur.rightNode, code);
+ code.remove(code.size() - 1);
+ }
+
+ private void flushHeader(ByteArrayOutputStream out) {
+ writeInt(recordnum, out); // write the number of records
+ totLength += 4;
+ writeInt(usednum, out); // Write how many character have been used in this section
+ totLength += 4;
+ for (int i = 0; i < symbolSetSize; i++) {
+ if (used[i]) {
+ writeInt(i, out);
+ writeInt(
+ huffmanCodes[i].huffmanCode.size(), out); // First we store the length of huffman code
+ totLength += 8;
+ for (boolean b : huffmanCodes[i].huffmanCode) // Then we store the huffman code
+ writeBit(b, out);
+ }
+ }
+ }
+
+ private void flushRecord(Integer rec, ByteArrayOutputStream out) {
+ for (boolean b : huffmanCodes[rec].huffmanCode) {
+ writeBit(b, out);
+// if (b)
+// System.out.print(1);
+// else
+// System.out.print(0);
+ }
+// System.out.println("");
+ }
+
+ private void reset() {
+ for (int i = 0; i < symbolSetSize; i++) {
+ byteFrequency[i].frequency = 0;
+ byteFrequency[i].originalbyte = i;
+ byteFrequency[i].isLeaf = true;
+ byteFrequency[i].isRecordEnd = false;
+ huffmanCodes[i].huffmanCode.clear();
+ used[i] = false;
+ }
+ records.clear();
+ huffmanQueue.clear();
+ usednum = 0;
+ maxRecordLength = 0;
+ totLength = 0;
+ recordnum = 0;
+ treeTop.clear();
+ }
+
+ public static Comparator<HuffmanTree> huffmanTreeComparator =
+ new Comparator<HuffmanTree>() {
+ @Override
+ public int compare(HuffmanTree o1, HuffmanTree o2) {
+ return o1.frequency - o2.frequency;
+ }
+ };
+
+ protected void writeBit(boolean b, ByteArrayOutputStream out) {
+ byteBuffer <<= 1;
+ if (b) {
+ byteBuffer |= 1;
+ }
+
+ numberLeftInBuffer++;
+ if (numberLeftInBuffer == 8) {
+ clearBuffer(out);
+ }
+ }
+
+ protected void clearBuffer(ByteArrayOutputStream out) {
+ if (numberLeftInBuffer == 0) return;
+ if (numberLeftInBuffer > 0) byteBuffer <<= (8 - numberLeftInBuffer);
+ out.write(byteBuffer);
+ totLength++;
+ numberLeftInBuffer = 0;
+ byteBuffer = 0;
+ }
+
+ private void writeInt(int val, ByteArrayOutputStream out) {
+ for (int i = 31; i >= 0; i--) {
+ if ((val & (1 << i)) > 0) writeBit(true, out);
+ else writeBit(false, out);
+ }
+ }
+
+ private void writeByte(byte val, ByteArrayOutputStream out) {
+ for (int i = 7; i >= 0; i--) {
+ if ((val & (1 << i)) > 0) writeBit(true, out);
+ else writeBit(false, out);
+ }
+ }
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java
index 74f9c3c4c4..ba19062ff0 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java
@@ -70,6 +70,21 @@ public abstract class TSEncodingBuilder {
return new GorillaV2();
case DICTIONARY:
return new Dictionary();
+<<<<<<< Updated upstream
+=======
+ case RAKE:
+ return new Rake();
+ case SPRINTZ:
+ return new Sprintz();
+ case RLBE:
+ return new RLBE();
+ case TEXTRLE:
+ return new TEXTRLE();
+ case HUFFMAN:
+ return new HUFFMAN();
+ case BUCKET:
+ return new BucketEncoder();
+>>>>>>> Stashed changes
default:
throw new UnsupportedOperationException(type.toString());
}
@@ -140,7 +155,13 @@ public abstract class TSEncodingBuilder {
return new LongRleEncoder();
case FLOAT:
case DOUBLE:
+<<<<<<< Updated upstream
return new FloatEncoder(TSEncoding.RLE, type, maxPointNumber);
+=======
+ return new FloatEncoder(TSEncoding.RLE, type, 6);//maxPointNumber);
+ case TEXT:
+ return new TextRleEncoder();
+>>>>>>> Stashed changes
default:
throw new UnSupportedDataTypeException("RLE doesn't support data type: " + type);
}
@@ -182,7 +203,7 @@ public abstract class TSEncodingBuilder {
/** for INT32, INT64, FLOAT, DOUBLE. */
public static class Ts2Diff extends TSEncodingBuilder {
- private int maxPointNumber = 0;
+ private int maxPointNumber = 5;
@Override
public Encoder getEncoder(TSDataType type) {
@@ -193,6 +214,8 @@ public abstract class TSEncodingBuilder {
return new DeltaBinaryEncoder.LongDeltaEncoder();
case FLOAT:
case DOUBLE:
+// maxPointNumber
+// System.out.println(maxPointNumber);
return new FloatEncoder(TSEncoding.TS_2DIFF, type, maxPointNumber);
default:
throw new UnSupportedDataTypeException("TS_2DIFF doesn't support data type: " + type);
@@ -207,8 +230,10 @@ public abstract class TSEncodingBuilder {
public void initFromProps(Map<String, String> props) {
// set max error from initialized map or default value if not set
if (props == null || !props.containsKey(Encoder.MAX_POINT_NUMBER)) {
+ System.out.println(Encoder.MAX_POINT_NUMBER);
maxPointNumber = TSFileDescriptor.getInstance().getConfig().getFloatPrecision();
} else {
+ System.out.println(Encoder.MAX_POINT_NUMBER);
try {
this.maxPointNumber = Integer.parseInt(props.get(Encoder.MAX_POINT_NUMBER));
} catch (NumberFormatException e) {
@@ -314,4 +339,140 @@ public abstract class TSEncodingBuilder {
// do nothing
}
}
+<<<<<<< Updated upstream
+=======
+
+ public static class Rake extends TSEncodingBuilder {
+
+ @Override
+ public Encoder getEncoder(TSDataType type) {
+ switch (type) {
+ case FLOAT:
+ return new FloatRAKEEncoder();
+ case DOUBLE:
+ return new DoubleRAKEEncoder();
+ case INT32:
+ return new IntRAKEEncoder();
+ case INT64:
+ return new LongRAKEEncoder();
+ default:
+ throw new UnSupportedDataTypeException("Rake doesn't support data type: " + type);
+ }
+ }
+
+ @Override
+ public void initFromProps(Map<String, String> props) {
+ // do nothing
+ }
+ }
+
+ public static class Sprintz extends TSEncodingBuilder {
+ @Override
+ public Encoder getEncoder(TSDataType type) {
+ switch (type) {
+ case INT32:
+ return new IntSprintzEncoder();
+ case INT64:
+ return new LongSprintzEncoder();
+ case FLOAT:
+ return new FloatSprintzEncoder();
+ case DOUBLE:
+ return new DoubleSprintzEncoder();
+ default:
+ throw new UnSupportedDataTypeException("Sprintz doesn't support data type: " + type);
+ }
+ }
+
+ @Override
+ public void initFromProps(Map<String, String> props) {
+ // do nothing
+ }
+ }
+
+ public static class RLBE extends TSEncodingBuilder {
+ @Override
+ public Encoder getEncoder(TSDataType type) {
+ switch (type) {
+ case INT32:
+ return new IntRLBE();
+ case INT64:
+ return new LongRLBE();
+ case FLOAT:
+ return new FloatRLBE();
+ case DOUBLE:
+ return new DoubleRLBE();
+ default:
+ throw new UnSupportedDataTypeException("RLBE doesn't support data type: " + type);
+ }
+ }
+
+ @Override
+ public void initFromProps(Map<String, String> props) {
+ // do nothing
+ }
+ }
+
+ public static class HUFFMAN extends TSEncodingBuilder {
+ @Override
+ public Encoder getEncoder(TSDataType type) {
+ switch (type) {
+ case TEXT:
+ return new HuffmanEncoder();
+ case INT32:
+ case INT64:
+ case FLOAT:
+ case DOUBLE:
+ default:
+ throw new UnSupportedDataTypeException("HUFFMAN doesn't support data type: " + type);
+ }
+ }
+
+ @Override
+ public void initFromProps(Map<String, String> props) {
+ // do nothing
+ }
+ }
+
+ public static class TEXTRLE extends TSEncodingBuilder {
+ @Override
+ public Encoder getEncoder(TSDataType type) {
+ switch (type) {
+ case TEXT:
+ return new TextRleEncoder();
+ case INT32:
+ case INT64:
+ case FLOAT:
+ case DOUBLE:
+ default:
+ throw new UnSupportedDataTypeException("TEXTRLE doesn't support data type: " + type);
+ }
+ }
+
+ @Override
+ public void initFromProps(Map<String, String> props) {
+ // do nothing
+ }
+ }
+ public static class BucketEncoder extends TSEncodingBuilder {
+ @Override
+ public Encoder getEncoder(TSDataType type) {
+ switch (type) {
+ case INT32:
+ return new org.apache.iotdb.tsfile.encoding.encoder.BucketEncoder();
+ case FLOAT:
+ return new org.apache.iotdb.tsfile.encoding.encoder.BucketEncoder();
+ case TEXT:
+ case INT64:
+ case DOUBLE:
+ default:
+ throw new UnSupportedDataTypeException("TEXTRLE doesn't support data type: " + type);
+ }
+ }
+
+ @Override
+ public void initFromProps(Map<String, String> props) {
+ // do nothing
+ }
+ }
+>>>>>>> Stashed changes
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoderbuff/BuffEncoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoderbuff/BuffEncoder.java
new file mode 100644
index 0000000000..6a3e27778d
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoderbuff/BuffEncoder.java
@@ -0,0 +1,260 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.apache.iotdb.tsfile.encoding.encoderbuff;
+
+//import cn.edu.thu.diq.compression.BitConstructor;
+//import cn.edu.thu.diq.compression.BitReader;
+//import org.eclipse.collections.impl.list.mutable.primitive.DoubleArrayList;
+
+import jodd.util.collection.DoubleArrayList;
+import org.apache.iotdb.tsfile.encoding.BitConstructor;
+import org.apache.iotdb.tsfile.encoding.BitReader;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ *
+ * @author Wang Haoyu
+ */
+public class BuffEncoder implements Encoder {
+
+ private final double sparseThreshold = 0.9;
+ private final int blocksize = 1024;
+ private final int precision;
+ private final double eps;
+
+ public BuffEncoder(int precision) {
+ this.precision = Math.max(0, precision);
+ this.eps = Math.pow(2, -precision);
+ }
+
+ @Override
+ public byte[] encode(double[] origin) {
+ BitConstructor constructor = new BitConstructor();
+ for (int i = 0; i < origin.length; i += blocksize) {
+ int n = Math.min(origin.length, i + blocksize) - i;
+ encodeBlock(constructor, origin, i, i + n);
+ }
+ return constructor.toByteArray();
+ }
+
+ @Override
+ public double[] decode(byte[] bytes) {
+ BitReader reader = new BitReader(bytes);
+ DoubleArrayList decoded = new DoubleArrayList();
+ while (reader.hasNext()) {
+ decoded.addAll(decodeBlock(reader));
+ }
+ return decoded.toArray();
+ }
+
+ private void encodeBlock(BitConstructor constructor, double[] values, int from, int to) {
+ //获取整数部分的范围
+ int min = Integer.MAX_VALUE, max = Integer.MIN_VALUE;
+ for (int i = from; i < to; i++) {
+ min = Math.min(min, (int) Math.floor(values[i]));
+ max = Math.max(max, (int) Math.floor(values[i]));
+ }
+ //保存元数据
+ constructor.add(to - from, 32);
+// System.out.println(to - from);
+ constructor.add(this.precision, 32);
+ constructor.add(min, 32);
+ constructor.add(max, 32);
+ //浮点数转定点数
+ long[] fixed = new long[to - from];
+ for (int i = 0; i < fixed.length; i++) {
+ fixed[i] = (long) (Math.round((values[i + from] - min) / eps));
+ }
+ //按照子列分别进行存储
+ int[] masks = {0, 0x0001, 0x0003, 0x0007, 0x000f, 0x001f, 0x003f, 0x007f, 0x00ff};
+ int totalWidth = getValueWidth(max - min) + precision;
+ for (int i = totalWidth; i > 0; i -= 8) {
+ int shift = Math.max(0, i - 8), len = Math.min(8, i);
+ byte[] bytes = new byte[fixed.length];
+ for (int j = 0; j < fixed.length; j++) {
+ bytes[j] = (byte) ((fixed[j] >> shift) & masks[len]);
+ }
+ encodeSubColumn(constructor, bytes, len);
+ }
+ constructor.pad();
+ }
+
+ private double[] decodeBlock(BitReader reader) {
+ //读取元数据
+ int n = (int) reader.next(32);
+// System.out.println(n);
+ int p = (int) reader.next(32);
+ int min = (int) reader.next(32);
+ int max = (int) reader.next(32);
+ //读取子列
+ long[] fixed = new long[n];
+ int totalWidth = getValueWidth(max - min) + p;
+ for (int i = totalWidth; i > 0; i -= 8) {
+ int len = Math.min(8, i);
+ byte[] bytes = decodeSubColumn(reader, n, len);
+ for (int j = 0; j < n; j++) {
+ fixed[j] = (fixed[j] << len) | (bytes[j] & 0xff);
+ }
+ }
+ reader.skip();
+ //定点数转换为浮点数
+ double[] values = new double[n];
+ double eps1 = Math.pow(2, -p);
+ for (int i = 0; i < n; i++) {
+ values[i] = fixed[i] * eps1 + min;
+ }
+ return values;
+ }
+
+ private void encodeSubColumn(BitConstructor constructor, byte[] bytes, int len) {
+ //统计各种数值出现的次数，判断是否采用稀疏表示
+ Byte frequentValue = count(bytes, new HashMap<>());
+ if (frequentValue == null) {
+ constructor.add(0, 8);
+ encodeDenseSubColumn(constructor, bytes, len);
+ } else {
+ constructor.add(1, 8);
+ encodeSparseSubColumn(constructor, bytes, frequentValue, len);
+ }
+ }
+
+ private byte[] decodeSubColumn(BitReader reader, int n, int len) {
+ int sparseFlag = (int) reader.next(8);
+ switch (sparseFlag) {
+ case 0:
+ return decodeDenseSubColumn(reader, n, len);
+ case 1:
+ return decodeSparseSubColumn(reader, n, len);
+ default:
+ throw new RuntimeException("It cannot be reached.");
+ }
+ }
+
+ private void encodeDenseSubColumn(BitConstructor constructor, byte[] bytes, int len) {
+ //存储数据
+ for (byte b : bytes) {
+ constructor.add(b, len);
+ }
+ }
+
+ private byte[] decodeDenseSubColumn(BitReader reader, int n, int len) {
+ byte[] bytes = new byte[n];
+ for (int i = 0; i < n; i++) {
+ bytes[i] = (byte) reader.next(len);
+ }
+ return bytes;
+ }
+
+ private void encodeSparseSubColumn(BitConstructor constructor, byte[] bytes, byte frequentValue, int len) {
+ //存储众数
+ constructor.add(frequentValue, len);
+ //存储RLE压缩的比特向量
+ BitConstructor rle = new BitConstructor();
+ int cnt = encodeRLEVector(rle, bytes, frequentValue);
+ byte[] rleBytes = rle.toByteArray();
+ constructor.pad();
+ constructor.add(rleBytes.length, 32);
+ constructor.add(rleBytes);
+ //存储离群点
+ constructor.add(cnt, 32);
+ for (byte b : bytes) {
+ if (b != frequentValue) {
+ constructor.add(b, len);
+ }
+ }
+ }
+
+ private int encodeRLEVector(BitConstructor constructor, byte[] bytes, byte frequentValue) {
+ int width = getValueWidth(bytes.length);
+ boolean outlier = false;
+ int run = 0;
+ int cnt = 0;
+ for (byte b : bytes) {
+ if ((b != frequentValue) == outlier) {
+ run++;
+ } else {
+ outlier = !outlier;
+ constructor.add(run, width);
+ run = 1;
+ }
+ if (b != frequentValue) {
+ cnt++;
+ }
+ }
+ constructor.add(run, width);
+ return cnt;
+ }
+
+ private boolean[] decodeRLEVector(byte[] bytes, int n) {
+ BitReader reader = new BitReader(bytes);
+ boolean[] vector = new boolean[n];
+ int width = getValueWidth(n);
+ int i = 0;
+ boolean bit = false;
+ while (i < n) {
+ int run = (int) reader.next(width);
+ int j = i + run;
+ for (; i < j; i++) {
+ vector[i] = bit;
+ }
+ bit = !bit;
+ }
+ return vector;
+ }
+
+ private byte[] decodeSparseSubColumn(BitReader reader, int n, int len) {
+ //读取众数
+ byte frequentValue = (byte) reader.next(len);
+ //读取RLE比特向量
+ reader.skip();
+ int rleByteSize = (int) reader.next(32);
+ boolean vector[] = decodeRLEVector(reader.nextBytes(rleByteSize), n);
+ //读取离群点
+ int cnt = (int) reader.next(32);
+ byte[] bytes = new byte[n];
+ for (int i = 0; i < n; i++) {
+ if (vector[i]) {
+ bytes[i] = (byte) reader.next(len);
+ } else {
+ bytes[i] = frequentValue;
+ }
+ }
+ return bytes;
+ }
+
+ private Byte count(byte[] bytes, HashMap<Byte, Integer> map) {
+ for (byte x : bytes) {
+ map.put(x, map.getOrDefault(x, 0) + 1);
+ }
+ Byte maxByte = null;
+ int maxTimes = 0;
+ for (Map.Entry<Byte, Integer> entry : map.entrySet()) {
+ Byte key = entry.getKey();
+ Integer value = entry.getValue();
+ if (value > maxTimes) {
+ maxTimes = value;
+ maxByte = key;
+ }
+ }
+ if (maxTimes > sparseThreshold * bytes.length) {
+ return maxByte;
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * 计算x的数据宽度
+ *
+ * @param x
+ * @return 数据宽度
+ */
+ public int getValueWidth(long x) {
+ return 64 - Long.numberOfLeadingZeros(x);
+ }
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoderbuff/Encoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoderbuff/Encoder.java
new file mode 100644
index 0000000000..80503a061a
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoderbuff/Encoder.java
@@ -0,0 +1,17 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.apache.iotdb.tsfile.encoding.encoderbuff;
+
+/**
+ *
+ * @author Wang
+ */
+public interface Encoder {
+
+ public byte[] encode(double[] origin);
+
+ public double[] decode(byte[] bytes);
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/CompressionType.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/CompressionType.java
index 85893e7bd1..ee0c2bcc54 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/CompressionType.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/CompressionType.java
@@ -41,7 +41,10 @@ public enum CompressionType {
PLA(".pla", (byte) 6),
/** LZ4 */
- LZ4(".lz4", (byte) 7);
+ LZ4(".lz4", (byte) 7),
+
+ /** LZMA2 */
+ LZMA2(".lzma2", (byte) 8);
private final String extensionName;
private final byte index;
@@ -75,6 +78,8 @@ public enum CompressionType {
return CompressionType.PLA;
case 7:
return CompressionType.LZ4;
+ case 8:
+ return CompressionType.LZMA2;
default:
throw new IllegalArgumentException("Invalid input: " + compressor);
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/TSEncoding.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/TSEncoding.java
index de95facbfd..bb79319e46 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/TSEncoding.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/TSEncoding.java
@@ -27,7 +27,18 @@ public enum TSEncoding {
BITMAP((byte) 5),
GORILLA_V1((byte) 6),
REGULAR((byte) 7),
+<<<<<<< Updated upstream
GORILLA((byte) 8);
+=======
+ GORILLA((byte) 8),
+ SPRINTZ((byte) 9),
+ RAKE((byte) 10),
+ RLBE((byte) 11),
+ TEXTRLE((byte) 12),
+ HUFFMAN((byte) 13),
+ HUFFMANV2((byte) 14),
+ BUCKET((byte) 15);
+>>>>>>> Stashed changes
private final byte type;
@@ -65,6 +76,23 @@ public enum TSEncoding {
return TSEncoding.REGULAR;
case 8:
return TSEncoding.GORILLA;
+<<<<<<< Updated upstream
+=======
+ case 9:
+ return TSEncoding.SPRINTZ;
+ case 10:
+ return TSEncoding.RAKE;
+ case 11:
+ return TSEncoding.RLBE;
+ case 12:
+ return TSEncoding.TEXTRLE;
+ case 13:
+ return TSEncoding.HUFFMAN;
+ case 14:
+ return TSEncoding.HUFFMANV2;
+ case 15:
+ return TSEncoding.BUCKET;
+>>>>>>> Stashed changes
default:
throw new IllegalArgumentException("Invalid input: " + encoding);
}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/compress/CompressTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/compress/CompressTest.java
index d3d0406d41..5cbf80ac00 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/compress/CompressTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/compress/CompressTest.java
@@ -103,4 +103,16 @@ public class CompressTest {
String result = new String(uncompressed, StandardCharsets.UTF_8);
assertEquals(inputString, result);
}
+
+ @Test
+ public void lzma2CompressorTest() throws IOException {
+ PublicBAOS out = new PublicBAOS();
+ out.write(inputString.getBytes(StandardCharsets.UTF_8));
+ ICompressor compressor = new ICompressor.LZMA2Compressor();
+ IUnCompressor unCompressor = new IUnCompressor.LZMA2UnCompressor();
+ byte[] compressed = compressor.compress(out.getBuf());
+ byte[] uncompressed = unCompressor.uncompress(compressed);
+ String result = new String(uncompressed, StandardCharsets.UTF_8);
+ assertEquals(inputString, result);
+ }
}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/BucketDecoderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/BucketDecoderTest.java
new file mode 100644
index 0000000000..a0ef966ba5
--- /dev/null
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/BucketDecoderTest.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import org.apache.iotdb.tsfile.encoding.encoder.BucketEncoder;
+import org.apache.iotdb.tsfile.encoding.encoder.HuffmanEncoderV2;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.nio.ByteBuffer;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class BucketDecoderTest {
+ private BucketEncoder encoder = new BucketEncoder();
+ private BucketDecoder decoder = new BucketDecoder();
+ private ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ @Test
+ public void testSingle() {
+ int a[] = {1};
+ testAll(a);
+ a[0] = 2;
+ testAll(a);
+ a[0] = 1023;
+ testAll(a);
+ }
+
+ @Test
+ public void testAllUnique() {
+ int a[] = {1, 2, 3};
+ int b[] = {52, 123, 432};
+ int c[] = {54, 76, 42, 27, 35};
+ testAll(a);
+ testAll(b);
+ testAll(c);
+ }
+
+ @Test
+ public void testAllSame() {
+ int a[] = {20, 20, 20};
+ int b[] = {166, 166, 166};
+ testAll(a);
+ testAll(b);
+ }
+
+ @Test
+ public void testMixed() {
+ // all characters
+ int[] allChars = new int[1000000];
+ for (int i = 0; i < 1000000; i++) {
+ allChars[i] = (int)(1000000 * Math.random());
+ }
+ testAll(allChars);
+ }
+
+ private void testAll(int[] all) {
+ for (int s : all) {
+ encoder.encode(s, baos);
+ }
+ encoder.flush(baos);
+
+ ByteBuffer out = ByteBuffer.wrap(baos.toByteArray());
+
+ for (int s : all) {
+ assertTrue(decoder.hasNext(out));
+ int b = decoder.readInt(out);
+ assertEquals(s, b);
+ }
+
+ decoder.reset();
+ baos.reset();
+ }
+}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/ContestEncodeTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/ContestEncodeTest.java
new file mode 100644
index 0000000000..a6068eb6df
--- /dev/null
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/ContestEncodeTest.java
@@ -0,0 +1,328 @@
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import org.apache.iotdb.tsfile.compress.ICompressor;
+import org.apache.iotdb.tsfile.compress.IUnCompressor;
+import org.apache.iotdb.tsfile.encoding.encoder.Encoder;
+import org.apache.iotdb.tsfile.encoding.encoder.TSEncodingBuilder;
+import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+import java.io.*;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+
+public class ContestEncodeTest {
+
+ public static void main(@org.jetbrains.annotations.NotNull String[] args) throws IOException {
+ String inputPath = "C:\\Users\\xiaoj\\Desktop\\float",
+ Output = "C:\\Users\\xiaoj\\Desktop\\compressedResult.csv";
+ if (args.length >= 2) inputPath = args[1];
+ if (args.length >= 3) Output = args[2];
+ long s = System.nanoTime();
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+ TSEncoding[] schemeList = {
+// TSEncoding.PLAIN,
+ TSEncoding.TS_2DIFF
+// TSEncoding.RLE,
+// TSEncoding.SPRINTZ, TSEncoding.GORILLA, TSEncoding.RLBE, TSEncoding.RAKE
+ };
+ // CompressionType[] compressList = {CompressionType.LZ4, CompressionType.GZIP,
+ // CompressionType.SNAPPY};
+ CompressionType[] compressList = {CompressionType.UNCOMPRESSED}; //
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Encoding",
+ "Compress",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ assert tempList != null;
+ double ratio = 0;
+ for (File f : tempList) {
+// System.out.println(f);
+ InputStream inputStream = new FileInputStream(f);
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ String fileName = f.getAbsolutePath();
+ ArrayList<String> data = new ArrayList<>();
+ TSDataType dataType = TSDataType.INT32;
+
+ loader.readHeaders();
+ while (loader.readRecord()) {
+ data.add(loader.getValues()[1]);
+ }
+ loader.close();
+ inputStream.close();
+ if (fileName.contains("float")) {
+ dataType = TSDataType.FLOAT;
+ ArrayList<Float> tmp = new ArrayList<>();
+ for (String value : data) {
+ tmp.add(Float.valueOf(value));
+ }
+ for (TSEncoding scheme : schemeList) {
+ Encoder encoder = TSEncodingBuilder.getEncodingBuilder(scheme).getEncoder(dataType);
+ Decoder decoder = Decoder.getDecoderByType(scheme, dataType);
+ for (CompressionType comp : compressList) {
+ ICompressor compressor = ICompressor.getCompressor(comp);
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(comp);
+
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+ for (float val : tmp) encoder.encode(val, buffer);
+ encoder.flush(buffer);
+ byte[] elems = buffer.toByteArray();
+ byte[] compressed = compressor.compress(elems);
+ float ratioTmp = (float) compressed.length / (float) (tmp.size() * Long.BYTES);
+ ratio += ratioTmp;
+ buffer.close();
+ System.out.println(scheme);
+
+ }
+ }
+
+ }
+ }
+ String[] record = {
+ "TS_2DIFF",
+ "GZIP",
+ String.valueOf(ratio/71)
+ };
+ writer.writeRecord(record);
+ writer.close();
+ long e = System.nanoTime();
+ long encodeTime = (e - s);
+ System.out.println("encodeTime:");
+ System.out.println(encodeTime);
+ }
+}
+// if (fileName.contains("int")) {
+// dataType = TSDataType.INT32;
+// ArrayList<Integer> tmp = new ArrayList<>();
+// for (String value : data) {
+// tmp.add(Integer.valueOf(value));
+// }
+// for (TSEncoding scheme : schemeList) {
+// Encoder encoder = TSEncodingBuilder.getEncodingBuilder(scheme).getEncoder(dataType);
+// Decoder decoder = Decoder.getDecoderByType(scheme, dataType);
+// long encodeTime = 0;
+// long decodeTime = 0;
+// for (CompressionType comp : compressList) {
+// ICompressor compressor = ICompressor.getCompressor(comp);
+// IUnCompressor unCompressor = IUnCompressor.getUnCompressor(comp);
+// long compressTime = 0;
+// long uncompressTime = 0;
+// double ratio = 0;
+// for (int i = 0; i < repeatTime; i++) {
+// ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+// long s = System.nanoTime();
+// for (int val : tmp) encoder.encode(val, buffer);
+// long e = System.nanoTime();
+// encodeTime += (e - s);
+// System.out.println("encodeTime:");
+// System.out.println(e - s);
+// encoder.flush(buffer);
+//
+// byte[] elems = buffer.toByteArray();
+// s = System.nanoTime();
+// byte[] compressed = compressor.compress(elems);
+// e = System.nanoTime();
+// compressTime += (e - s);
+//
+// int ratioTmp = compressed.length / (tmp.size() * Long.BYTES);
+// ratio += ratioTmp;
+//
+// s = System.nanoTime();
+// byte[] x = unCompressor.uncompress(compressed);
+// e = System.nanoTime();
+// uncompressTime += (e - s);
+//
+// ByteBuffer ebuffer = ByteBuffer.wrap(buffer.toByteArray());
+// s = System.nanoTime();
+// while (decoder.hasNext(ebuffer)) {
+// decoder.readInt(ebuffer);
+// }
+// e = System.nanoTime();
+// decodeTime += (e - s);
+// buffer.close();
+// }
+// ratio /= repeatTime;
+// encodeTime = encodeTime / ((long) repeatTime);
+// decodeTime = decodeTime / ((long) repeatTime);
+// // compressTime = compressTime / ((long) repeatTime);
+// // uncompressTime = uncompressTime / ((long) repeatTime);
+// System.out.println(scheme.toString());
+// // System.out.println(String.valueOf(encodeTime));
+// String[] record = {
+// scheme.toString(),
+// comp.toString(),
+// String.valueOf(encodeTime),
+// "",
+// "",
+// String.valueOf(decodeTime),
+// String.valueOf(ratio)
+// };
+//
+// // String[] record = {scheme.toString(), comp.toString(),
+// // String.valueOf(encodeTime), String.valueOf(compressTime),
+// // String.valueOf(uncompressTime),
+// // String.valueOf(decodeTime), String.valueOf(ratio)};
+// writer.writeRecord(record);
+// }
+// }
+// }
+// else if (fileName.contains("long")) {
+// dataType = TSDataType.INT64;
+// ArrayList<Long> tmp = new ArrayList<>();
+// for (String value : data) {
+// tmp.add(Long.valueOf(value));
+// }
+// for (TSEncoding scheme : schemeList) {
+// Encoder encoder = TSEncodingBuilder.getEncodingBuilder(scheme).getEncoder(dataType);
+// Decoder decoder = Decoder.getDecoderByType(scheme, dataType);
+// long encodeTime = 0;
+// long decodeTime = 0;
+// for (CompressionType comp : compressList) {
+// ICompressor compressor = ICompressor.getCompressor(comp);
+// IUnCompressor unCompressor = IUnCompressor.getUnCompressor(comp);
+// long compressTime = 0;
+// long uncompressTime = 0;
+// double ratio = 0;
+// for (int i = 0; i < repeatTime; i++) {
+// ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+// long s = System.nanoTime();
+// for (long val : tmp) encoder.encode(val, buffer);
+// long e = System.nanoTime();
+// encodeTime += (e - s);
+// System.out.println("encodeTime:");
+// System.out.println(e - s);
+// encoder.flush(buffer);
+//
+// byte[] elems = buffer.toByteArray();
+// s = System.nanoTime();
+// byte[] compressed = compressor.compress(elems);
+// e = System.nanoTime();
+// compressTime += (e - s);
+//
+// int ratioTmp = compressed.length / (tmp.size() * Long.BYTES);
+// ratio += ratioTmp;
+//
+// s = System.nanoTime();
+// byte[] x = unCompressor.uncompress(compressed);
+// e = System.nanoTime();
+// uncompressTime += (e - s);
+//
+// ByteBuffer ebuffer = ByteBuffer.wrap(buffer.toByteArray());
+// s = System.nanoTime();
+// while (decoder.hasNext(ebuffer)) {
+// decoder.readLong(ebuffer);
+// }
+// e = System.nanoTime();
+// decodeTime += (e - s);
+// buffer.close();
+// }
+// ratio /= repeatTime;
+// encodeTime = encodeTime / ((long) repeatTime);
+// decodeTime = decodeTime / ((long) repeatTime);
+// // compressTime = compressTime / ((long) repeatTime);
+// // uncompressTime = uncompressTime / ((long) repeatTime);
+// System.out.println(scheme.toString());
+// // System.out.println(String.valueOf(encodeTime));
+// String[] record = {
+// scheme.toString(),
+// comp.toString(),
+// String.valueOf(encodeTime),
+// "",
+// "",
+// String.valueOf(decodeTime),
+// String.valueOf(ratio)
+// };
+//
+// // String[] record = {scheme.toString(), comp.toString(),
+// // String.valueOf(encodeTime), String.valueOf(compressTime),
+// // String.valueOf(uncompressTime),
+// // String.valueOf(decodeTime), String.valueOf(ratio)};
+// writer.writeRecord(record);
+// }
+// }
+// }
+// else if (fileName.contains("double")) {
+// dataType = TSDataType.DOUBLE;
+// ArrayList<Double> tmp = new ArrayList<>();
+// for (String value : data) {
+// tmp.add(Double.valueOf(value));
+// }
+// for (TSEncoding scheme : schemeList) {
+// Encoder encoder = TSEncodingBuilder.getEncodingBuilder(scheme).getEncoder(dataType);
+// Decoder decoder = Decoder.getDecoderByType(scheme, dataType);
+// long encodeTime = 0;
+// long decodeTime = 0;
+// for (CompressionType comp : compressList) {
+// ICompressor compressor = ICompressor.getCompressor(comp);
+// IUnCompressor unCompressor = IUnCompressor.getUnCompressor(comp);
+// long compressTime = 0;
+// long uncompressTime = 0;
+// double ratio = 0;
+// for (int i = 0; i < repeatTime; i++) {
+// ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+// long s = System.nanoTime();
+// for (double val : tmp) encoder.encode(val, buffer);
+// long e = System.nanoTime();
+// encodeTime += (e - s);
+// System.out.println("encodeTime:");
+// System.out.println(e - s);
+// encoder.flush(buffer);
+//
+// byte[] elems = buffer.toByteArray();
+// s = System.nanoTime();
+// byte[] compressed = compressor.compress(elems);
+// e = System.nanoTime();
+// compressTime += (e - s);
+//
+// int ratioTmp = compressed.length / (tmp.size() * Long.BYTES);
+// ratio += ratioTmp;
+//
+// s = System.nanoTime();
+// byte[] x = unCompressor.uncompress(compressed);
+// e = System.nanoTime();
+// uncompressTime += (e - s);
+//
+// ByteBuffer ebuffer = ByteBuffer.wrap(buffer.toByteArray());
+// s = System.nanoTime();
+// while (decoder.hasNext(ebuffer)) {
+// decoder.readDouble(ebuffer);
+// }
+// e = System.nanoTime();
+// decodeTime += (e - s);
+// buffer.close();
+// }
+// ratio /= repeatTime;
+// encodeTime = encodeTime / ((long) repeatTime);
+// decodeTime = decodeTime / ((long) repeatTime);
+// // compressTime = compressTime / ((long) repeatTime);
+// // uncompressTime = uncompressTime / ((long) repeatTime);
+// System.out.println(scheme.toString());
+// // System.out.println(String.valueOf(encodeTime));
+// String[] record = {
+// scheme.toString(),
+// comp.toString(),
+// String.valueOf(encodeTime),
+// "",
+// "",
+// String.valueOf(decodeTime),
+// String.valueOf(ratio)
+// };
+//
+// // String[] record = {scheme.toString(), comp.toString(),
+// // String.valueOf(encodeTime), String.valueOf(compressTime),
+// // String.valueOf(uncompressTime),
+// // String.valueOf(decodeTime), String.valueOf(ratio)};
+// writer.writeRecord(record);
+// }
+// }
+// }
\ No newline at end of file
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/EncodeTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/EncodeTest.java
new file mode 100644
index 0000000000..69c0facf2e
--- /dev/null
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/EncodeTest.java
@@ -0,0 +1,453 @@
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import org.apache.iotdb.tsfile.compress.ICompressor;
+import org.apache.iotdb.tsfile.compress.IUnCompressor;
+import org.apache.iotdb.tsfile.encoding.encoder.*;
+import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.encoding.DST.DST;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.apache.iotdb.tsfile.utils.Binary;
+
+import java.io.*;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+
+public class EncodeTest {
+
+ public static void main(@org.jetbrains.annotations.NotNull String[] args) throws IOException {
+ String inputPath = "C:\\Users\\xiaoj\\Desktop\\bufftest"; // the direction of input compressed data
+ String Output = "C:\\Users\\xiaoj\\Desktop\\enocderle.csv"; // the direction of output compression ratio and speed
+ int repeatTime = 1; // set repeat time
+ String dataTypeName = "double"; // set dataType
+// if (args.length >= 2) inputPath = args[1];
+// if (args.length >= 3) Output = args[2];
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ // select encoding algorithms
+ TSEncoding[] encodingList = {
+// TSEncoding.PLAIN ,
+// TSEncoding.TS_2DIFF,
+ TSEncoding.RLE//,
+// TSEncoding.SPRINTZ,
+// TSEncoding.GORILLA,
+// TSEncoding.RLBE,
+// TSEncoding.RAKE
+ };
+ // select compression algorithms
+ CompressionType[] compressList = {
+ CompressionType.UNCOMPRESSED,
+// CompressionType.LZ4,
+// CompressionType.GZIP,
+// CompressionType.SNAPPY
+ };
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Column Index",
+ "Encoding Algorithm",
+ "Compress Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Compress Time",
+ "Uncompress Time",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+ int fileRepeat = 0;
+ int[] columnIndexes = new int[71]; // set the column indexes of compressed
+
+ for(int i=0;i<71;i++) columnIndexes[i] = i;
+ for (File f : tempList) {
+ fileRepeat += 1;
+ InputStream inputStream = new FileInputStream(f);
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ String fileName = f.getAbsolutePath();
+ ArrayList<String> data = new ArrayList<>();
+
+ for(int index :columnIndexes){
+ // add a column to "data"
+ System.out.println(index);
+ loader.readHeaders();
+ while (loader.readRecord()) {
+ data.add(loader.getValues()[index]);
+ }
+// loader.close();
+ inputStream.close();
+
+
+ switch (dataTypeName) {
+ case "int": {
+ TSDataType dataType = TSDataType.INT32; // set TSDataType
+ ArrayList<Integer> tmp = new ArrayList<>();
+ for (String value : data) {
+ tmp.add(Integer.valueOf(value));
+ }
+ // Iterate over each encoding algorithm
+ for (TSEncoding encoding : encodingList) {
+ Encoder encoder = TSEncodingBuilder.getEncodingBuilder(encoding).getEncoder(dataType);
+ Decoder decoder = Decoder.getDecoderByType(encoding, dataType);
+ long encodeTime = 0;
+ long decodeTime = 0;
+
+ // Iterate over each compression algorithm
+ for (CompressionType comp : compressList) {
+ ICompressor compressor = ICompressor.getCompressor(comp);
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(comp);
+
+ double ratio = 0;
+ double compressed_size = 0;
+
+ long compressTime = 0;
+ long uncompressTime = 0;
+
+ // repeat many times to test time
+ for (int i = 0; i < repeatTime; i++) {
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+
+ // test encode time
+ long s = System.nanoTime();
+ for (int val : tmp) encoder.encode(val, buffer);
+ encoder.flush(buffer);
+ long e = System.nanoTime();
+ encodeTime += (e - s);
+
+ // test compress time
+ byte[] elems = buffer.toByteArray();
+ s = System.nanoTime();
+ byte[] compressed = compressor.compress(elems);
+ e = System.nanoTime();
+ compressTime += (e - s);
+
+ // test compression ratio and compressed size
+ compressed_size += compressed.length;
+ double ratioTmp = (double) compressed.length / (double) (tmp.size()* Integer.BYTES);
+ ratio += ratioTmp;
+
+ // test uncompress time
+ s = System.nanoTime();
+ byte[] x = unCompressor.uncompress(compressed);
+ e = System.nanoTime();
+ uncompressTime += (e - s);
+
+ // test decode time
+ ByteBuffer ebuffer = ByteBuffer.wrap(buffer.toByteArray());
+ s = System.nanoTime();
+ while (decoder.hasNext(ebuffer)) {
+ decoder.readInt(ebuffer);
+ }
+ e = System.nanoTime();
+ decodeTime += (e - s);
+
+ buffer.close();
+ }
+
+ ratio /= repeatTime;
+ compressed_size /= repeatTime;
+ encodeTime /= repeatTime;
+ decodeTime /= repeatTime;
+ compressTime /= repeatTime;
+ uncompressTime /= repeatTime;
+
+ String[] record = {
+ f.toString(),
+ String.valueOf(index),
+ encoding.toString(),
+ comp.toString(),
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(compressTime),
+ String.valueOf(uncompressTime),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ }
+ }
+ break;
+ }
+ case "long": {
+ TSDataType dataType = TSDataType.INT64;
+ ArrayList<Long> tmp = new ArrayList<>();
+ for (String value : data) {
+ tmp.add(Long.valueOf(value));
+ }
+ // Iterate over each encoding algorithm
+ for (TSEncoding encoding : encodingList) {
+ Encoder encoder = TSEncodingBuilder.getEncodingBuilder(encoding).getEncoder(dataType);
+ Decoder decoder = Decoder.getDecoderByType(encoding, dataType);
+ long encodeTime = 0;
+ long decodeTime = 0;
+
+ // Iterate over each compression algorithm
+ for (CompressionType comp : compressList) {
+ ICompressor compressor = ICompressor.getCompressor(comp);
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(comp);
+ double ratio = 0;
+ double compressed_size = 0;
+
+ long compressTime = 0;
+ long uncompressTime = 0;
+ for (int i = 0; i < repeatTime; i++) {
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+
+ // test encode time
+ long s = System.nanoTime();
+ for (long val : tmp) encoder.encode(val, buffer);
+ encoder.flush(buffer);
+ long e = System.nanoTime();
+ encodeTime += (e - s);
+
+ // test compress time
+ byte[] elems = buffer.toByteArray();
+ s = System.nanoTime();
+ byte[] compressed = compressor.compress(elems);
+ e = System.nanoTime();
+ compressTime += (e - s);
+
+ // test compression ratio and compressed size
+ compressed_size = compressed.length;
+ double ratioTmp = (double) compressed.length / (double) (tmp.size() * Long.BYTES);
+ ratio += ratioTmp;
+
+ // test uncompress time
+ s = System.nanoTime();
+ byte[] x = unCompressor.uncompress(compressed);
+ e = System.nanoTime();
+ uncompressTime += (e - s);
+
+ // test decode time
+ ByteBuffer ebuffer = ByteBuffer.wrap(buffer.toByteArray());
+ s = System.nanoTime();
+ while (decoder.hasNext(ebuffer)) {
+ decoder.readInt(ebuffer);
+ }
+ e = System.nanoTime();
+ decodeTime += (e - s);
+
+ buffer.close();
+ }
+
+ ratio /= repeatTime;
+ compressed_size /= repeatTime;
+ encodeTime /= repeatTime;
+ decodeTime /= repeatTime;
+ compressTime /= repeatTime;
+ uncompressTime /= repeatTime;
+
+ // write info to file
+ String[] record = {
+ f.toString(),
+ String.valueOf(index),
+ encoding.toString(),
+ comp.toString(),
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(compressTime),
+ String.valueOf(uncompressTime),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+
+ }
+ }
+ break;
+ }
+ case "double": {
+ TSDataType dataType = TSDataType.DOUBLE;
+ ArrayList<Double> tmp = new ArrayList<>();
+ for (String value : data) {
+ tmp.add(Double.valueOf(value));
+ }
+ // Iterate over each encoding algorithm
+ for (TSEncoding encoding : encodingList) {
+ Encoder encoder = TSEncodingBuilder.getEncodingBuilder(encoding).getEncoder(dataType);
+ Decoder decoder = Decoder.getDecoderByType(encoding, dataType);
+ long encodeTime = 0;
+ long decodeTime = 0;
+
+ // Iterate over each compression algorithm
+ for (CompressionType comp : compressList) {
+ ICompressor compressor = ICompressor.getCompressor(comp);
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(comp);
+ long compressTime = 0;
+ long uncompressTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ // repeat many times to test time
+ for (int i = 0; i < repeatTime; i++) {
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+
+ // test encode time
+ long s = System.nanoTime();
+ for (double val : tmp) encoder.encode(val, buffer);
+ encoder.flush(buffer);
+ long e = System.nanoTime();
+ encodeTime += (e - s);
+
+ // test compress time
+ byte[] elems = buffer.toByteArray();
+ s = System.nanoTime();
+ byte[] compressed = compressor.compress(elems);
+ e = System.nanoTime();
+ compressTime += (e - s);
+
+ // test compression ratio and compressed size
+ compressed_size = compressed.length;
+ double ratioTmp = (double) compressed.length / (double)(tmp.size() * Double.BYTES);
+ ratio += ratioTmp;
+
+ // test uncompress time
+ s = System.nanoTime();
+ byte[] x = unCompressor.uncompress(compressed);
+ e = System.nanoTime();
+ uncompressTime += (e - s);
+
+ // test decode time
+ ByteBuffer ebuffer = ByteBuffer.wrap(buffer.toByteArray());
+ s = System.nanoTime();
+ while (decoder.hasNext(ebuffer)) {
+ decoder.readDouble(ebuffer);
+ }
+ e = System.nanoTime();
+ decodeTime += (e - s);
+
+ buffer.close();
+ }
+
+ ratio /= repeatTime;
+ compressed_size /= repeatTime;
+ encodeTime /= repeatTime;
+ decodeTime /= repeatTime;
+ compressTime /= repeatTime;
+ uncompressTime /= repeatTime;
+
+ // write info to file
+ String[] record = {
+ f.toString(),
+ String.valueOf(index),
+ encoding.toString(),
+ comp.toString(),
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(compressTime),
+ String.valueOf(uncompressTime),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+
+ }
+ }
+ break;
+ }
+ case "float": {
+ TSDataType dataType = TSDataType.FLOAT;
+ ArrayList<Float> tmp = new ArrayList<>();
+ for (String value : data) {
+ tmp.add(Float.valueOf(value));
+ }
+
+ // Iterate over each encoding algorithm
+ for (TSEncoding encoding : encodingList) {
+ Encoder encoder = TSEncodingBuilder.getEncodingBuilder(encoding).getEncoder(dataType);
+ Decoder decoder = Decoder.getDecoderByType(encoding, dataType);
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ // Iterate over each compression algorithm
+ for (CompressionType comp : compressList) {
+ ICompressor compressor = ICompressor.getCompressor(comp);
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(comp);
+ long compressTime = 0;
+ long uncompressTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ // repeat many times to test time
+ for (int i = 0; i < repeatTime; i++) {
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+
+ // test encode time
+ long s = System.nanoTime();
+ for (float val : tmp) {
+ encoder.encode(val, buffer);
+ }
+ encoder.flush(buffer);
+ long e = System.nanoTime();
+ encodeTime += (e - s);
+
+ // test compress time
+ byte[] elems = buffer.toByteArray();
+ s = System.nanoTime();
+ byte[] compressed = compressor.compress(elems);
+ e = System.nanoTime();
+ compressTime += (e - s);
+
+ // test compression ratio and compressed size
+ compressed_size += compressed.length;
+ double ratioTmp =(double) compressed.length / (double)(tmp.size() * Float.BYTES);
+ ratio += ratioTmp;
+
+ // test uncompress time
+ s = System.nanoTime();
+ byte[] x = unCompressor.uncompress(compressed);
+ e = System.nanoTime();
+ uncompressTime += (e - s);
+
+ // test decode time
+ ByteBuffer ebuffer = ByteBuffer.wrap(buffer.toByteArray());
+ while (decoder.hasNext(ebuffer)) {
+ decoder.readFloat(ebuffer);
+ }
+ e = System.nanoTime();
+ decodeTime += (e - s);
+
+ buffer.close();
+ }
+ ratio /= repeatTime;
+ compressed_size /= repeatTime;
+ encodeTime /= repeatTime;
+ decodeTime /= repeatTime;
+ compressTime /= repeatTime;
+ uncompressTime /= repeatTime;
+
+ // write info to file
+ String[] record = {
+ f.toString(),
+ String.valueOf(index),
+ encoding.toString(),
+ comp.toString(),
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(compressTime),
+ String.valueOf(uncompressTime),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ }
+ }
+ break;
+ }
+ }
+
+ if (fileRepeat > repeatTime) break;
+ }
+
+ }
+ writer.close();
+ }
+}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/EncodeTestBuff.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/EncodeTestBuff.java
new file mode 100644
index 0000000000..e807a1add5
--- /dev/null
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/EncodeTestBuff.java
@@ -0,0 +1,72 @@
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+//import cn.edu.thu.diq.compression.encoderbuff.BuffEncoder;
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.apache.iotdb.tsfile.compress.ICompressor;
+import org.apache.iotdb.tsfile.encoding.encoderbuff.BuffEncoder;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+
+public class EncodeTestBuff {
+
+ public static void main(@org.jetbrains.annotations.NotNull String[] args) throws IOException {
+ BuffEncoder buffEncoder = new BuffEncoder(18);
+ String inputPath = "C:\\Users\\xiaoj\\Desktop\\bufftest";
+ String Output = "C:\\Users\\xiaoj\\Desktop\\compressedResultBUFF.csv"; // the direction of output compression ratio and speed
+
+
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+ String[] head = {"compressed size","encoded size"};
+ writer.writeRecord(head); // write header to output file
+
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+ for(int columnIndex=0;columnIndex<71;columnIndex++){
+ assert tempList != null;
+ for (File f : tempList) {
+ InputStream inputStream = new FileInputStream(f);
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ String fileName = f.getAbsolutePath();
+ ArrayList<String> data = new ArrayList<>();
+ loader.readHeaders();
+ while (loader.readRecord()) {
+ data.add(loader.getValues()[columnIndex]);
+ }
+ loader.close();
+ inputStream.close();
+ int len = data.size();
+ double[] tmp = new double[len];
+ for (int i=0;i<len;i++) {
+ tmp[i] = Double.parseDouble(data.get(i));
+ }
+ System.out.println(len);
+ // System.out.println(tmp[0]);
+ // System.out.println(tmp[1]);
+ // System.out.println(tmp[2]);
+ byte[] encodedBytes = buffEncoder.encode(tmp);
+ ICompressor.GZIPCompressor compressor = new ICompressor.GZIPCompressor();
+ System.out.println(encodedBytes.length);
+ byte[] compressed = compressor.compress(encodedBytes);
+ System.out.println(compressed.length);
+
+ // double[] decoded = buffEncoder.decode(encodedBytes);
+ // for (int i=0;i<len;i++) {
+ // String a_str = String.format("%.5f", decoded[i]);
+ // if(tmp[i] != Double.parseDouble(a_str)) System.out.println(i);
+ // }
+ String[] record = {String.valueOf(compressed.length),String.valueOf(encodedBytes.length)};
+ writer.writeRecord(record);
+ }
+
+ }
+
+ writer.close();
+ }
+}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/FloatDecoderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/FloatDecoderTest.java
index cbbdc8e6ef..4f5c524746 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/FloatDecoderTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/FloatDecoderTest.java
@@ -40,8 +40,8 @@ public class FloatDecoderTest {
private static final Logger logger = LoggerFactory.getLogger(FloatDecoderTest.class);
private final double delta = 0.0000001;
- private final int floatMaxPointValue = 10000;
- private final int floatMaxPointNumber = 4;
+ private final int floatMaxPointValue = 100000;
+ private final int floatMaxPointNumber = 5;
private final long doubleMaxPointValue = 1000000000000000L;
private final int doubleMaxPointNumber = 15;
private List<Float> floatList;
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/HuffmanV2DecoderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/HuffmanV2DecoderTest.java
new file mode 100644
index 0000000000..5b4b803a04
--- /dev/null
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/HuffmanV2DecoderTest.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import org.apache.iotdb.tsfile.encoding.encoder.HuffmanEncoder;
+import org.apache.iotdb.tsfile.encoding.encoder.HuffmanEncoderV2;
+import org.apache.iotdb.tsfile.utils.Binary;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.nio.ByteBuffer;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class HuffmanV2DecoderTest {
+ private HuffmanEncoderV2 encoder = new HuffmanEncoderV2(1024);
+ private HuffmanDecoderV2 decoder = new HuffmanDecoderV2();
+ private ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ @Test
+ public void testSingle() {
+ int a[] = {1};
+ testAll(a);
+ a[0] = 2;
+ testAll(a);
+ a[0] = 1023;
+ testAll(a);
+ }
+
+ @Test
+ public void testAllUnique() {
+ int a[] = {1, 2, 3};
+ int b[] = {52, 123, 432};
+ int c[] = {54, 76, 42, 27, 35};
+ testAll(a);
+ testAll(b);
+ testAll(c);
+ }
+
+ @Test
+ public void testAllSame() {
+ int a[] = {20, 20, 20};
+ int b[] = {166, 166, 166};
+ testAll(a);
+ testAll(b);
+ }
+
+ @Test
+ public void testMixed() {
+ // all characters
+ int[] allChars = new int[700];
+ for (int i = 0; i < 700; i++) {
+ allChars[i] = i;
+ }
+ testAll(allChars);
+ }
+
+ private void testAll(int[] all) {
+ for (int s : all) {
+ encoder.encode(s, baos);
+ }
+ encoder.flush(baos);
+
+ ByteBuffer out = ByteBuffer.wrap(baos.toByteArray());
+
+ for (int s : all) {
+ assertTrue(decoder.hasNext(out));
+ int b = decoder.readInt(out);
+ assertEquals(s, b);
+ }
+
+ decoder.reset();
+ baos.reset();
+ }
+}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/RLBEDecoderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/RLBEDecoderTest.java
new file mode 100644
index 0000000000..5a7547ef99
--- /dev/null
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/RLBEDecoderTest.java
@@ -0,0 +1,257 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import org.apache.iotdb.tsfile.encoding.encoder.*;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public class RLBEDecoderTest {
+
+ private static final Logger logger = LoggerFactory.getLogger(RLBEDecoderTest.class);
+ private final double delta = 0.0000001;
+ private final int floatMaxPointValue = 10000;
+ private final long doubleMaxPointValue = 1000000000000000L;
+ private List<Float> floatList;
+ private List<Double> doubleList;
+
+ @Before
+ public void setUp() {
+ floatList = new ArrayList<Float>();
+ int hybridCount = 11;
+ int hybridNum = 50;
+ int hybridStart = 2000;
+ for (int i = 0; i < hybridNum; i++) {
+ for (int j = 0; j < hybridCount; j++) {
+ floatList.add((float) hybridStart / floatMaxPointValue);
+ }
+ for (int j = 0; j < hybridCount; j++) {
+ floatList.add((float) hybridStart / floatMaxPointValue);
+ hybridStart += 3;
+ }
+
+ hybridCount += 2;
+ }
+
+ doubleList = new ArrayList<Double>();
+ int hybridCountDouble = 11;
+ int hybridNumDouble = 50;
+ long hybridStartDouble = 2000;
+
+ for (int i = 0; i < hybridNumDouble; i++) {
+ for (int j = 0; j < hybridCountDouble; j++) {
+ doubleList.add((double) hybridStartDouble / doubleMaxPointValue);
+ }
+ for (int j = 0; j < hybridCountDouble; j++) {
+ doubleList.add((double) hybridStartDouble / doubleMaxPointValue);
+ hybridStart += 3;
+ }
+
+ hybridCountDouble += 2;
+ }
+ }
+
+ @After
+ public void tearDown() {}
+
+ @Test
+ public void testNegativeNumber() throws IOException {
+ Encoder encoder = new DoubleRLBE();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ double value = -7.101f;
+ encoder.encode(value, baos);
+ encoder.encode(value - 2, baos);
+ encoder.encode(value - 4, baos);
+ encoder.flush(baos);
+ encoder.encode(value, baos);
+ encoder.encode(value - 2, baos);
+ encoder.encode(value - 4, baos);
+ encoder.flush(baos);
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+ for (int i = 0; i < 2; i++) {
+ Decoder decoder = new DoubleRLBEDecoder();
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readDouble(buffer), delta);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value - 2, decoder.readDouble(buffer), delta);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value - 4, decoder.readDouble(buffer), delta);
+ }
+ }
+ }
+
+ @Test
+ public void testZeroNumber() throws IOException {
+ Encoder encoder = new DoubleRLBE();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ double value = 0f;
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.flush(baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.flush(baos);
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+ for (int i = 0; i < 2; i++) {
+ Decoder decoder = new DoubleRLBEDecoder();
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readDouble(buffer), delta);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readDouble(buffer), delta);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readDouble(buffer), delta);
+ }
+ }
+ }
+
+ @Test
+ public void testFloatRepeat() throws Exception {
+ for (int i = 1; i <= 10; i++) {
+ testFloatLength(floatList, false, i);
+ }
+ }
+
+ @Test
+ public void testDoubleRepeat() throws Exception {
+ for (int i = 1; i <= 10; i++) {
+ testDoubleLength(doubleList, false, i);
+ }
+ }
+
+ @Test
+ public void testFloat() throws IOException {
+ Encoder encoder = new FloatRLBE();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ float value = 7.101f;
+ int num = 10000;
+ for (int i = 0; i < num; i++) {
+ encoder.encode(value + 2 * i, baos);
+ }
+ encoder.flush(baos);
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+ Decoder decoder = new FloatRLBEDecoder();
+ for (int i = 0; i < num; i++) {
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value + 2 * i, decoder.readFloat(buffer), delta);
+ continue;
+ }
+ fail();
+ }
+ }
+
+ @Test
+ public void testDouble() throws IOException {
+ Encoder encoder = new DoublePrecisionEncoderV1();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ double value = 7.101f;
+ int num = 1000;
+ for (int i = 0; i < num; i++) {
+ encoder.encode(value + 2 * i, baos);
+ }
+ encoder.flush(baos);
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+ Decoder decoder = new DoublePrecisionDecoderV1();
+ for (int i = 0; i < num; i++) {
+ if (decoder.hasNext(buffer)) {
+ // System.out.println("turn "+i);
+ assertEquals(value + 2 * i, decoder.readDouble(buffer), delta);
+ continue;
+ }
+ fail();
+ }
+ }
+
+ private void testFloatLength(List<Float> valueList, boolean isDebug, int repeatCount)
+ throws Exception {
+ Encoder encoder = new FloatRLBE();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ for (int i = 0; i < repeatCount; i++) {
+ for (float value : valueList) {
+ encoder.encode(value, baos);
+ }
+ encoder.flush(baos);
+ }
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+ for (int i = 0; i < repeatCount; i++) {
+
+ Decoder decoder = new FloatRLBEDecoder();
+ for (float value : valueList) {
+ // System.out.println("Repeat: "+i+" value: "+value);
+ if (decoder.hasNext(buffer)) {
+ float value_ = decoder.readFloat(buffer);
+ if (isDebug) {
+ logger.debug("{} // {}", value_, value);
+ }
+ assertEquals(value, value_, delta);
+ continue;
+ }
+ fail();
+ }
+ }
+ }
+
+ private void testDoubleLength(List<Double> valueList, boolean isDebug, int repeatCount)
+ throws Exception {
+ Encoder encoder = new DoublePrecisionEncoderV1();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ for (int i = 0; i < repeatCount; i++) {
+ for (double value : valueList) {
+ encoder.encode(value, baos);
+ }
+ encoder.flush(baos);
+ }
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ for (int i = 0; i < repeatCount; i++) {
+ Decoder decoder = new DoublePrecisionDecoderV1();
+ for (double value : valueList) {
+ if (decoder.hasNext(buffer)) {
+ double value_ = decoder.readDouble(buffer);
+ if (isDebug) {
+ logger.debug("{} // {}", value_, value);
+ }
+ assertEquals(value, value_, delta);
+ continue;
+ }
+ fail();
+ }
+ }
+ }
+}
+
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/SprintzDecoderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/SprintzDecoderTest.java
new file mode 100644
index 0000000000..a7d7947ff1
--- /dev/null
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/SprintzDecoderTest.java
@@ -0,0 +1,589 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import org.apache.iotdb.tsfile.encoding.encoder.*;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public class SprintzDecoderTest {
+ private static final double DELTA = 0;
+ private static final int FLOAT_MAX_POINT_VALUE = 10000;
+ private static final long DOUBLE_MAX_POINT_VALUE = 1000000000000000L;
+
+ private static final List<Float> floatList = new ArrayList<>();
+ private static final List<Double> doubleList = new ArrayList<>();
+ private static final List<Integer> intList = new ArrayList<>();
+ private static final List<Long> longList = new ArrayList<>();
+
+ private static final List<Integer> iterations = new ArrayList<>();
+
+ @BeforeClass
+ public static void setUp() {
+ int hybridCount = 11;
+ int hybridNum = 50;
+ int hybridStart = 2000;
+ for (int i = 0; i < hybridNum; i++) {
+ for (int j = 0; j < hybridCount; j++) {
+ floatList.add((float) hybridStart / FLOAT_MAX_POINT_VALUE);
+ doubleList.add((double) hybridStart / DOUBLE_MAX_POINT_VALUE);
+ intList.add(hybridStart);
+ longList.add((long) hybridStart);
+ }
+ for (int j = 0; j < hybridCount; j++) {
+ floatList.add((float) hybridStart / FLOAT_MAX_POINT_VALUE);
+ doubleList.add((double) hybridStart / DOUBLE_MAX_POINT_VALUE);
+ intList.add(hybridStart);
+ longList.add((long) hybridStart);
+ hybridStart += 3;
+ }
+ hybridCount += 2;
+ }
+
+ iterations.add(1);
+ iterations.add(3);
+ iterations.add(8);
+ iterations.add(16);
+ iterations.add(1000);
+ iterations.add(10000);
+ }
+
+ @Test
+ public void testIntSingleValue() throws IOException {
+ Encoder encoder = new IntSprintzEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ encoder.encode(777, baos);
+ encoder.flush(baos);
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ Decoder decoder = new IntSprintzDecoder();
+ if (decoder.hasNext(buffer)) {
+ assertEquals(777, decoder.readInt(buffer));
+ }
+ if (decoder.hasNext(buffer)) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testLongSingleValue() throws IOException {
+ Encoder encoder = new LongSprintzEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ encoder.encode((long) Integer.MAX_VALUE + 10, baos);
+ encoder.flush(baos);
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ Decoder decoder = new LongSprintzDecoder();
+ if (decoder.hasNext(buffer)) {
+ assertEquals((long) Integer.MAX_VALUE + 10, decoder.readLong(buffer));
+ }
+ if (decoder.hasNext(buffer)) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testIntZeroNumber() throws IOException {
+ Encoder encoder = new IntSprintzEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ int value = 0;
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.flush(baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.flush(baos);
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+ for (int i = 0; i < 2; i++) {
+ Decoder decoder = new IntSprintzDecoder();
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readInt(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readInt(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readInt(buffer), DELTA);
+ }
+ }
+ }
+
+ @Test
+ public void testLongZeroNumber() throws IOException {
+ Encoder encoder = new LongSprintzEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ long value = 0;
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.flush(baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.flush(baos);
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+ for (int i = 0; i < 2; i++) {
+ Decoder decoder = new LongSprintzDecoder();
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readLong(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readLong(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readLong(buffer), DELTA);
+ }
+ }
+ }
+
+ @Test
+ public void testInteger() throws IOException {
+ for (Integer num : iterations) {
+ Encoder encoder = new IntSprintzEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ int value = 7;
+ for (int i = 0; i < num; i++) {
+ encoder.encode(value + 2 * i, baos);
+ }
+ encoder.flush(baos);
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ Decoder decoder = new IntSprintzDecoder();
+ for (int i = 0; i < num; i++) {
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value + 2 * i, decoder.readInt(buffer));
+ continue;
+ }
+ fail();
+ }
+ if (decoder.hasNext(buffer)) {
+ fail();
+ }
+ }
+ }
+
+ @Test
+ public void testLong() throws IOException {
+ for (Integer num : iterations) {
+ Encoder encoder = new LongSprintzEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ long value = 7;
+ for (int i = 0; i < num; i++) {
+ encoder.encode(value + 2 * i, baos);
+ }
+ encoder.flush(baos);
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ Decoder decoder = new LongSprintzDecoder();
+ for (int i = 0; i < num; i++) {
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value + 2 * i, decoder.readLong(buffer));
+ continue;
+ }
+ fail();
+ }
+ if (decoder.hasNext(buffer)) {
+ fail();
+ }
+ }
+ }
+
+ @Test
+ public void testIntegerRepeat() throws Exception {
+ for (int i = 1; i <= 10; i++) {
+ testInteger(i);
+ }
+ }
+
+ private void testInteger(int repeatCount) throws Exception {
+ Encoder encoder = new IntSprintzEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ for (int i = 0; i < repeatCount; i++) {
+ for (int value : SprintzDecoderTest.intList) {
+ encoder.encode(value, baos);
+ }
+ encoder.flush(baos);
+ }
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ for (int i = 0; i < repeatCount; i++) {
+ Decoder decoder = new IntSprintzDecoder();
+ for (int expected : SprintzDecoderTest.intList) {
+ if (decoder.hasNext(buffer)) {
+ int actual = decoder.readInt(buffer);
+ assertEquals(expected, actual);
+ continue;
+ }
+ fail();
+ }
+ }
+
+ encoder = new IntSprintzEncoder();
+ baos = new ByteArrayOutputStream();
+ for (int i = 0; i < repeatCount; i++) {
+ for (int value : SprintzDecoderTest.intList) {
+ encoder.encode(-value, baos);
+ }
+ encoder.flush(baos);
+ }
+
+ buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ for (int i = 0; i < repeatCount; i++) {
+ Decoder decoder = new IntSprintzDecoder();
+ for (int expected : SprintzDecoderTest.intList) {
+ if (decoder.hasNext(buffer)) {
+ int actual = decoder.readInt(buffer);
+ assertEquals(expected, -actual);
+ continue;
+ }
+ fail();
+ }
+ }
+ }
+
+ @Test
+ public void testLongRepeat() throws Exception {
+ for (int i = 1; i <= 10; i++) {
+ testLong(i);
+ }
+ }
+
+ private void testLong(int repeatCount) throws Exception {
+ Encoder encoder = new LongSprintzEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ for (int i = 0; i < repeatCount; i++) {
+ for (long value : SprintzDecoderTest.longList) {
+ encoder.encode(value, baos);
+ }
+ encoder.flush(baos);
+ }
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ for (int i = 0; i < repeatCount; i++) {
+ Decoder decoder = new LongSprintzDecoder();
+ for (long expected : SprintzDecoderTest.longList) {
+ if (decoder.hasNext(buffer)) {
+ long actual = decoder.readLong(buffer);
+ assertEquals(expected, actual);
+ continue;
+ }
+ fail();
+ }
+ }
+
+ encoder = new LongSprintzEncoder();
+ baos = new ByteArrayOutputStream();
+ for (int i = 0; i < repeatCount; i++) {
+ for (long value : SprintzDecoderTest.longList) {
+ encoder.encode(-value, baos);
+ }
+ encoder.flush(baos);
+ }
+
+ buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ for (int i = 0; i < repeatCount; i++) {
+ Decoder decoder = new LongSprintzDecoder();
+ for (long expected : SprintzDecoderTest.longList) {
+ if (decoder.hasNext(buffer)) {
+ long actual = decoder.readLong(buffer);
+ assertEquals(expected, -actual);
+ continue;
+ }
+ fail();
+ }
+ }
+ }
+
+ @Test
+ public void testFloatSingleValue() throws IOException {
+ Encoder encoder = new FloatSprintzEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ encoder.encode(Float.MAX_VALUE, baos);
+ encoder.flush(baos);
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ Decoder decoder = new FloatSprintzDecoder();
+ if (decoder.hasNext(buffer)) {
+ assertEquals(Float.MAX_VALUE, decoder.readFloat(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testDoubleSingleValue() throws IOException {
+ Encoder encoder = new DoubleSprintzEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ encoder.encode(Double.MAX_VALUE, baos);
+ encoder.flush(baos);
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ Decoder decoder = new DoubleSprintzDecoder();
+ if (decoder.hasNext(buffer)) {
+ assertEquals(Double.MAX_VALUE, decoder.readDouble(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testFloatZeroNumber() throws IOException {
+ Encoder encoder = new FloatSprintzEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ float value = 0f;
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.flush(baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.flush(baos);
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+ for (int i = 0; i < 2; i++) {
+ Decoder decoder = new FloatSprintzDecoder();
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readFloat(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readFloat(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readFloat(buffer), DELTA);
+ }
+ }
+ }
+
+ @Test
+ public void testDoubleZeroNumber() throws IOException {
+ Encoder encoder = new DoubleSprintzEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ double value = 0f;
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.flush(baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.flush(baos);
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+ for (int i = 0; i < 2; i++) {
+ Decoder decoder = new DoubleSprintzDecoder();
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readDouble(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readDouble(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readDouble(buffer), DELTA);
+ }
+ }
+ }
+
+ @Test
+ public void testFloat() throws IOException {
+ for (Integer num : iterations) {
+ Encoder encoder = new FloatSprintzEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ float value = 7.101f;
+ for (int i = 0; i < num; i++) {
+ encoder.encode(value + 2 * i, baos);
+ }
+ encoder.flush(baos);
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ Decoder decoder = new FloatSprintzDecoder();
+ for (int i = 0; i < num; i++) {
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value + 2 * i, decoder.readFloat(buffer), DELTA);
+ continue;
+ }
+ fail();
+ }
+ if (decoder.hasNext(buffer)) {
+ fail();
+ }
+ }
+ }
+
+ @Test
+ public void testDouble() throws IOException {
+ for (Integer num : iterations) {
+ Encoder encoder = new DoubleSprintzEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ double value = 7.101f;
+ for (int i = 0; i < num; i++) {
+ encoder.encode(value + 2 * i, baos);
+ }
+ encoder.flush(baos);
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ Decoder decoder = new DoubleSprintzDecoder();
+ for (int i = 0; i < num; i++) {
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value + 2 * i, decoder.readDouble(buffer), DELTA);
+ continue;
+ }
+ fail();
+ }
+ if (decoder.hasNext(buffer)) {
+ fail();
+ }
+ }
+ }
+
+ @Test
+ public void testFloatRepeat() throws Exception {
+ for (int i = 1; i <= 10; i++) {
+ testFloat(i);
+ }
+ }
+
+ private void testFloat(int repeatCount) throws Exception {
+ Encoder encoder = new FloatSprintzEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ for (int i = 0; i < repeatCount; i++) {
+ for (float value : SprintzDecoderTest.floatList) {
+ encoder.encode(value, baos);
+ }
+ encoder.flush(baos);
+ }
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ for (int i = 0; i < repeatCount; i++) {
+ Decoder decoder = new FloatSprintzDecoder();
+ for (float expected : SprintzDecoderTest.floatList) {
+ if (decoder.hasNext(buffer)) {
+ float actual = decoder.readFloat(buffer);
+ assertEquals(expected, actual, DELTA);
+ continue;
+ }
+ fail();
+ }
+ }
+
+ encoder = new FloatSprintzEncoder();
+ baos = new ByteArrayOutputStream();
+ for (int i = 0; i < repeatCount; i++) {
+ for (float value : SprintzDecoderTest.floatList) {
+ encoder.encode(-value, baos);
+ }
+ encoder.flush(baos);
+ }
+
+ buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ for (int i = 0; i < repeatCount; i++) {
+ Decoder decoder = new FloatSprintzDecoder();
+ for (float expected : SprintzDecoderTest.floatList) {
+ if (decoder.hasNext(buffer)) {
+ float actual = decoder.readFloat(buffer);
+ assertEquals(expected, -actual, DELTA);
+ continue;
+ }
+ fail();
+ }
+ }
+ }
+
+ @Test
+ public void testDoubleRepeat() throws Exception {
+ for (int i = 1; i <= 10; i++) {
+ testDouble(i);
+ }
+ }
+
+ private void testDouble(int repeatCount) throws Exception {
+ Encoder encoder = new DoubleSprintzEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ for (int i = 0; i < repeatCount; i++) {
+ for (double value : SprintzDecoderTest.doubleList) {
+ encoder.encode(value, baos);
+ }
+ encoder.flush(baos);
+ }
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ for (int i = 0; i < repeatCount; i++) {
+ Decoder decoder = new DoubleSprintzDecoder();
+ for (double expected : SprintzDecoderTest.doubleList) {
+ if (decoder.hasNext(buffer)) {
+ double actual = decoder.readDouble(buffer);
+ assertEquals(expected, actual, DELTA);
+ continue;
+ }
+ fail();
+ }
+ }
+
+ encoder = new DoubleSprintzEncoder();
+ baos = new ByteArrayOutputStream();
+ for (int i = 0; i < repeatCount; i++) {
+ for (double value : SprintzDecoderTest.doubleList) {
+ encoder.encode(-value, baos);
+ }
+ encoder.flush(baos);
+ }
+
+ buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ for (int i = 0; i < repeatCount; i++) {
+ Decoder decoder = new DoubleSprintzDecoder();
+ for (double expected : SprintzDecoderTest.doubleList) {
+ if (decoder.hasNext(buffer)) {
+ double actual = decoder.readDouble(buffer);
+ assertEquals(expected, -actual, DELTA);
+ continue;
+ }
+ fail();
+ }
+ }
+ }
+}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/delta/DeltaBinaryEncoderIntegerTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/delta/DeltaBinaryEncoderIntegerTest.java
index 5043a23a3c..2a289596af 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/delta/DeltaBinaryEncoderIntegerTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/delta/DeltaBinaryEncoderIntegerTest.java
@@ -29,6 +29,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Random;
+import static java.lang.Math.pow;
import static org.junit.Assert.assertEquals;
public class DeltaBinaryEncoderIntegerTest {
@@ -46,6 +47,15 @@ public class DeltaBinaryEncoderIntegerTest {
reader = new DeltaBinaryDecoder.IntDeltaDecoder();
}
+ @Test
+ public void testtest() throws IOException{
+ int[] data = new int[20];
+ for (int i = 0; i < 20; i++) {
+ data[i] = i * (int)pow(-1, i);
+ }
+ shouldReadAndWrite(data, data.length);
+ }
+
@Test
public void testBasic() throws IOException {
int[] data = new int[ROW_NUM];
@@ -65,7 +75,7 @@ public class DeltaBinaryEncoderIntegerTest {
private void boundInt(int power, int[] data) throws IOException {
for (int i = 0; i < ROW_NUM; i++) {
- data[i] = ran.nextInt((int) Math.pow(2, power));
+ data[i] = ran.nextInt((int) pow(2, power));
}
shouldReadAndWrite(data, ROW_NUM);
}
@@ -106,5 +116,6 @@ public class DeltaBinaryEncoderIntegerTest {
while (reader.hasNext(buffer)) {
assertEquals(data[i++], reader.readInt(buffer));
}
+
}
}