You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hu...@apache.org on 2022/11/01 15:04:31 UTC
[iotdb] 01/01: update
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch research/encoding-exp
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 8b2f277d15e4b577e89f69c4a9d237576c2bfa4f
Author: xjz17 <67...@users.noreply.github.com>
AuthorDate: Tue Nov 1 22:55:18 2022 +0800
update
---
docs/UserGuide/Data-Concept/Encoding.md | 37 +-
.../resources/conf/iotdb-engine.properties | 2 +-
tsfile/pom.xml | 33 ++
.../iotdb/tsfile/common/conf/TSFileConfig.java | 3 +-
.../apache/iotdb/tsfile/compress/ICompressor.java | 99 ++++
.../iotdb/tsfile/compress/IUnCompressor.java | 46 ++
.../iotdb/tsfile/encoding/BitConstructor.java | 94 ++++
.../apache/iotdb/tsfile/encoding/BitReader.java | 67 +++
.../org/apache/iotdb/tsfile/encoding/DST/DST.java | 237 +++++++++
.../tsfile/encoding/HuffmanTree/HuffmanTree.java | 21 +
.../tsfile/encoding/decoder/BucketDecoder.java | 109 ++++
.../tsfile/encoding/decoder/HuffmanDecoder.java | 164 ++++++
.../tsfile/encoding/decoder/HuffmanDecoderV2.java | 138 +++++
.../tsfile/encoding/encoder/BucketEncoder.java | 145 +++++
.../encoding/encoder/DeltaBinaryEncoder.java | 3 +-
.../iotdb/tsfile/encoding/encoder/Encoder.java | 85 +++
.../tsfile/encoding/encoder/FloatEncoder.java | 5 +
.../tsfile/encoding/encoder/HuffmanEncoderV2.java | 257 +++++++++
.../tsfile/encoding/encoder/TSEncodingBuilder.java | 163 +++++-
.../tsfile/encoding/encoderbuff/BuffEncoder.java | 260 +++++++++
.../iotdb/tsfile/encoding/encoderbuff/Encoder.java | 17 +
.../file/metadata/enums/CompressionType.java | 7 +-
.../tsfile/file/metadata/enums/TSEncoding.java | 28 +
.../apache/iotdb/tsfile/compress/CompressTest.java | 12 +
.../tsfile/encoding/decoder/BucketDecoderTest.java | 92 ++++
.../tsfile/encoding/decoder/ContestEncodeTest.java | 328 ++++++++++++
.../iotdb/tsfile/encoding/decoder/EncodeTest.java | 453 ++++++++++++++++
.../tsfile/encoding/decoder/EncodeTestBuff.java | 72 +++
.../tsfile/encoding/decoder/FloatDecoderTest.java | 4 +-
.../encoding/decoder/HuffmanV2DecoderTest.java | 93 ++++
.../tsfile/encoding/decoder/RLBEDecoderTest.java | 257 +++++++++
.../encoding/decoder/SprintzDecoderTest.java | 589 +++++++++++++++++++++
.../delta/DeltaBinaryEncoderIntegerTest.java | 13 +-
33 files changed, 3917 insertions(+), 16 deletions(-)
diff --git a/docs/UserGuide/Data-Concept/Encoding.md b/docs/UserGuide/Data-Concept/Encoding.md
index 0bcafdd6ef..e277fb113c 100644
--- a/docs/UserGuide/Data-Concept/Encoding.md
+++ b/docs/UserGuide/Data-Concept/Encoding.md
@@ -50,10 +50,31 @@ Currently, there are two versions of GORILLA encoding implementation, it is reco
Usage restrictions: When using GORILLA to encode INT32 data, you need to ensure that there is no data point with the value `Integer.MIN_VALUE` in the sequence. When using GORILLA to encode INT64 data, you need to ensure that there is no data point with the value `Long.MIN_VALUE` in the sequence.
+* RAKE
+
+The RAKE encoding is based only on bits counting operations. It is more suitable for the ‘1’s of binary numbers to be more sparsely.
+
+* RLBE
+
+The RLBE encoding proposes to combine delta, run-length and Fibonacci based encoding ideas. It has five steps: differential coding, binary encoding, run-length, Fibonacci coding and concatenation.
+It is more suitable for the differential value of time series is positive and small.
+
+* SPRINTZ
+
+The SPRINTZ encoding combines encodings in four steps: predicting, bit-packing, run-length encoding and entropy encoding. SPRINTZ algorithm is suitable for predictable time series. For delta function, the vast repeats or linearly increasing time series is the best target.
+
* DICTIONARY
DICTIONARY encoding is lossless. It is suitable for TEXT data with low cardinality (i.e. low number of distinct values). It is not recommended to use it for high-cardinality data.
+* TEXTRLE
+
+TEXT Run-Length Encoding (TEXTRLE) performs especially for data with strings of repeated characters (the length of the string is called a run).
+
+* HUFFMAN
+
+It is more suitable for data with many high frequency values in skewed data distribution and many repeated characters.
+
## Correspondence between data type and encoding
@@ -63,13 +84,13 @@ The five encodings described in the previous sections are applicable to differen
**The correspondence between the data type and its supported encodings**
-|Data Type |Supported Encoding|
-|:---:|:---:|
-|BOOLEAN| PLAIN, RLE|
-|INT32 |PLAIN, RLE, TS_2DIFF, GORILLA|
-|INT64 |PLAIN, RLE, TS_2DIFF, GORILLA|
-|FLOAT |PLAIN, RLE, TS_2DIFF, GORILLA|
-|DOUBLE |PLAIN, RLE, TS_2DIFF, GORILLA|
-|TEXT |PLAIN, DICTIONARY|
+|Data Type | Supported Encoding |
+|:---:|:--------------------------------------------------:|
+|BOOLEAN| PLAIN, RLE |
+|INT32 | PLAIN, RLE, TS_2DIFF, GORILLA, RAKE, RLBE, SPRINTZ |
+|INT64 | PLAIN, RLE, TS_2DIFF, GORILLA, RAKE, RLBE, SPRINTZ |
+|FLOAT | PLAIN, RLE, TS_2DIFF, GORILLA, RAKE, RLBE, SPRINTZ |
+|DOUBLE | PLAIN, RLE, TS_2DIFF, GORILLA, RAKE, RLBE, SPRINTZ |
+|TEXT | PLAIN, DICTIONARY, TEXTRLE, HUFFMAN |
</div>
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index 33e9a767c4..b29a2074df 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -697,7 +697,7 @@ timestamp_precision=ms
# Floating-point precision
# Datatype: int
-# float_precision=2
+#float_precision=5
# Encoder configuration
# Encoder of time series, supports TS_2DIFF, PLAIN and RLE(run-length encoding), REGULAR and default value is TS_2DIFF
diff --git a/tsfile/pom.xml b/tsfile/pom.xml
index 10a39adbe7..7ca6e4d1db 100644
--- a/tsfile/pom.xml
+++ b/tsfile/pom.xml
@@ -58,6 +58,39 @@
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
</dependency>
+<<<<<<< Updated upstream
+=======
+ <dependency>
+ <groupId>net.sourceforge.javacsv</groupId>
+ <artifactId>javacsv</artifactId>
+ <version>2.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.jetbrains</groupId>
+ <artifactId>annotations</artifactId>
+ <version>RELEASE</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.carrotsearch</groupId>
+ <artifactId>hppc</artifactId>
+ <version>0.7.2</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.parquet</groupId>
+ <artifactId>parquet-hadoop-bundle</artifactId>
+ <version>1.8.1</version>
+ <groupId>org.tukaani</groupId>
+ <artifactId>xz</artifactId>
+ <version>1.9</version>
+ <scope>compile</scope>
+ </dependency>
+>>>>>>> Stashed changes
</dependencies>
<build>
<plugins>
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
index d13f817fd1..e76d8bafe6 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
@@ -82,7 +82,7 @@ public class TSFileConfig implements Serializable {
/** Max length limitation of input string. */
private int maxStringLength = 128;
/** Floating-point precision. */
- private int floatPrecision = 2;
+ private int floatPrecision = 5;
/**
* Encoder of time column, TsFile supports TS_2DIFF, PLAIN and RLE(run-length encoding) Default
* value is TS_2DIFF.
@@ -201,6 +201,7 @@ public class TSFileConfig implements Serializable {
}
public int getFloatPrecision() {
+ System.out.println(floatPrecision);
return floatPrecision;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/compress/ICompressor.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/compress/ICompressor.java
index 9243e53417..3ced75d0a7 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/compress/ICompressor.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/compress/ICompressor.java
@@ -25,7 +25,15 @@ import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import net.jpountz.lz4.LZ4Compressor;
import net.jpountz.lz4.LZ4Factory;
+<<<<<<< Updated upstream
+=======
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.tukaani.xz.XZInputStream;
+import org.tukaani.xz.XZOutputStream;
+>>>>>>> Stashed changes
import org.xerial.snappy.Snappy;
+import org.tukaani.xz.LZMA2Options;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
@@ -38,6 +46,7 @@ import java.util.zip.GZIPOutputStream;
import static org.apache.iotdb.tsfile.file.metadata.enums.CompressionType.GZIP;
import static org.apache.iotdb.tsfile.file.metadata.enums.CompressionType.LZ4;
import static org.apache.iotdb.tsfile.file.metadata.enums.CompressionType.SNAPPY;
+import static org.apache.iotdb.tsfile.file.metadata.enums.CompressionType.LZMA2;
/** compress data according to type in schema. */
public interface ICompressor extends Serializable {
@@ -65,6 +74,8 @@ public interface ICompressor extends Serializable {
return new IOTDBLZ4Compressor();
case GZIP:
return new GZIPCompressor();
+ case LZMA2:
+ return new LZMA2Compressor();
default:
throw new CompressionTypeNotSupportedException(name.toString());
}
@@ -311,4 +322,92 @@ public interface ICompressor extends Serializable {
return GZIP;
}
}
+ class LZMA2Compress {
+ private static LZMA2Options options;
+
+ public LZMA2Compress() {
+ options = new LZMA2Options();
+ }
+
+ public static byte[] compress(byte[] data) throws IOException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ XZOutputStream lzma2 = new XZOutputStream(out, options);
+ lzma2.write(data);
+ lzma2.close();
+ byte[] r = out.toByteArray();
+ return r;
+ }
+
+ public static byte[] uncompress(byte[] data) throws IOException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ByteArrayInputStream in = new ByteArrayInputStream(data);
+
+ XZInputStream unlzma2 = new XZInputStream(in);
+
+ byte[] buffer = new byte[256];
+ int n;
+ while ((n = unlzma2.read(buffer)) > 0) {
+ out.write(buffer, 0, n);
+ }
+ in.close();
+ byte[] r = out.toByteArray();
+ return r;
+ }
+ }
+
+ class LZMA2Compressor implements ICompressor {
+
+ private static LZMA2Compress Compress;
+ public LZMA2Compressor() {
+ Compress = new LZMA2Compress();
+ }
+
+ @Override
+ public byte[] compress(byte[] data) throws IOException {
+ if (null == data) {
+ return new byte[0];
+ }
+ byte[] r = Compress.compress(data);
+ return r;
+ }
+
+ @Override
+ public byte[] compress(byte[] data, int offset, int length) throws IOException {
+ byte[] dataBefore = new byte[length];
+ System.arraycopy(data, offset, dataBefore, 0, length);
+ byte[] r = Compress.compress(dataBefore);
+ return r;
+ }
+
+ @Override
+ public int compress(byte[] data, int offset, int length, byte[] compressed) throws IOException {
+ byte[] dataBefore = new byte[length];
+ System.arraycopy(data, offset, dataBefore, 0, length);
+ byte[] res = Compress.compress(dataBefore);
+ System.arraycopy(res, 0, compressed, 0, res.length);
+ return res.length;
+ }
+
+ @Override
+ public int compress(ByteBuffer data, ByteBuffer compressed) throws IOException {
+ int length = data.remaining();
+ byte[] dataBefore = new byte[length];
+ data.get(dataBefore, 0, length);
+ byte[] res = LZMA2Compress.compress(dataBefore);
+ compressed.put(res);
+ return res.length;
+ }
+
+ @Override
+ public int getMaxBytesForCompression(int uncompressedDataSize) {
+ // hard to estimate
+ return 40 + uncompressedDataSize;
+ //return Math.max(40 + uncompressedDataSize / 2, uncompressedDataSize);
+ }
+
+ @Override
+ public CompressionType getType() {
+ return LZMA2;
+ }
+ }
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/compress/IUnCompressor.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/compress/IUnCompressor.java
index b4b333e949..6c31b658ac 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/compress/IUnCompressor.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/compress/IUnCompressor.java
@@ -312,4 +312,50 @@ public interface IUnCompressor {
return CompressionType.GZIP;
}
}
+ class LZMA2UnCompressor implements IUnCompressor {
+
+ @Override
+ public int getUncompressedLength(byte[] array, int offset, int length) {
+ throw new UnsupportedOperationException("unsupported get uncompress length");
+ }
+
+ @Override
+ public int getUncompressedLength(ByteBuffer buffer) {
+ throw new UnsupportedOperationException("unsupported get uncompress length");
+ }
+
+ @Override
+ public byte[] uncompress(byte[] byteArray) throws IOException {
+ if (null == byteArray) {
+ return new byte[0];
+ }
+ return ICompressor.LZMA2Compress.uncompress(byteArray);
+ }
+
+ @Override
+ public int uncompress(byte[] byteArray, int offset, int length, byte[] output, int outOffset)
+ throws IOException {
+ byte[] dataBefore = new byte[length];
+ System.arraycopy(byteArray, offset, dataBefore, 0, length);
+ byte[] res = ICompressor.LZMA2Compress.uncompress(dataBefore);
+ System.arraycopy(res, 0, output, outOffset, res.length);
+ return res.length;
+ }
+
+ @Override
+ public int uncompress(ByteBuffer compressed, ByteBuffer uncompressed) throws IOException {
+ int length = compressed.remaining();
+ byte[] dataBefore = new byte[length];
+ compressed.get(dataBefore, 0, length);
+
+ byte[] res = ICompressor.LZMA2Compress.uncompress(dataBefore);
+ uncompressed.put(res);
+ return res.length;
+ }
+
+ @Override
+ public CompressionType getCodecName() {
+ return CompressionType.LZMA2;
+ }
+ }
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/BitConstructor.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/BitConstructor.java
new file mode 100644
index 0000000000..7c66bef620
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/BitConstructor.java
@@ -0,0 +1,94 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.apache.iotdb.tsfile.encoding;
+
+//import org.eclipse.collections.impl.list.mutable.primitive.ByteArrayList;
+
+import com.carrotsearch.hppc.ByteArrayList;
+
+public class BitConstructor {
+
+// public static long time = 0;
+ private static final int BITS_IN_A_BYTE = 8;
+ private static final long ALL_MASK = -1;
+ private final ByteArrayList data;
+ private byte cache = 0;
+ private int cnt = 0;
+
+ public BitConstructor() {
+ this.data = new ByteArrayList();
+ }
+
+ public BitConstructor(int initialCapacity) {
+ this.data = new ByteArrayList(initialCapacity);
+ }
+
+ public void add(long x, int len) {
+// long before = System.nanoTime();
+ x = x & ~(ALL_MASK << len);//保证x除最低的len位之外都是0
+ while (len > 0) {
+ int m = len + cnt >= BITS_IN_A_BYTE ? BITS_IN_A_BYTE - cnt : len;//向cache中插入的bit数
+ len -= m;
+ cnt += m;
+ byte y = (byte) (x >> len);
+ y = (byte) (y << (BITS_IN_A_BYTE - cnt));
+ cache = (byte) (cache | y);
+ x = x & ~(ALL_MASK << len);
+ if (cnt == BITS_IN_A_BYTE) {
+ pad();
+ }
+ }
+// long after = System.nanoTime();
+// time += (after - before);
+ }
+
+ public byte[] toByteArray() {
+ byte[] ret;
+ if (cnt > 0) {
+ data.add(cache);
+ ret = data.toArray();
+ data.remove(data.size() - 1);
+ } else {
+ ret = data.toArray();
+ }
+ return ret;
+ }
+
+ public void clear() {
+ data.clear();
+ cache = 0x00;
+ cnt = 0;
+ }
+
+ /**
+ * 如果当前字节存在剩余位，则全部填充0
+ */
+ public void pad() {
+ if (cnt > 0) {
+ data.add(cache);
+ cache = 0x00;
+ cnt = 0;
+ }
+ }
+
+ public void add(byte[] bytes) {
+ if (cnt == 0) {
+ data.add(bytes);
+ } else {
+ for (int i = 0; i < bytes.length; i++) {
+ add(bytes[i], 8);
+ }
+ }
+ }
+
+ public int length() {
+ return data.size();
+ }
+
+ public int lengthInBits() {
+ return data.size() * 8 + cnt;
+ }
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/BitReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/BitReader.java
new file mode 100644
index 0000000000..47568ed04e
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/BitReader.java
@@ -0,0 +1,67 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.apache.iotdb.tsfile.encoding;
+
+import java.util.Arrays;
+
+/**
+ *
+ * @author Wang Haoyu
+ */
+public class BitReader {
+
+ private static final int BITS_IN_A_BYTE = 8;
+ private static final byte MASKS[] = {(byte) 0xff, 0x7f, 0x3f, 0x1f, 0x0f, 0x07, 0x03, 0x01};
+ private final byte[] data;
+ private int byteCnt = 0, bitCnt = 0;
+
+ public BitReader(byte[] data) {
+ this.data = data;
+ }
+
+ public long next(int len) {
+ long ret = 0;
+ while (len > 0) {
+ int m = len + bitCnt >= BITS_IN_A_BYTE ? BITS_IN_A_BYTE - bitCnt : len;//从当前byte中读取的bit数
+ len -= m;
+ ret = ret << m;
+ byte y = (byte) (data[byteCnt] & MASKS[bitCnt]);//运算时byte自动转化为int，需要&截取低位
+ y = (byte) ((y & 0xff) >>> (BITS_IN_A_BYTE - bitCnt - m));//逻辑右移，高位补0
+ ret = ret | (y & 0xff);
+ bitCnt += m;
+ if (bitCnt == BITS_IN_A_BYTE) {
+ skip();
+ }
+ }
+ return ret;
+ }
+
+ public byte[] nextBytes(int len) {
+ byte[] ret;
+ if (bitCnt == 0) {
+ ret = Arrays.copyOfRange(data, byteCnt, byteCnt + len);
+ byteCnt += len;
+ } else {
+ ret = new byte[len];
+ for (int i = 0; i < len; i++) {
+ ret[i] = (byte) next(8);
+ }
+ }
+ return ret;
+ }
+
+ public void skip() {
+ if (bitCnt > 0) {
+ bitCnt = 0;
+ byteCnt++;
+ }
+ }
+
+ public boolean hasNext() {
+ return byteCnt < data.length;
+ }
+
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/DST/DST.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/DST/DST.java
new file mode 100644
index 0000000000..6a6222ce3e
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/DST/DST.java
@@ -0,0 +1,237 @@
+package org.apache.iotdb.tsfile.encoding.DST;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+
+public class DST {
+
+ private List<Integer> factor(int k, boolean[] isprime) {
+ List<Integer> ret = new ArrayList<>();
+ if(k == 1) {
+ ret.add(1);
+ return ret;
+ }
+ int p = 2;
+ while (k > 1) {
+ while(!isprime[p] && p <= k) {
+ p++;
+ }
+ while(k % p == 0) {
+ ret.add(p);
+ k /= p;
+ }
+ p++;
+ }
+ return ret;
+ }
+
+ private int gcd(int p, int q)
+ {
+ if(q==0)
+ return p;
+ int r = p%q;
+ return gcd(q, r);
+ }
+
+ public int[][] diffSeq(int N) {
+
+ boolean isprime[] = new boolean[N+1];
+ for(int i = 1; i <=N; i++) {
+ isprime[i] = true;
+ }
+ for(int i=2;i*i<=N;i++)
+ {
+ if(isprime[i])
+ {
+ int tmp = i+i;
+ while(tmp<=N)
+ {
+ isprime[tmp] = false;
+ tmp+=i;
+ }
+ }
+ }
+ isprime[1] = false;
+ //prime selector------------------
+
+ int W2[][] = new int[N+1][N+1];
+ for (int i = 1; i <= N; i++) {
+ for (int j = 1; j <= N; j++) {
+ W2[i][j] = 0;
+ if (j == 1) {
+ W2[i][j] = 1;
+ }
+ }
+ }
+ int a[] = {0, 1, -1};
+
+ for (int k = 1; k <= N; k++) {
+ if (k <= 2) {
+ W2[k][2] = a[k];
+ } else {
+ if (k % 2 == 0) {
+ W2[k][2] = a[2];
+ } else {
+ W2[k][2] = a[k%2];
+ }
+ }
+ }
+
+ for (int i = 3; i <= N; i++) {
+ if(isprime[i]) {
+ int b[] = new int[i - 1];
+ int c[] = new int[i + 1];
+ c[i - 1] = 1;
+ c[i] = -1;
+ for (int k = 1; k <= N; k++) {
+ if (k <= i)
+ W2[k][i] = c[k];
+ else {
+ if ((k) % (i) == 0)
+ W2[k][i] = c[i];
+ else
+ W2[k][i] = c[(k) % (i)];
+ }
+ }
+ } else {
+ List<Integer> d = factor(i, isprime);
+ d.add(0, 0);
+ HashSet<Integer> unidt = new HashSet<>(d);
+// for(int dd : d) {
+// System.out.print(dd);
+// System.out.print(" ");
+// }
+// System.out.println("");
+// for(int dd : unidt) {
+// System.out.print(dd);
+// System.out.print(" ");
+// }
+// System.out.println("");
+ List<Integer> unid = new ArrayList<>(unidt);
+// for(int dd : unid) {
+// System.out.print(dd);
+// System.out.print(" ");
+// }
+// System.out.println("");
+ int lenu = unid.size() - 1;
+ if (lenu == 1) {
+ int e[] = new int[i+1];
+
+ for (int k = 1; k <= d.get(1); k++) {
+ e[k+((k-1)*(i/d.get(1)-1))] = W2[k][d.get(1)];
+ }
+// for(int t = 1; t <= i; t++) {
+// System.out.print(e[t]);
+// System.out.print(" ");
+// }
+// System.out.println("");
+
+ for (int k = 1; k <= N; k++) {
+ if(k <= i)
+ W2[k][i] = e[k];
+ else {
+ if (k%i == 0)
+ W2[k][i] = e[i];
+ else
+ W2[k][i] = e[k%i];
+ }
+ }
+ }
+ else {
+ int muld[] = new int[lenu+1];
+ for (int k = 1; k <= lenu; k++) {
+ int tempk = unid.get(k);
+ int lend = 0;
+ for (int t = 1; t < d.size(); t++)
+ if (d.get(t) == tempk)
+ lend++;
+ muld[k] = (int)Math.pow(tempk, lend);
+ }
+ int f[] = new int[i+1];
+ for (int k = 1; k <= i; k++)
+ f[k] = 1;
+ for (int k = 1; k <= lenu; k++) {
+ for (int t = 1; t <= i; t++) {
+ f[t] *= W2[t][muld[k]];
+ }
+ }
+ for (int k = 1; k <= N; k++) {
+ if(k <= i)
+ W2[k][i] = f[k];
+ else {
+ if (k%i == 0)
+ W2[k][i] = f[i];
+ else
+ W2[k][i] = f[k%i];
+ }
+ }
+ }
+ }
+ }
+
+ return W2;
+ }
+
+ public int[][] DSTmatrix(int N) {
+ int z[][] = diffSeq(N);
+ int F[][] = new int[N+1][N+1];
+ int k = 1;
+ for (int i = 1; i <= N; i++) {
+ if(N%i == 0) {
+ int X[] = new int[N + 1];
+ for (int j = 1; j <= N; j++) {
+ F[j][k] = z[j][i];
+ X[j] = F[j][k];
+ }
+ int rp = 0;
+ for (int j = 1; j <= i; j++) {
+ if (gcd(j, i) == 1)
+ rp++;
+ }
+ for (int j = 1; j <= rp - 1; j++) {
+ F[1][k + j] = X[N];
+ for (int t = 2; t <= N; t++) {
+ F[t][k + j] = X[t - 1];
+ }
+ for (int t = 1; t <= N; t++) {
+ X[t] = F[t][k + j];
+ }
+ }
+ k += rp;
+ }
+ }
+ int FT[][] = new int[N+1][N+1];
+ for (int i = 1; i <= N; i++) {
+ for (int j = 1; j <= N; j++) {
+ FT[i][j] = F[j][i];
+ }
+ }
+ return FT;
+ }
+
+ public List<Integer> fit(List<Integer> org, int N) {
+ int MT[][] = DSTmatrix(N);
+ List<Integer> ret = new ArrayList<>();
+ int s = org.size() / N + 1;
+ for (int k = 0; k < s - 1; k++) {
+ for (int i = 0; i < N; i++) {
+ int tmp = 0;
+ for (int j = 0; j < N; j++) {
+ tmp += MT[i+1][j+1]*org.get(k*N+j);
+ }
+ ret.add(tmp);
+ }
+ }
+ for (int i = 0; i < org.size()-s*N; i++) {
+ int tmp = 0;
+ for (int j = 0; j < org.size()-s*N; j++) {
+ tmp += MT[i+1][j+1]*org.get(s*N+j);
+ }
+ ret.add(tmp);
+ }
+ return ret;
+ }
+
+}
+
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/HuffmanTree/HuffmanTree.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/HuffmanTree/HuffmanTree.java
new file mode 100644
index 0000000000..fd3193b7c1
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/HuffmanTree/HuffmanTree.java
@@ -0,0 +1,21 @@
+package org.apache.iotdb.tsfile.encoding.HuffmanTree;
+
+public class HuffmanTree {
+ public boolean isLeaf = false;
+ public int frequency;
+ public int originalbyte;
+ public boolean isRecordEnd = false;
+ public HuffmanTree leftNode;
+ public HuffmanTree rightNode;
+
+ public void HuffmanTree() {}
+
+ public void clear() {
+ isLeaf = false;
+ isRecordEnd = false;
+ frequency = 0;
+ originalbyte = 0;
+ leftNode = null;
+ rightNode = null;
+ }
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/BucketDecoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/BucketDecoder.java
new file mode 100644
index 0000000000..40efdfaf3b
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/BucketDecoder.java
@@ -0,0 +1,109 @@
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import org.apache.iotdb.tsfile.encoding.HuffmanTree.HuffmanCode;
+import org.apache.iotdb.tsfile.encoding.HuffmanTree.HuffmanTree;
+import org.apache.iotdb.tsfile.encoding.encoder.IntRleEncoder;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.PriorityQueue;
+
+public class BucketDecoder extends Decoder {
+ private int recordNum = -1;
+ private int minNum;
+ private List<Integer> level;
+ private List<Integer> rec;
+ private int curPos;
+ private int segmentLength;
+ public BucketDecoder() {
+ super(TSEncoding.BUCKET);
+ recordNum = -1;
+ level = new ArrayList<>();
+ rec = new ArrayList<>();
+ curPos = 0;
+ }
+
+ @Override
+ public void reset() {
+ recordNum = -1;
+ level.clear();
+ rec.clear();
+ curPos = 0;
+ }
+
+ public int readInt(ByteBuffer buffer) {
+ if(recordNum == -1) {
+ reset();
+ minNum = buffer.getInt();
+ segmentLength = buffer.getInt();
+ HuffmanDecoderV2 decoder = new HuffmanDecoderV2();
+ int tmp = decoder.readInt(buffer);
+ level.add(tmp);
+ recordNum = decoder.recordnum;
+ for(int i = 0; i < recordNum-1; i++) {
+ tmp = decoder.readInt(buffer);
+ level.add(tmp);
+ }
+ int bitwidth = 32-Integer.numberOfLeadingZeros(segmentLength) - 1;
+ for(int i = 0; i < recordNum; i++) {
+ tmp = 0;
+ for(int j = bitwidth-1; j >=0; j--) {
+ int b = readbit(buffer);
+ tmp |= (b<<j);
+ }
+ int lv = level.get(i);
+ rec.add(tmp + minNum + lv * segmentLength);
+ }
+ }
+ int ret = rec.get(curPos++);
+ if(curPos == recordNum) {
+ reset();
+ clearBuffer(buffer);
+ }
+ return ret;
+ }
+
+ @Override
+ public boolean hasNext(ByteBuffer buffer) {
+ if(recordNum == -1) {
+ if (buffer.hasRemaining())
+ return true;
+ else
+ return false;
+ }
+ else if(curPos < recordNum)
+ return true;
+ else
+ return false;
+ }
+
+ private int numberLeftInBuffer = 0;
+ private int byteBuffer = 0;
+
+
+ private int readbit(ByteBuffer buffer) {
+ if (numberLeftInBuffer == 0) {
+ loadBuffer(buffer);
+ numberLeftInBuffer = 8;
+ }
+ int top = ((byteBuffer >> 7) & 1);
+ byteBuffer <<= 1;
+ numberLeftInBuffer--;
+ return top;
+ }
+
+ private void loadBuffer(ByteBuffer buffer) {
+ byteBuffer = buffer.get();
+ }
+
+ private void clearBuffer(ByteBuffer buffer) {
+ while (numberLeftInBuffer > 0) {
+ readbit(buffer);
+ }
+ }
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/HuffmanDecoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/HuffmanDecoder.java
new file mode 100644
index 0000000000..4f5b8d7935
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/HuffmanDecoder.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import org.apache.iotdb.tsfile.encoding.HuffmanTree.HuffmanTree;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.utils.Binary;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Queue;
+
+public class HuffmanDecoder extends Decoder {
+
+ private int recordnum;
+ private int numberLeftInBuffer;
+ private byte byteBuffer;
+ private Queue<Binary> records;
+ private HuffmanTree tree;
+
+ HuffmanDecoder() {
+ super(TSEncoding.HUFFMAN);
+ records = new LinkedList<>();
+ tree = new HuffmanTree();
+ reset();
+ }
+
+ @Override
+ public Binary readBinary(ByteBuffer buffer) {
+ if (records.isEmpty()) {
+ reset();
+ loadTree(buffer);
+ loadRecords(buffer);
+ clearBuffer(buffer);
+ }
+ return records.poll();
+ }
+
+ public boolean hasNext(ByteBuffer buffer) {
+ return ((!records.isEmpty()) || buffer.hasRemaining());
+ }
+
+ private void loadTree(ByteBuffer buffer) {
+ recordnum = getInt(buffer);
+ int endOfRecordLength = getInt(buffer);
+ HuffmanTree header = tree;
+ for (int i = 0; i < endOfRecordLength; i++) {
+ if (readbit(buffer) == 0) {
+ if (header.leftNode == null) header.leftNode = new HuffmanTree();
+ header = header.leftNode;
+ } else {
+ if (header.rightNode == null) header.rightNode = new HuffmanTree();
+ header = header.rightNode;
+ }
+ if (i == endOfRecordLength - 1) {
+ header.isLeaf = true;
+ header.isRecordEnd = true;
+ }
+ }
+
+ int usednum = getInt(buffer);
+ for (int i = 0; i < usednum; i++) {
+ byte cha = getByte(buffer);
+ int codeLength = getInt(buffer);
+ String s = new String();
+ HuffmanTree tempTree = tree;
+ for (int j = 0; j < codeLength; j++) {
+ int b = readbit(buffer);
+ if (b == 0) {
+ if (tempTree.leftNode == null) tempTree.leftNode = new HuffmanTree();
+ tempTree = tempTree.leftNode;
+ } else {
+ if (tempTree.rightNode == null) tempTree.rightNode = new HuffmanTree();
+ tempTree = tempTree.rightNode;
+ }
+ if (j == codeLength - 1) {
+ tempTree.isLeaf = true;
+ tempTree.originalbyte = cha;
+ }
+ }
+ }
+ }
+
+ private void loadRecords(ByteBuffer buffer) {
+ for (int i = 0; i < recordnum; i++) {
+ HuffmanTree tempTree = tree;
+ List<Byte> rec = new ArrayList<>();
+ while (true) {
+ tempTree = tree;
+ while (!tempTree.isLeaf) {
+ if (readbit(buffer) == 0) tempTree = tempTree.leftNode;
+ else tempTree = tempTree.rightNode;
+ }
+ if (tempTree.isRecordEnd) break;
+ rec.add((byte) tempTree.originalbyte);
+ }
+ byte[] currec = new byte[rec.size()];
+ for (int j = 0; j < rec.size(); j++) currec[j] = rec.get(j);
+ records.add(new Binary(currec));
+ }
+ }
+
+ public void reset() {
+ recordnum = 0;
+ records.clear();
+ tree.clear();
+ }
+
+ private int getInt(ByteBuffer buffer) {
+ int val = 0;
+ for (int i = 31; i >= 0; i--) {
+ val |= (readbit(buffer) << i);
+ }
+ return val;
+ }
+
+ private byte getByte(ByteBuffer buffer) {
+ byte val = 0;
+ for (int i = 7; i >= 0; i--) {
+ val |= (readbit(buffer) << i);
+ }
+ return val;
+ }
+
+ private int readbit(ByteBuffer buffer) {
+ if (numberLeftInBuffer == 0) {
+ loadBuffer(buffer);
+ numberLeftInBuffer = 8;
+ }
+ int top = ((byteBuffer >> 7) & 1);
+ byteBuffer <<= 1;
+ numberLeftInBuffer--;
+ return top;
+ }
+
+ private void loadBuffer(ByteBuffer buffer) {
+ byteBuffer = buffer.get();
+ }
+
+ private void clearBuffer(ByteBuffer buffer) {
+ while (numberLeftInBuffer > 0) {
+ readbit(buffer);
+ }
+ }
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/HuffmanDecoderV2.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/HuffmanDecoderV2.java
new file mode 100644
index 0000000000..2da75ed03f
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/HuffmanDecoderV2.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import org.apache.iotdb.tsfile.encoding.HuffmanTree.HuffmanTree;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.utils.Binary;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Queue;
+
+public class HuffmanDecoderV2 extends Decoder {
+
+ public int recordnum;
+ private int numberLeftInBuffer;
+ private byte byteBuffer;
+ private Queue<Integer> records;
+ private HuffmanTree tree;
+
+ HuffmanDecoderV2() {
+ super(TSEncoding.HUFFMAN);
+ records = new LinkedList<>();
+ tree = new HuffmanTree();
+ reset();
+ }
+
+ @Override
+ public int readInt(ByteBuffer buffer) {
+ if (records.isEmpty()) {
+ reset();
+ loadTree(buffer);
+ loadRecords(buffer);
+ clearBuffer(buffer);
+ }
+ return records.poll();
+ }
+
+ public boolean hasNext(ByteBuffer buffer) {
+ return ((!records.isEmpty()) || buffer.hasRemaining());
+ }
+
+ private void loadTree(ByteBuffer buffer) {
+ recordnum = getInt(buffer);
+ int usednum = getInt(buffer);
+ for (int i = 0; i < usednum; i++) {
+ int cha = getInt(buffer);
+ int codeLength = getInt(buffer);
+ HuffmanTree tempTree = tree;
+ for (int j = 0; j < codeLength; j++) {
+ int b = readbit(buffer);
+ if (b == 0) {
+ if (tempTree.leftNode == null) tempTree.leftNode = new HuffmanTree();
+ tempTree = tempTree.leftNode;
+ } else {
+ if (tempTree.rightNode == null) tempTree.rightNode = new HuffmanTree();
+ tempTree = tempTree.rightNode;
+ }
+ }
+ tempTree.isLeaf = true;
+ tempTree.originalbyte = cha;
+ }
+ }
+
+ private void loadRecords(ByteBuffer buffer) {
+ HuffmanTree tempTree = tree;
+ for(int i = 0; i < recordnum; i++) {
+ tempTree = tree;
+ while (!tempTree.isLeaf) {
+ if (readbit(buffer) == 0) tempTree = tempTree.leftNode;
+ else tempTree = tempTree.rightNode;
+ }
+ records.add(tempTree.originalbyte);
+ }
+ }
+
+ public void reset() {
+ recordnum = 0;
+ records.clear();
+ tree.clear();
+ }
+
+ private int getInt(ByteBuffer buffer) {
+ int val = 0;
+ for (int i = 31; i >= 0; i--) {
+ val |= (readbit(buffer) << i);
+ }
+ return val;
+ }
+
+ private byte getByte(ByteBuffer buffer) {
+ byte val = 0;
+ for (int i = 7; i >= 0; i--) {
+ val |= (readbit(buffer) << i);
+ }
+ return val;
+ }
+
+ private int readbit(ByteBuffer buffer) {
+ if (numberLeftInBuffer == 0) {
+ loadBuffer(buffer);
+ numberLeftInBuffer = 8;
+ }
+ int top = ((byteBuffer >> 7) & 1);
+ byteBuffer <<= 1;
+ numberLeftInBuffer--;
+ return top;
+ }
+
+ private void loadBuffer(ByteBuffer buffer) {
+ byteBuffer = buffer.get();
+ }
+
+ private void clearBuffer(ByteBuffer buffer) {
+ while (numberLeftInBuffer > 0) {
+ readbit(buffer);
+ }
+ }
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/BucketEncoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/BucketEncoder.java
new file mode 100644
index 0000000000..1c99307cec
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/BucketEncoder.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.encoder;
+
+import org.apache.iotdb.tsfile.encoding.DST.DST;
+import org.apache.iotdb.tsfile.encoding.HuffmanTree.HuffmanCode;
+import org.apache.iotdb.tsfile.encoding.HuffmanTree.HuffmanTree;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.read.filter.operator.In;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.PriorityQueue;
+
+public class BucketEncoder extends Encoder {
+
+ private byte byteBuffer;
+ private int numberLeftInBuffer = 0;
+ private int totLength;
+
+ private int minNum = 0x7FFFFFFF;
+ private int maxNum = -0x7FFFFFFF;
+ private int segmentLength =64;//2048; //65536 / 128;
+ private List<Integer> valueBuffer;
+
+ public BucketEncoder() {
+ super(TSEncoding.BUCKET);
+ valueBuffer = new ArrayList<>();
+ }
+
+ public BucketEncoder(int _segmentLength) {
+ super(TSEncoding.HUFFMANV2);
+ segmentLength = _segmentLength;
+ valueBuffer = new ArrayList<>();
+ }
+
+ private void reset() {
+ minNum = 0x7FFFFFFF;
+ maxNum = -0x7FFFFFFF;
+ valueBuffer.clear();
+ }
+
+ @Override
+ public void encode(int value, ByteArrayOutputStream out) {
+ valueBuffer.add(value);
+ minNum = Math.min(minNum, value);
+ maxNum = Math.max(maxNum, value);
+ }
+
+ @Override
+ public void encode(float value, ByteArrayOutputStream out) {
+ int _value = convertFloatToInt(value);
+ encode(_value, out);
+ }
+
+ private int convertFloatToInt(float value) {
+ return (int) Math.round(value * 100000);
+ }
+
+ @Override
+ public void flush(ByteArrayOutputStream out) {
+ writeInt(minNum, out);
+ writeInt(segmentLength, out);
+ int segmentNum = (maxNum - minNum) / segmentLength + 1;
+ HuffmanEncoderV2 huffmanEncoder = new HuffmanEncoderV2(segmentNum);
+ for (int e : valueBuffer) {
+ huffmanEncoder.encode((e - minNum) / segmentLength, out);
+ }
+ huffmanEncoder.flush(out);
+// System.out.println(" huffman length: ");
+// System.out.print(" ");
+// System.out.println(out.size());
+ int temp = out.size();
+
+ int bitwidth = 32 - Integer.numberOfLeadingZeros(segmentLength) - 1;
+ for (int e : valueBuffer) {
+ int encodedValue = (e - minNum) % segmentLength;
+ for (int i = bitwidth - 1; i >= 0; i--) {
+ writeBit((encodedValue & (1 << i)) > 0, out);
+ }
+ }
+ clearBuffer(out);
+
+// System.out.println(" total length: ");
+// System.out.print(" ");
+// System.out.println(out.size() - temp);
+ reset();
+ }
+
+
+ protected void writeBit(boolean b, ByteArrayOutputStream out) {
+ byteBuffer <<= 1;
+ if (b) {
+ byteBuffer |= 1;
+ }
+
+ numberLeftInBuffer++;
+ if (numberLeftInBuffer == 8) {
+ clearBuffer(out);
+ }
+ }
+
+ protected void clearBuffer(ByteArrayOutputStream out) {
+ if (numberLeftInBuffer == 0) return;
+ if (numberLeftInBuffer > 0) byteBuffer <<= (8 - numberLeftInBuffer);
+ out.write(byteBuffer);
+ totLength++;
+ numberLeftInBuffer = 0;
+ byteBuffer = 0;
+ }
+
+ private void writeInt(int val, ByteArrayOutputStream out) {
+ for (int i = 31; i >= 0; i--) {
+ if ((val & (1 << i)) > 0) writeBit(true, out);
+ else writeBit(false, out);
+ }
+ }
+
+ private void writeByte(byte val, ByteArrayOutputStream out) {
+ for (int i = 7; i >= 0; i--) {
+ if ((val & (1 << i)) > 0) writeBit(true, out);
+ else writeBit(false, out);
+ }
+ }
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DeltaBinaryEncoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DeltaBinaryEncoder.java
index b95846085e..767d74dd76 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DeltaBinaryEncoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DeltaBinaryEncoder.java
@@ -45,7 +45,7 @@ import java.io.IOException;
*/
public abstract class DeltaBinaryEncoder extends Encoder {
- protected static final int BLOCK_DEFAULT_SIZE = 128;
+ protected static final int BLOCK_DEFAULT_SIZE = 256;
private static final Logger logger = LoggerFactory.getLogger(DeltaBinaryEncoder.class);
protected ByteArrayOutputStream out;
protected int blockSize;
@@ -211,6 +211,7 @@ public abstract class DeltaBinaryEncoder extends Encoder {
@Override
public void encode(int value, ByteArrayOutputStream out) {
+// System.out.println(value);
encodeValue(value, out);
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/Encoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/Encoder.java
index b201fb97da..56350cbdcb 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/Encoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/Encoder.java
@@ -32,6 +32,7 @@ import java.math.BigDecimal;
*/
public abstract class Encoder {
+<<<<<<< Updated upstream
public static final String MAX_STRING_LENGTH = "max_string_length";
public static final String MAX_POINT_NUMBER = "max_point_number";
@@ -108,4 +109,88 @@ public abstract class Encoder {
public long getMaxByteSize() {
throw new UnsupportedOperationException();
}
+=======
+ public static final String MAX_STRING_LENGTH = "max_string_length";
+ public static final String MAX_POINT_NUMBER = "max_point_number";
+
+ private TSEncoding type;
+
+ public void setType(TSEncoding type) {
+ this.type = type;
+ }
+
+ public TSEncoding getType() {
+ return type;
+ }
+
+ public Encoder(TSEncoding type) {
+ this.type = type;
+ }
+
+ public void encode(boolean value, ByteArrayOutputStream out) {
+ throw new TsFileEncodingException("Method encode boolean is not supported by Encoder");
+ }
+
+ public void encode(short value, ByteArrayOutputStream out) {
+ throw new TsFileEncodingException("Method encode short is not supported by Encoder");
+ }
+
+ public void encode(int value, ByteArrayOutputStream out) {
+ throw new TsFileEncodingException("Method encode int is not supported by Encoder");
+ }
+
+ public void encode(long value, ByteArrayOutputStream out) {
+ throw new TsFileEncodingException("Method encode long is not supported by Encoder");
+ }
+
+ public void encode(String value, ByteArrayOutputStream out) {
+ throw new TsFileEncodingException("Method encode text is not supported by Encoder");
+ }
+
+ ;
+
+ public void encode(float value, ByteArrayOutputStream out) {
+ throw new TsFileEncodingException("Method encode float is not supported by Encoder");
+ }
+
+ public void encode(double value, ByteArrayOutputStream out) {
+ throw new TsFileEncodingException("Method encode double is not supported by Encoder");
+ }
+
+ public void encode(Binary value, ByteArrayOutputStream out) {
+ throw new TsFileEncodingException("Method encode Binary is not supported by Encoder");
+ }
+
+ public void encode(BigDecimal value, ByteArrayOutputStream out) {
+ throw new TsFileEncodingException("Method encode BigDecimal is not supported by Encoder");
+ }
+
+ /**
+ * Write all values buffered in memory cache to OutputStream.
+ *
+ * @param out - ByteArrayOutputStream
+ * @throws IOException cannot flush to OutputStream
+ */
+ public abstract void flush(ByteArrayOutputStream out) throws IOException;
+
+ /**
+ * When encoder accepts a new incoming data point, the maximal possible size in byte it takes to
+ * store in memory.
+ *
+ * @return the maximal possible size of one data item encoded by this encoder
+ */
+ public int getOneItemMaxSize() {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * The maximal possible memory size occupied by current Encoder. This statistic value doesn't
+ * involve OutputStream.
+ *
+ * @return the maximal size of possible memory occupied by current encoder
+ */
+ public long getMaxByteSize() {
+ throw new UnsupportedOperationException();
+ }
+>>>>>>> Stashed changes
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/FloatEncoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/FloatEncoder.java
index 2d477cb936..4f5546601c 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/FloatEncoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/FloatEncoder.java
@@ -51,6 +51,7 @@ public class FloatEncoder extends Encoder {
public FloatEncoder(TSEncoding encodingType, TSDataType dataType, int maxPointNumber) {
super(encodingType);
+// System.out.println(maxPointNumber);
this.maxPointNumber = maxPointNumber;
calculateMaxPonitNum();
isMaxPointNumberSaved = false;
@@ -81,7 +82,9 @@ public class FloatEncoder extends Encoder {
@Override
public void encode(float value, ByteArrayOutputStream out) {
saveMaxPointNumber(out);
+// System.out.println(value);
int valueInt = convertFloatToInt(value);
+// System.out.println(valueInt);
encoder.encode(valueInt, out);
}
@@ -98,10 +101,12 @@ public class FloatEncoder extends Encoder {
maxPointValue = 1;
} else {
maxPointValue = Math.pow(10, maxPointNumber);
+ System.out.println(maxPointNumber);
}
}
private int convertFloatToInt(float value) {
+// Math.round(value * maxPointValue);
return (int) Math.round(value * maxPointValue);
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/HuffmanEncoderV2.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/HuffmanEncoderV2.java
new file mode 100644
index 0000000000..ad75f6359a
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/HuffmanEncoderV2.java
@@ -0,0 +1,257 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.encoder;
+
+import org.apache.iotdb.tsfile.encoding.HuffmanTree.HuffmanCode;
+import org.apache.iotdb.tsfile.encoding.HuffmanTree.HuffmanTree;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.utils.Binary;
+
+import java.io.ByteArrayOutputStream;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.PriorityQueue;
+
+public class HuffmanEncoderV2 extends Encoder {
+
+ private HuffmanTree[] byteFrequency;
+ private List<Integer> records;
+ private HuffmanCode[] huffmanCodes;
+ PriorityQueue<HuffmanTree> huffmanQueue;
+ private HuffmanTree treeTop;
+ private byte byteBuffer;
+ private int numberLeftInBuffer = 0;
+ private boolean[] used;
+ private int usednum;
+ private int maxRecordLength;
+ private int totLength;
+ public int recordnum;
+
+ private int symbolSetSize;
+
+ public HuffmanEncoderV2(int _symbolSetSize) {
+ super(TSEncoding.HUFFMANV2);
+ symbolSetSize = _symbolSetSize;
+ byteFrequency =
+ new HuffmanTree[_symbolSetSize]; // byteFrequency[256] is used to save the frequency of end-of-records
+ for (int i = 0; i < _symbolSetSize; i++)
+ byteFrequency[i] = new HuffmanTree();
+ records = new ArrayList<Integer>();
+ huffmanQueue = new PriorityQueue<HuffmanTree>(huffmanTreeComparator);
+ huffmanCodes = new HuffmanCode[_symbolSetSize];
+ for (int i = 0; i < _symbolSetSize; i++) {
+ huffmanCodes[i] = new HuffmanCode();
+ }
+ used = new boolean[_symbolSetSize];
+ treeTop = new HuffmanTree();
+ reset();
+ }
+
+ @Override
+ public void encode(int value, ByteArrayOutputStream out) {
+ recordnum++;
+ records.add(value);
+ byteFrequency[value].frequency++;
+ }
+
+ @Override
+ public void flush(ByteArrayOutputStream out) {
+ buildHuffmanTree();
+ double cnt = 0;
+ double cnt2 = 0;
+ int tot = 0;
+ for (int i = 0; i < symbolSetSize; i++) {
+ if (byteFrequency[i].frequency != 0)
+ tot += 1;
+ }
+ for (int i = 0; i < symbolSetSize; i++) {
+ if (byteFrequency[i].frequency != 0) {
+ double shang = - (Math.log((double) byteFrequency[i].frequency / recordnum) / Math.log(2));
+ cnt += Math.round(shang) * byteFrequency[i].frequency;
+ }
+ }
+ cnt /= 8;
+ // System.out.println(cnt);
+ List<Boolean> code = new ArrayList<>();
+ getHuffmanCode(treeTop, code);
+
+// for (int i = 0; i < symbolSetSize; i++) {
+// if (byteFrequency[i].frequency != 0) {
+// System.out.print(i);
+// System.out.print(": ");
+// for (Boolean b : huffmanCodes[i].huffmanCode)
+// System.out.print((b.toString()));
+// System.out.println("");
+// }
+// }
+
+ flushHeader(out);
+// System.out.println(" huffman heaeder:");
+// System.out.print(" ");
+// System.out.println(out.size());
+ int temp = out.size();
+ for (int i = 0; i < records.size(); i++) {
+ flushRecord(records.get(i), out);
+ }
+ reset();
+ clearBuffer(out);
+
+// System.out.println(" huffman body:");
+// System.out.print(" ");
+// System.out.println(out.size() - temp);
+ }
+
+ @Override
+ public int getOneItemMaxSize() {
+ return maxRecordLength;
+ }
+
+ @Override
+ public long getMaxByteSize() {
+ return totLength;
+ }
+
+ private void buildHuffmanTree() {
+ for (int i = 0; i < symbolSetSize; i++) {
+ if (byteFrequency[i].frequency != 0) {
+ huffmanQueue.add(byteFrequency[i]);
+ used[i] = true;
+ usednum++;
+ }
+ }
+ while (huffmanQueue.size() > 1) {
+ HuffmanTree cur = new HuffmanTree();
+ cur.leftNode = huffmanQueue.poll();
+ cur.rightNode = huffmanQueue.poll();
+ cur.frequency = cur.leftNode.frequency + cur.rightNode.frequency;
+ cur.isRecordEnd = false;
+ cur.isLeaf = false;
+ huffmanQueue.add(cur);
+ }
+ treeTop = huffmanQueue.poll();
+ }
+
+ private void getHuffmanCode(HuffmanTree cur, List<Boolean> code) {
+ if (cur.isLeaf) {
+ for (int i = 0; i < code.size(); i++) {
+ int idx = (int) cur.originalbyte;
+ if (idx < 0) idx += (1 << 8);
+ huffmanCodes[idx].huffmanCode.add(code.get(i));
+ }
+ return;
+ }
+ code.add(false);
+ getHuffmanCode(cur.leftNode, code);
+ code.remove(code.size() - 1);
+ code.add(true);
+ getHuffmanCode(cur.rightNode, code);
+ code.remove(code.size() - 1);
+ }
+
+ private void flushHeader(ByteArrayOutputStream out) {
+ writeInt(recordnum, out); // write the number of records
+ totLength += 4;
+ writeInt(usednum, out); // Write how many character have been used in this section
+ totLength += 4;
+ for (int i = 0; i < symbolSetSize; i++) {
+ if (used[i]) {
+ writeInt(i, out);
+ writeInt(
+ huffmanCodes[i].huffmanCode.size(), out); // First we store the length of huffman code
+ totLength += 8;
+ for (boolean b : huffmanCodes[i].huffmanCode) // Then we store the huffman code
+ writeBit(b, out);
+ }
+ }
+ }
+
+ private void flushRecord(Integer rec, ByteArrayOutputStream out) {
+ for (boolean b : huffmanCodes[rec].huffmanCode) {
+ writeBit(b, out);
+// if (b)
+// System.out.print(1);
+// else
+// System.out.print(0);
+ }
+// System.out.println("");
+ }
+
+ private void reset() {
+ for (int i = 0; i < symbolSetSize; i++) {
+ byteFrequency[i].frequency = 0;
+ byteFrequency[i].originalbyte = i;
+ byteFrequency[i].isLeaf = true;
+ byteFrequency[i].isRecordEnd = false;
+ huffmanCodes[i].huffmanCode.clear();
+ used[i] = false;
+ }
+ records.clear();
+ huffmanQueue.clear();
+ usednum = 0;
+ maxRecordLength = 0;
+ totLength = 0;
+ recordnum = 0;
+ treeTop.clear();
+ }
+
+ public static Comparator<HuffmanTree> huffmanTreeComparator =
+ new Comparator<HuffmanTree>() {
+ @Override
+ public int compare(HuffmanTree o1, HuffmanTree o2) {
+ return o1.frequency - o2.frequency;
+ }
+ };
+
+ protected void writeBit(boolean b, ByteArrayOutputStream out) {
+ byteBuffer <<= 1;
+ if (b) {
+ byteBuffer |= 1;
+ }
+
+ numberLeftInBuffer++;
+ if (numberLeftInBuffer == 8) {
+ clearBuffer(out);
+ }
+ }
+
+ protected void clearBuffer(ByteArrayOutputStream out) {
+ if (numberLeftInBuffer == 0) return;
+ if (numberLeftInBuffer > 0) byteBuffer <<= (8 - numberLeftInBuffer);
+ out.write(byteBuffer);
+ totLength++;
+ numberLeftInBuffer = 0;
+ byteBuffer = 0;
+ }
+
+ private void writeInt(int val, ByteArrayOutputStream out) {
+ for (int i = 31; i >= 0; i--) {
+ if ((val & (1 << i)) > 0) writeBit(true, out);
+ else writeBit(false, out);
+ }
+ }
+
+ private void writeByte(byte val, ByteArrayOutputStream out) {
+ for (int i = 7; i >= 0; i--) {
+ if ((val & (1 << i)) > 0) writeBit(true, out);
+ else writeBit(false, out);
+ }
+ }
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java
index 74f9c3c4c4..ba19062ff0 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java
@@ -70,6 +70,21 @@ public abstract class TSEncodingBuilder {
return new GorillaV2();
case DICTIONARY:
return new Dictionary();
+<<<<<<< Updated upstream
+=======
+ case RAKE:
+ return new Rake();
+ case SPRINTZ:
+ return new Sprintz();
+ case RLBE:
+ return new RLBE();
+ case TEXTRLE:
+ return new TEXTRLE();
+ case HUFFMAN:
+ return new HUFFMAN();
+ case BUCKET:
+ return new BucketEncoder();
+>>>>>>> Stashed changes
default:
throw new UnsupportedOperationException(type.toString());
}
@@ -140,7 +155,13 @@ public abstract class TSEncodingBuilder {
return new LongRleEncoder();
case FLOAT:
case DOUBLE:
+<<<<<<< Updated upstream
return new FloatEncoder(TSEncoding.RLE, type, maxPointNumber);
+=======
+ return new FloatEncoder(TSEncoding.RLE, type, 6);//maxPointNumber);
+ case TEXT:
+ return new TextRleEncoder();
+>>>>>>> Stashed changes
default:
throw new UnSupportedDataTypeException("RLE doesn't support data type: " + type);
}
@@ -182,7 +203,7 @@ public abstract class TSEncodingBuilder {
/** for INT32, INT64, FLOAT, DOUBLE. */
public static class Ts2Diff extends TSEncodingBuilder {
- private int maxPointNumber = 0;
+ private int maxPointNumber = 5;
@Override
public Encoder getEncoder(TSDataType type) {
@@ -193,6 +214,8 @@ public abstract class TSEncodingBuilder {
return new DeltaBinaryEncoder.LongDeltaEncoder();
case FLOAT:
case DOUBLE:
+// maxPointNumber
+// System.out.println(maxPointNumber);
return new FloatEncoder(TSEncoding.TS_2DIFF, type, maxPointNumber);
default:
throw new UnSupportedDataTypeException("TS_2DIFF doesn't support data type: " + type);
@@ -207,8 +230,10 @@ public abstract class TSEncodingBuilder {
public void initFromProps(Map<String, String> props) {
// set max error from initialized map or default value if not set
if (props == null || !props.containsKey(Encoder.MAX_POINT_NUMBER)) {
+ System.out.println(Encoder.MAX_POINT_NUMBER);
maxPointNumber = TSFileDescriptor.getInstance().getConfig().getFloatPrecision();
} else {
+ System.out.println(Encoder.MAX_POINT_NUMBER);
try {
this.maxPointNumber = Integer.parseInt(props.get(Encoder.MAX_POINT_NUMBER));
} catch (NumberFormatException e) {
@@ -314,4 +339,140 @@ public abstract class TSEncodingBuilder {
// do nothing
}
}
+<<<<<<< Updated upstream
+=======
+
+ public static class Rake extends TSEncodingBuilder {
+
+ @Override
+ public Encoder getEncoder(TSDataType type) {
+ switch (type) {
+ case FLOAT:
+ return new FloatRAKEEncoder();
+ case DOUBLE:
+ return new DoubleRAKEEncoder();
+ case INT32:
+ return new IntRAKEEncoder();
+ case INT64:
+ return new LongRAKEEncoder();
+ default:
+ throw new UnSupportedDataTypeException("Rake doesn't support data type: " + type);
+ }
+ }
+
+ @Override
+ public void initFromProps(Map<String, String> props) {
+ // do nothing
+ }
+ }
+
+ public static class Sprintz extends TSEncodingBuilder {
+ @Override
+ public Encoder getEncoder(TSDataType type) {
+ switch (type) {
+ case INT32:
+ return new IntSprintzEncoder();
+ case INT64:
+ return new LongSprintzEncoder();
+ case FLOAT:
+ return new FloatSprintzEncoder();
+ case DOUBLE:
+ return new DoubleSprintzEncoder();
+ default:
+ throw new UnSupportedDataTypeException("Sprintz doesn't support data type: " + type);
+ }
+ }
+
+ @Override
+ public void initFromProps(Map<String, String> props) {
+ // do nothing
+ }
+ }
+
+ public static class RLBE extends TSEncodingBuilder {
+ @Override
+ public Encoder getEncoder(TSDataType type) {
+ switch (type) {
+ case INT32:
+ return new IntRLBE();
+ case INT64:
+ return new LongRLBE();
+ case FLOAT:
+ return new FloatRLBE();
+ case DOUBLE:
+ return new DoubleRLBE();
+ default:
+ throw new UnSupportedDataTypeException("RLBE doesn't support data type: " + type);
+ }
+ }
+
+ @Override
+ public void initFromProps(Map<String, String> props) {
+ // do nothing
+ }
+ }
+
+ public static class HUFFMAN extends TSEncodingBuilder {
+ @Override
+ public Encoder getEncoder(TSDataType type) {
+ switch (type) {
+ case TEXT:
+ return new HuffmanEncoder();
+ case INT32:
+ case INT64:
+ case FLOAT:
+ case DOUBLE:
+ default:
+ throw new UnSupportedDataTypeException("HUFFMAN doesn't support data type: " + type);
+ }
+ }
+
+ @Override
+ public void initFromProps(Map<String, String> props) {
+ // do nothing
+ }
+ }
+
+ public static class TEXTRLE extends TSEncodingBuilder {
+ @Override
+ public Encoder getEncoder(TSDataType type) {
+ switch (type) {
+ case TEXT:
+ return new TextRleEncoder();
+ case INT32:
+ case INT64:
+ case FLOAT:
+ case DOUBLE:
+ default:
+ throw new UnSupportedDataTypeException("TEXTRLE doesn't support data type: " + type);
+ }
+ }
+
+ @Override
+ public void initFromProps(Map<String, String> props) {
+ // do nothing
+ }
+ }
+ public static class BucketEncoder extends TSEncodingBuilder {
+ @Override
+ public Encoder getEncoder(TSDataType type) {
+ switch (type) {
+ case INT32:
+ return new org.apache.iotdb.tsfile.encoding.encoder.BucketEncoder();
+ case FLOAT:
+ return new org.apache.iotdb.tsfile.encoding.encoder.BucketEncoder();
+ case TEXT:
+ case INT64:
+ case DOUBLE:
+ default:
+ throw new UnSupportedDataTypeException("TEXTRLE doesn't support data type: " + type);
+ }
+ }
+
+ @Override
+ public void initFromProps(Map<String, String> props) {
+ // do nothing
+ }
+ }
+>>>>>>> Stashed changes
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoderbuff/BuffEncoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoderbuff/BuffEncoder.java
new file mode 100644
index 0000000000..6a3e27778d
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoderbuff/BuffEncoder.java
@@ -0,0 +1,260 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.apache.iotdb.tsfile.encoding.encoderbuff;
+
+//import cn.edu.thu.diq.compression.BitConstructor;
+//import cn.edu.thu.diq.compression.BitReader;
+//import org.eclipse.collections.impl.list.mutable.primitive.DoubleArrayList;
+
+import jodd.util.collection.DoubleArrayList;
+import org.apache.iotdb.tsfile.encoding.BitConstructor;
+import org.apache.iotdb.tsfile.encoding.BitReader;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ *
+ * @author Wang Haoyu
+ */
+public class BuffEncoder implements Encoder {
+
+ private final double sparseThreshold = 0.9;
+ private final int blocksize = 1024;
+ private final int precision;
+ private final double eps;
+
+ public BuffEncoder(int precision) {
+ this.precision = Math.max(0, precision);
+ this.eps = Math.pow(2, -precision);
+ }
+
+ @Override
+ public byte[] encode(double[] origin) {
+ BitConstructor constructor = new BitConstructor();
+ for (int i = 0; i < origin.length; i += blocksize) {
+ int n = Math.min(origin.length, i + blocksize) - i;
+ encodeBlock(constructor, origin, i, i + n);
+ }
+ return constructor.toByteArray();
+ }
+
+ @Override
+ public double[] decode(byte[] bytes) {
+ BitReader reader = new BitReader(bytes);
+ DoubleArrayList decoded = new DoubleArrayList();
+ while (reader.hasNext()) {
+ decoded.addAll(decodeBlock(reader));
+ }
+ return decoded.toArray();
+ }
+
+ private void encodeBlock(BitConstructor constructor, double[] values, int from, int to) {
+ //获取整数部分的范围
+ int min = Integer.MAX_VALUE, max = Integer.MIN_VALUE;
+ for (int i = from; i < to; i++) {
+ min = Math.min(min, (int) Math.floor(values[i]));
+ max = Math.max(max, (int) Math.floor(values[i]));
+ }
+ //保存元数据
+ constructor.add(to - from, 32);
+// System.out.println(to - from);
+ constructor.add(this.precision, 32);
+ constructor.add(min, 32);
+ constructor.add(max, 32);
+ //浮点数转定点数
+ long[] fixed = new long[to - from];
+ for (int i = 0; i < fixed.length; i++) {
+ fixed[i] = (long) (Math.round((values[i + from] - min) / eps));
+ }
+ //按照子列分别进行存储
+ int[] masks = {0, 0x0001, 0x0003, 0x0007, 0x000f, 0x001f, 0x003f, 0x007f, 0x00ff};
+ int totalWidth = getValueWidth(max - min) + precision;
+ for (int i = totalWidth; i > 0; i -= 8) {
+ int shift = Math.max(0, i - 8), len = Math.min(8, i);
+ byte[] bytes = new byte[fixed.length];
+ for (int j = 0; j < fixed.length; j++) {
+ bytes[j] = (byte) ((fixed[j] >> shift) & masks[len]);
+ }
+ encodeSubColumn(constructor, bytes, len);
+ }
+ constructor.pad();
+ }
+
+ private double[] decodeBlock(BitReader reader) {
+ //读取元数据
+ int n = (int) reader.next(32);
+// System.out.println(n);
+ int p = (int) reader.next(32);
+ int min = (int) reader.next(32);
+ int max = (int) reader.next(32);
+ //读取子列
+ long[] fixed = new long[n];
+ int totalWidth = getValueWidth(max - min) + p;
+ for (int i = totalWidth; i > 0; i -= 8) {
+ int len = Math.min(8, i);
+ byte[] bytes = decodeSubColumn(reader, n, len);
+ for (int j = 0; j < n; j++) {
+ fixed[j] = (fixed[j] << len) | (bytes[j] & 0xff);
+ }
+ }
+ reader.skip();
+ //定点数转换为浮点数
+ double[] values = new double[n];
+ double eps1 = Math.pow(2, -p);
+ for (int i = 0; i < n; i++) {
+ values[i] = fixed[i] * eps1 + min;
+ }
+ return values;
+ }
+
+ private void encodeSubColumn(BitConstructor constructor, byte[] bytes, int len) {
+ //统计各种数值出现的次数，判断是否采用稀疏表示
+ Byte frequentValue = count(bytes, new HashMap<>());
+ if (frequentValue == null) {
+ constructor.add(0, 8);
+ encodeDenseSubColumn(constructor, bytes, len);
+ } else {
+ constructor.add(1, 8);
+ encodeSparseSubColumn(constructor, bytes, frequentValue, len);
+ }
+ }
+
+ private byte[] decodeSubColumn(BitReader reader, int n, int len) {
+ int sparseFlag = (int) reader.next(8);
+ switch (sparseFlag) {
+ case 0:
+ return decodeDenseSubColumn(reader, n, len);
+ case 1:
+ return decodeSparseSubColumn(reader, n, len);
+ default:
+ throw new RuntimeException("It cannot be reached.");
+ }
+ }
+
+ private void encodeDenseSubColumn(BitConstructor constructor, byte[] bytes, int len) {
+ //存储数据
+ for (byte b : bytes) {
+ constructor.add(b, len);
+ }
+ }
+
+ private byte[] decodeDenseSubColumn(BitReader reader, int n, int len) {
+ byte[] bytes = new byte[n];
+ for (int i = 0; i < n; i++) {
+ bytes[i] = (byte) reader.next(len);
+ }
+ return bytes;
+ }
+
+ private void encodeSparseSubColumn(BitConstructor constructor, byte[] bytes, byte frequentValue, int len) {
+ //存储众数
+ constructor.add(frequentValue, len);
+ //存储RLE压缩的比特向量
+ BitConstructor rle = new BitConstructor();
+ int cnt = encodeRLEVector(rle, bytes, frequentValue);
+ byte[] rleBytes = rle.toByteArray();
+ constructor.pad();
+ constructor.add(rleBytes.length, 32);
+ constructor.add(rleBytes);
+ //存储离群点
+ constructor.add(cnt, 32);
+ for (byte b : bytes) {
+ if (b != frequentValue) {
+ constructor.add(b, len);
+ }
+ }
+ }
+
+ private int encodeRLEVector(BitConstructor constructor, byte[] bytes, byte frequentValue) {
+ int width = getValueWidth(bytes.length);
+ boolean outlier = false;
+ int run = 0;
+ int cnt = 0;
+ for (byte b : bytes) {
+ if ((b != frequentValue) == outlier) {
+ run++;
+ } else {
+ outlier = !outlier;
+ constructor.add(run, width);
+ run = 1;
+ }
+ if (b != frequentValue) {
+ cnt++;
+ }
+ }
+ constructor.add(run, width);
+ return cnt;
+ }
+
+ private boolean[] decodeRLEVector(byte[] bytes, int n) {
+ BitReader reader = new BitReader(bytes);
+ boolean[] vector = new boolean[n];
+ int width = getValueWidth(n);
+ int i = 0;
+ boolean bit = false;
+ while (i < n) {
+ int run = (int) reader.next(width);
+ int j = i + run;
+ for (; i < j; i++) {
+ vector[i] = bit;
+ }
+ bit = !bit;
+ }
+ return vector;
+ }
+
+ private byte[] decodeSparseSubColumn(BitReader reader, int n, int len) {
+ //读取众数
+ byte frequentValue = (byte) reader.next(len);
+ //读取RLE比特向量
+ reader.skip();
+ int rleByteSize = (int) reader.next(32);
+ boolean vector[] = decodeRLEVector(reader.nextBytes(rleByteSize), n);
+ //读取离群点
+ int cnt = (int) reader.next(32);
+ byte[] bytes = new byte[n];
+ for (int i = 0; i < n; i++) {
+ if (vector[i]) {
+ bytes[i] = (byte) reader.next(len);
+ } else {
+ bytes[i] = frequentValue;
+ }
+ }
+ return bytes;
+ }
+
+ private Byte count(byte[] bytes, HashMap<Byte, Integer> map) {
+ for (byte x : bytes) {
+ map.put(x, map.getOrDefault(x, 0) + 1);
+ }
+ Byte maxByte = null;
+ int maxTimes = 0;
+ for (Map.Entry<Byte, Integer> entry : map.entrySet()) {
+ Byte key = entry.getKey();
+ Integer value = entry.getValue();
+ if (value > maxTimes) {
+ maxTimes = value;
+ maxByte = key;
+ }
+ }
+ if (maxTimes > sparseThreshold * bytes.length) {
+ return maxByte;
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * 计算x的数据宽度
+ *
+ * @param x
+ * @return 数据宽度
+ */
+ public int getValueWidth(long x) {
+ return 64 - Long.numberOfLeadingZeros(x);
+ }
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoderbuff/Encoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoderbuff/Encoder.java
new file mode 100644
index 0000000000..80503a061a
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoderbuff/Encoder.java
@@ -0,0 +1,17 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.apache.iotdb.tsfile.encoding.encoderbuff;
+
+/**
+ *
+ * @author Wang
+ */
+public interface Encoder {
+
+ public byte[] encode(double[] origin);
+
+ public double[] decode(byte[] bytes);
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/CompressionType.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/CompressionType.java
index 85893e7bd1..ee0c2bcc54 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/CompressionType.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/CompressionType.java
@@ -41,7 +41,10 @@ public enum CompressionType {
PLA(".pla", (byte) 6),
/** LZ4 */
- LZ4(".lz4", (byte) 7);
+ LZ4(".lz4", (byte) 7),
+
+ /** LZMA2 */
+ LZMA2(".lzma2", (byte) 8);
private final String extensionName;
private final byte index;
@@ -75,6 +78,8 @@ public enum CompressionType {
return CompressionType.PLA;
case 7:
return CompressionType.LZ4;
+ case 8:
+ return CompressionType.LZMA2;
default:
throw new IllegalArgumentException("Invalid input: " + compressor);
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/TSEncoding.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/TSEncoding.java
index de95facbfd..bb79319e46 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/TSEncoding.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/TSEncoding.java
@@ -27,7 +27,18 @@ public enum TSEncoding {
BITMAP((byte) 5),
GORILLA_V1((byte) 6),
REGULAR((byte) 7),
+<<<<<<< Updated upstream
GORILLA((byte) 8);
+=======
+ GORILLA((byte) 8),
+ SPRINTZ((byte) 9),
+ RAKE((byte) 10),
+ RLBE((byte) 11),
+ TEXTRLE((byte) 12),
+ HUFFMAN((byte) 13),
+ HUFFMANV2((byte) 14),
+ BUCKET((byte) 15);
+>>>>>>> Stashed changes
private final byte type;
@@ -65,6 +76,23 @@ public enum TSEncoding {
return TSEncoding.REGULAR;
case 8:
return TSEncoding.GORILLA;
+<<<<<<< Updated upstream
+=======
+ case 9:
+ return TSEncoding.SPRINTZ;
+ case 10:
+ return TSEncoding.RAKE;
+ case 11:
+ return TSEncoding.RLBE;
+ case 12:
+ return TSEncoding.TEXTRLE;
+ case 13:
+ return TSEncoding.HUFFMAN;
+ case 14:
+ return TSEncoding.HUFFMANV2;
+ case 15:
+ return TSEncoding.BUCKET;
+>>>>>>> Stashed changes
default:
throw new IllegalArgumentException("Invalid input: " + encoding);
}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/compress/CompressTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/compress/CompressTest.java
index d3d0406d41..5cbf80ac00 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/compress/CompressTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/compress/CompressTest.java
@@ -103,4 +103,16 @@ public class CompressTest {
String result = new String(uncompressed, StandardCharsets.UTF_8);
assertEquals(inputString, result);
}
+
+ @Test
+ public void lzma2CompressorTest() throws IOException {
+ PublicBAOS out = new PublicBAOS();
+ out.write(inputString.getBytes(StandardCharsets.UTF_8));
+ ICompressor compressor = new ICompressor.LZMA2Compressor();
+ IUnCompressor unCompressor = new IUnCompressor.LZMA2UnCompressor();
+ byte[] compressed = compressor.compress(out.getBuf());
+ byte[] uncompressed = unCompressor.uncompress(compressed);
+ String result = new String(uncompressed, StandardCharsets.UTF_8);
+ assertEquals(inputString, result);
+ }
}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/BucketDecoderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/BucketDecoderTest.java
new file mode 100644
index 0000000000..a0ef966ba5
--- /dev/null
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/BucketDecoderTest.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import org.apache.iotdb.tsfile.encoding.encoder.BucketEncoder;
+import org.apache.iotdb.tsfile.encoding.encoder.HuffmanEncoderV2;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.nio.ByteBuffer;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class BucketDecoderTest {
+ private BucketEncoder encoder = new BucketEncoder();
+ private BucketDecoder decoder = new BucketDecoder();
+ private ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ @Test
+ public void testSingle() {
+ int a[] = {1};
+ testAll(a);
+ a[0] = 2;
+ testAll(a);
+ a[0] = 1023;
+ testAll(a);
+ }
+
+ @Test
+ public void testAllUnique() {
+ int a[] = {1, 2, 3};
+ int b[] = {52, 123, 432};
+ int c[] = {54, 76, 42, 27, 35};
+ testAll(a);
+ testAll(b);
+ testAll(c);
+ }
+
+ @Test
+ public void testAllSame() {
+ int a[] = {20, 20, 20};
+ int b[] = {166, 166, 166};
+ testAll(a);
+ testAll(b);
+ }
+
+ @Test
+ public void testMixed() {
+ // all characters
+ int[] allChars = new int[1000000];
+ for (int i = 0; i < 1000000; i++) {
+ allChars[i] = (int)(1000000 * Math.random());
+ }
+ testAll(allChars);
+ }
+
+ private void testAll(int[] all) {
+ for (int s : all) {
+ encoder.encode(s, baos);
+ }
+ encoder.flush(baos);
+
+ ByteBuffer out = ByteBuffer.wrap(baos.toByteArray());
+
+ for (int s : all) {
+ assertTrue(decoder.hasNext(out));
+ int b = decoder.readInt(out);
+ assertEquals(s, b);
+ }
+
+ decoder.reset();
+ baos.reset();
+ }
+}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/ContestEncodeTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/ContestEncodeTest.java
new file mode 100644
index 0000000000..a6068eb6df
--- /dev/null
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/ContestEncodeTest.java
@@ -0,0 +1,328 @@
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import org.apache.iotdb.tsfile.compress.ICompressor;
+import org.apache.iotdb.tsfile.compress.IUnCompressor;
+import org.apache.iotdb.tsfile.encoding.encoder.Encoder;
+import org.apache.iotdb.tsfile.encoding.encoder.TSEncodingBuilder;
+import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+import java.io.*;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+
+public class ContestEncodeTest {
+
+ public static void main(@org.jetbrains.annotations.NotNull String[] args) throws IOException {
+ String inputPath = "C:\\Users\\xiaoj\\Desktop\\float",
+ Output = "C:\\Users\\xiaoj\\Desktop\\compressedResult.csv";
+ if (args.length >= 2) inputPath = args[1];
+ if (args.length >= 3) Output = args[2];
+ long s = System.nanoTime();
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+ TSEncoding[] schemeList = {
+// TSEncoding.PLAIN,
+ TSEncoding.TS_2DIFF
+// TSEncoding.RLE,
+// TSEncoding.SPRINTZ, TSEncoding.GORILLA, TSEncoding.RLBE, TSEncoding.RAKE
+ };
+ // CompressionType[] compressList = {CompressionType.LZ4, CompressionType.GZIP,
+ // CompressionType.SNAPPY};
+ CompressionType[] compressList = {CompressionType.UNCOMPRESSED}; //
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Encoding",
+ "Compress",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ assert tempList != null;
+ double ratio = 0;
+ for (File f : tempList) {
+// System.out.println(f);
+ InputStream inputStream = new FileInputStream(f);
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ String fileName = f.getAbsolutePath();
+ ArrayList<String> data = new ArrayList<>();
+ TSDataType dataType = TSDataType.INT32;
+
+ loader.readHeaders();
+ while (loader.readRecord()) {
+ data.add(loader.getValues()[1]);
+ }
+ loader.close();
+ inputStream.close();
+ if (fileName.contains("float")) {
+ dataType = TSDataType.FLOAT;
+ ArrayList<Float> tmp = new ArrayList<>();
+ for (String value : data) {
+ tmp.add(Float.valueOf(value));
+ }
+ for (TSEncoding scheme : schemeList) {
+ Encoder encoder = TSEncodingBuilder.getEncodingBuilder(scheme).getEncoder(dataType);
+ Decoder decoder = Decoder.getDecoderByType(scheme, dataType);
+ for (CompressionType comp : compressList) {
+ ICompressor compressor = ICompressor.getCompressor(comp);
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(comp);
+
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+ for (float val : tmp) encoder.encode(val, buffer);
+ encoder.flush(buffer);
+ byte[] elems = buffer.toByteArray();
+ byte[] compressed = compressor.compress(elems);
+ float ratioTmp = (float) compressed.length / (float) (tmp.size() * Long.BYTES);
+ ratio += ratioTmp;
+ buffer.close();
+ System.out.println(scheme);
+
+ }
+ }
+
+ }
+ }
+ String[] record = {
+ "TS_2DIFF",
+ "GZIP",
+ String.valueOf(ratio/71)
+ };
+ writer.writeRecord(record);
+ writer.close();
+ long e = System.nanoTime();
+ long encodeTime = (e - s);
+ System.out.println("encodeTime:");
+ System.out.println(encodeTime);
+ }
+}
+// if (fileName.contains("int")) {
+// dataType = TSDataType.INT32;
+// ArrayList<Integer> tmp = new ArrayList<>();
+// for (String value : data) {
+// tmp.add(Integer.valueOf(value));
+// }
+// for (TSEncoding scheme : schemeList) {
+// Encoder encoder = TSEncodingBuilder.getEncodingBuilder(scheme).getEncoder(dataType);
+// Decoder decoder = Decoder.getDecoderByType(scheme, dataType);
+// long encodeTime = 0;
+// long decodeTime = 0;
+// for (CompressionType comp : compressList) {
+// ICompressor compressor = ICompressor.getCompressor(comp);
+// IUnCompressor unCompressor = IUnCompressor.getUnCompressor(comp);
+// long compressTime = 0;
+// long uncompressTime = 0;
+// double ratio = 0;
+// for (int i = 0; i < repeatTime; i++) {
+// ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+// long s = System.nanoTime();
+// for (int val : tmp) encoder.encode(val, buffer);
+// long e = System.nanoTime();
+// encodeTime += (e - s);
+// System.out.println("encodeTime:");
+// System.out.println(e - s);
+// encoder.flush(buffer);
+//
+// byte[] elems = buffer.toByteArray();
+// s = System.nanoTime();
+// byte[] compressed = compressor.compress(elems);
+// e = System.nanoTime();
+// compressTime += (e - s);
+//
+// int ratioTmp = compressed.length / (tmp.size() * Long.BYTES);
+// ratio += ratioTmp;
+//
+// s = System.nanoTime();
+// byte[] x = unCompressor.uncompress(compressed);
+// e = System.nanoTime();
+// uncompressTime += (e - s);
+//
+// ByteBuffer ebuffer = ByteBuffer.wrap(buffer.toByteArray());
+// s = System.nanoTime();
+// while (decoder.hasNext(ebuffer)) {
+// decoder.readInt(ebuffer);
+// }
+// e = System.nanoTime();
+// decodeTime += (e - s);
+// buffer.close();
+// }
+// ratio /= repeatTime;
+// encodeTime = encodeTime / ((long) repeatTime);
+// decodeTime = decodeTime / ((long) repeatTime);
+// // compressTime = compressTime / ((long) repeatTime);
+// // uncompressTime = uncompressTime / ((long) repeatTime);
+// System.out.println(scheme.toString());
+// // System.out.println(String.valueOf(encodeTime));
+// String[] record = {
+// scheme.toString(),
+// comp.toString(),
+// String.valueOf(encodeTime),
+// "",
+// "",
+// String.valueOf(decodeTime),
+// String.valueOf(ratio)
+// };
+//
+// // String[] record = {scheme.toString(), comp.toString(),
+// // String.valueOf(encodeTime), String.valueOf(compressTime),
+// // String.valueOf(uncompressTime),
+// // String.valueOf(decodeTime), String.valueOf(ratio)};
+// writer.writeRecord(record);
+// }
+// }
+// }
+// else if (fileName.contains("long")) {
+// dataType = TSDataType.INT64;
+// ArrayList<Long> tmp = new ArrayList<>();
+// for (String value : data) {
+// tmp.add(Long.valueOf(value));
+// }
+// for (TSEncoding scheme : schemeList) {
+// Encoder encoder = TSEncodingBuilder.getEncodingBuilder(scheme).getEncoder(dataType);
+// Decoder decoder = Decoder.getDecoderByType(scheme, dataType);
+// long encodeTime = 0;
+// long decodeTime = 0;
+// for (CompressionType comp : compressList) {
+// ICompressor compressor = ICompressor.getCompressor(comp);
+// IUnCompressor unCompressor = IUnCompressor.getUnCompressor(comp);
+// long compressTime = 0;
+// long uncompressTime = 0;
+// double ratio = 0;
+// for (int i = 0; i < repeatTime; i++) {
+// ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+// long s = System.nanoTime();
+// for (long val : tmp) encoder.encode(val, buffer);
+// long e = System.nanoTime();
+// encodeTime += (e - s);
+// System.out.println("encodeTime:");
+// System.out.println(e - s);
+// encoder.flush(buffer);
+//
+// byte[] elems = buffer.toByteArray();
+// s = System.nanoTime();
+// byte[] compressed = compressor.compress(elems);
+// e = System.nanoTime();
+// compressTime += (e - s);
+//
+// int ratioTmp = compressed.length / (tmp.size() * Long.BYTES);
+// ratio += ratioTmp;
+//
+// s = System.nanoTime();
+// byte[] x = unCompressor.uncompress(compressed);
+// e = System.nanoTime();
+// uncompressTime += (e - s);
+//
+// ByteBuffer ebuffer = ByteBuffer.wrap(buffer.toByteArray());
+// s = System.nanoTime();
+// while (decoder.hasNext(ebuffer)) {
+// decoder.readLong(ebuffer);
+// }
+// e = System.nanoTime();
+// decodeTime += (e - s);
+// buffer.close();
+// }
+// ratio /= repeatTime;
+// encodeTime = encodeTime / ((long) repeatTime);
+// decodeTime = decodeTime / ((long) repeatTime);
+// // compressTime = compressTime / ((long) repeatTime);
+// // uncompressTime = uncompressTime / ((long) repeatTime);
+// System.out.println(scheme.toString());
+// // System.out.println(String.valueOf(encodeTime));
+// String[] record = {
+// scheme.toString(),
+// comp.toString(),
+// String.valueOf(encodeTime),
+// "",
+// "",
+// String.valueOf(decodeTime),
+// String.valueOf(ratio)
+// };
+//
+// // String[] record = {scheme.toString(), comp.toString(),
+// // String.valueOf(encodeTime), String.valueOf(compressTime),
+// // String.valueOf(uncompressTime),
+// // String.valueOf(decodeTime), String.valueOf(ratio)};
+// writer.writeRecord(record);
+// }
+// }
+// }
+// else if (fileName.contains("double")) {
+// dataType = TSDataType.DOUBLE;
+// ArrayList<Double> tmp = new ArrayList<>();
+// for (String value : data) {
+// tmp.add(Double.valueOf(value));
+// }
+// for (TSEncoding scheme : schemeList) {
+// Encoder encoder = TSEncodingBuilder.getEncodingBuilder(scheme).getEncoder(dataType);
+// Decoder decoder = Decoder.getDecoderByType(scheme, dataType);
+// long encodeTime = 0;
+// long decodeTime = 0;
+// for (CompressionType comp : compressList) {
+// ICompressor compressor = ICompressor.getCompressor(comp);
+// IUnCompressor unCompressor = IUnCompressor.getUnCompressor(comp);
+// long compressTime = 0;
+// long uncompressTime = 0;
+// double ratio = 0;
+// for (int i = 0; i < repeatTime; i++) {
+// ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+// long s = System.nanoTime();
+// for (double val : tmp) encoder.encode(val, buffer);
+// long e = System.nanoTime();
+// encodeTime += (e - s);
+// System.out.println("encodeTime:");
+// System.out.println(e - s);
+// encoder.flush(buffer);
+//
+// byte[] elems = buffer.toByteArray();
+// s = System.nanoTime();
+// byte[] compressed = compressor.compress(elems);
+// e = System.nanoTime();
+// compressTime += (e - s);
+//
+// int ratioTmp = compressed.length / (tmp.size() * Long.BYTES);
+// ratio += ratioTmp;
+//
+// s = System.nanoTime();
+// byte[] x = unCompressor.uncompress(compressed);
+// e = System.nanoTime();
+// uncompressTime += (e - s);
+//
+// ByteBuffer ebuffer = ByteBuffer.wrap(buffer.toByteArray());
+// s = System.nanoTime();
+// while (decoder.hasNext(ebuffer)) {
+// decoder.readDouble(ebuffer);
+// }
+// e = System.nanoTime();
+// decodeTime += (e - s);
+// buffer.close();
+// }
+// ratio /= repeatTime;
+// encodeTime = encodeTime / ((long) repeatTime);
+// decodeTime = decodeTime / ((long) repeatTime);
+// // compressTime = compressTime / ((long) repeatTime);
+// // uncompressTime = uncompressTime / ((long) repeatTime);
+// System.out.println(scheme.toString());
+// // System.out.println(String.valueOf(encodeTime));
+// String[] record = {
+// scheme.toString(),
+// comp.toString(),
+// String.valueOf(encodeTime),
+// "",
+// "",
+// String.valueOf(decodeTime),
+// String.valueOf(ratio)
+// };
+//
+// // String[] record = {scheme.toString(), comp.toString(),
+// // String.valueOf(encodeTime), String.valueOf(compressTime),
+// // String.valueOf(uncompressTime),
+// // String.valueOf(decodeTime), String.valueOf(ratio)};
+// writer.writeRecord(record);
+// }
+// }
+// }
\ No newline at end of file
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/EncodeTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/EncodeTest.java
new file mode 100644
index 0000000000..69c0facf2e
--- /dev/null
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/EncodeTest.java
@@ -0,0 +1,453 @@
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import org.apache.iotdb.tsfile.compress.ICompressor;
+import org.apache.iotdb.tsfile.compress.IUnCompressor;
+import org.apache.iotdb.tsfile.encoding.encoder.*;
+import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.encoding.DST.DST;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.apache.iotdb.tsfile.utils.Binary;
+
+import java.io.*;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+
+public class EncodeTest {
+
+ public static void main(@org.jetbrains.annotations.NotNull String[] args) throws IOException {
+ String inputPath = "C:\\Users\\xiaoj\\Desktop\\bufftest"; // the direction of input compressed data
+ String Output = "C:\\Users\\xiaoj\\Desktop\\enocderle.csv"; // the direction of output compression ratio and speed
+ int repeatTime = 1; // set repeat time
+ String dataTypeName = "double"; // set dataType
+// if (args.length >= 2) inputPath = args[1];
+// if (args.length >= 3) Output = args[2];
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ // select encoding algorithms
+ TSEncoding[] encodingList = {
+// TSEncoding.PLAIN ,
+// TSEncoding.TS_2DIFF,
+ TSEncoding.RLE//,
+// TSEncoding.SPRINTZ,
+// TSEncoding.GORILLA,
+// TSEncoding.RLBE,
+// TSEncoding.RAKE
+ };
+ // select compression algorithms
+ CompressionType[] compressList = {
+ CompressionType.UNCOMPRESSED,
+// CompressionType.LZ4,
+// CompressionType.GZIP,
+// CompressionType.SNAPPY
+ };
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Column Index",
+ "Encoding Algorithm",
+ "Compress Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Compress Time",
+ "Uncompress Time",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+ int fileRepeat = 0;
+ int[] columnIndexes = new int[71]; // set the column indexes of compressed
+
+ for(int i=0;i<71;i++) columnIndexes[i] = i;
+ for (File f : tempList) {
+ fileRepeat += 1;
+ InputStream inputStream = new FileInputStream(f);
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ String fileName = f.getAbsolutePath();
+ ArrayList<String> data = new ArrayList<>();
+
+ for(int index :columnIndexes){
+ // add a column to "data"
+ System.out.println(index);
+ loader.readHeaders();
+ while (loader.readRecord()) {
+ data.add(loader.getValues()[index]);
+ }
+// loader.close();
+ inputStream.close();
+
+
+ switch (dataTypeName) {
+ case "int": {
+ TSDataType dataType = TSDataType.INT32; // set TSDataType
+ ArrayList<Integer> tmp = new ArrayList<>();
+ for (String value : data) {
+ tmp.add(Integer.valueOf(value));
+ }
+ // Iterate over each encoding algorithm
+ for (TSEncoding encoding : encodingList) {
+ Encoder encoder = TSEncodingBuilder.getEncodingBuilder(encoding).getEncoder(dataType);
+ Decoder decoder = Decoder.getDecoderByType(encoding, dataType);
+ long encodeTime = 0;
+ long decodeTime = 0;
+
+ // Iterate over each compression algorithm
+ for (CompressionType comp : compressList) {
+ ICompressor compressor = ICompressor.getCompressor(comp);
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(comp);
+
+ double ratio = 0;
+ double compressed_size = 0;
+
+ long compressTime = 0;
+ long uncompressTime = 0;
+
+ // repeat many times to test time
+ for (int i = 0; i < repeatTime; i++) {
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+
+ // test encode time
+ long s = System.nanoTime();
+ for (int val : tmp) encoder.encode(val, buffer);
+ encoder.flush(buffer);
+ long e = System.nanoTime();
+ encodeTime += (e - s);
+
+ // test compress time
+ byte[] elems = buffer.toByteArray();
+ s = System.nanoTime();
+ byte[] compressed = compressor.compress(elems);
+ e = System.nanoTime();
+ compressTime += (e - s);
+
+ // test compression ratio and compressed size
+ compressed_size += compressed.length;
+ double ratioTmp = (double) compressed.length / (double) (tmp.size()* Integer.BYTES);
+ ratio += ratioTmp;
+
+ // test uncompress time
+ s = System.nanoTime();
+ byte[] x = unCompressor.uncompress(compressed);
+ e = System.nanoTime();
+ uncompressTime += (e - s);
+
+ // test decode time
+ ByteBuffer ebuffer = ByteBuffer.wrap(buffer.toByteArray());
+ s = System.nanoTime();
+ while (decoder.hasNext(ebuffer)) {
+ decoder.readInt(ebuffer);
+ }
+ e = System.nanoTime();
+ decodeTime += (e - s);
+
+ buffer.close();
+ }
+
+ ratio /= repeatTime;
+ compressed_size /= repeatTime;
+ encodeTime /= repeatTime;
+ decodeTime /= repeatTime;
+ compressTime /= repeatTime;
+ uncompressTime /= repeatTime;
+
+ String[] record = {
+ f.toString(),
+ String.valueOf(index),
+ encoding.toString(),
+ comp.toString(),
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(compressTime),
+ String.valueOf(uncompressTime),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ }
+ }
+ break;
+ }
+ case "long": {
+ TSDataType dataType = TSDataType.INT64;
+ ArrayList<Long> tmp = new ArrayList<>();
+ for (String value : data) {
+ tmp.add(Long.valueOf(value));
+ }
+ // Iterate over each encoding algorithm
+ for (TSEncoding encoding : encodingList) {
+ Encoder encoder = TSEncodingBuilder.getEncodingBuilder(encoding).getEncoder(dataType);
+ Decoder decoder = Decoder.getDecoderByType(encoding, dataType);
+ long encodeTime = 0;
+ long decodeTime = 0;
+
+ // Iterate over each compression algorithm
+ for (CompressionType comp : compressList) {
+ ICompressor compressor = ICompressor.getCompressor(comp);
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(comp);
+ double ratio = 0;
+ double compressed_size = 0;
+
+ long compressTime = 0;
+ long uncompressTime = 0;
+ for (int i = 0; i < repeatTime; i++) {
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+
+ // test encode time
+ long s = System.nanoTime();
+ for (long val : tmp) encoder.encode(val, buffer);
+ encoder.flush(buffer);
+ long e = System.nanoTime();
+ encodeTime += (e - s);
+
+ // test compress time
+ byte[] elems = buffer.toByteArray();
+ s = System.nanoTime();
+ byte[] compressed = compressor.compress(elems);
+ e = System.nanoTime();
+ compressTime += (e - s);
+
+ // test compression ratio and compressed size
+ compressed_size = compressed.length;
+ double ratioTmp = (double) compressed.length / (double) (tmp.size() * Long.BYTES);
+ ratio += ratioTmp;
+
+ // test uncompress time
+ s = System.nanoTime();
+ byte[] x = unCompressor.uncompress(compressed);
+ e = System.nanoTime();
+ uncompressTime += (e - s);
+
+ // test decode time
+ ByteBuffer ebuffer = ByteBuffer.wrap(buffer.toByteArray());
+ s = System.nanoTime();
+ while (decoder.hasNext(ebuffer)) {
+ decoder.readInt(ebuffer);
+ }
+ e = System.nanoTime();
+ decodeTime += (e - s);
+
+ buffer.close();
+ }
+
+ ratio /= repeatTime;
+ compressed_size /= repeatTime;
+ encodeTime /= repeatTime;
+ decodeTime /= repeatTime;
+ compressTime /= repeatTime;
+ uncompressTime /= repeatTime;
+
+ // write info to file
+ String[] record = {
+ f.toString(),
+ String.valueOf(index),
+ encoding.toString(),
+ comp.toString(),
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(compressTime),
+ String.valueOf(uncompressTime),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+
+ }
+ }
+ break;
+ }
+ case "double": {
+ TSDataType dataType = TSDataType.DOUBLE;
+ ArrayList<Double> tmp = new ArrayList<>();
+ for (String value : data) {
+ tmp.add(Double.valueOf(value));
+ }
+ // Iterate over each encoding algorithm
+ for (TSEncoding encoding : encodingList) {
+ Encoder encoder = TSEncodingBuilder.getEncodingBuilder(encoding).getEncoder(dataType);
+ Decoder decoder = Decoder.getDecoderByType(encoding, dataType);
+ long encodeTime = 0;
+ long decodeTime = 0;
+
+ // Iterate over each compression algorithm
+ for (CompressionType comp : compressList) {
+ ICompressor compressor = ICompressor.getCompressor(comp);
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(comp);
+ long compressTime = 0;
+ long uncompressTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ // repeat many times to test time
+ for (int i = 0; i < repeatTime; i++) {
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+
+ // test encode time
+ long s = System.nanoTime();
+ for (double val : tmp) encoder.encode(val, buffer);
+ encoder.flush(buffer);
+ long e = System.nanoTime();
+ encodeTime += (e - s);
+
+ // test compress time
+ byte[] elems = buffer.toByteArray();
+ s = System.nanoTime();
+ byte[] compressed = compressor.compress(elems);
+ e = System.nanoTime();
+ compressTime += (e - s);
+
+ // test compression ratio and compressed size
+ compressed_size = compressed.length;
+ double ratioTmp = (double) compressed.length / (double)(tmp.size() * Double.BYTES);
+ ratio += ratioTmp;
+
+ // test uncompress time
+ s = System.nanoTime();
+ byte[] x = unCompressor.uncompress(compressed);
+ e = System.nanoTime();
+ uncompressTime += (e - s);
+
+ // test decode time
+ ByteBuffer ebuffer = ByteBuffer.wrap(buffer.toByteArray());
+ s = System.nanoTime();
+ while (decoder.hasNext(ebuffer)) {
+ decoder.readDouble(ebuffer);
+ }
+ e = System.nanoTime();
+ decodeTime += (e - s);
+
+ buffer.close();
+ }
+
+ ratio /= repeatTime;
+ compressed_size /= repeatTime;
+ encodeTime /= repeatTime;
+ decodeTime /= repeatTime;
+ compressTime /= repeatTime;
+ uncompressTime /= repeatTime;
+
+ // write info to file
+ String[] record = {
+ f.toString(),
+ String.valueOf(index),
+ encoding.toString(),
+ comp.toString(),
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(compressTime),
+ String.valueOf(uncompressTime),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+
+ }
+ }
+ break;
+ }
+ case "float": {
+ TSDataType dataType = TSDataType.FLOAT;
+ ArrayList<Float> tmp = new ArrayList<>();
+ for (String value : data) {
+ tmp.add(Float.valueOf(value));
+ }
+
+ // Iterate over each encoding algorithm
+ for (TSEncoding encoding : encodingList) {
+ Encoder encoder = TSEncodingBuilder.getEncodingBuilder(encoding).getEncoder(dataType);
+ Decoder decoder = Decoder.getDecoderByType(encoding, dataType);
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ // Iterate over each compression algorithm
+ for (CompressionType comp : compressList) {
+ ICompressor compressor = ICompressor.getCompressor(comp);
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(comp);
+ long compressTime = 0;
+ long uncompressTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ // repeat many times to test time
+ for (int i = 0; i < repeatTime; i++) {
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+
+ // test encode time
+ long s = System.nanoTime();
+ for (float val : tmp) {
+ encoder.encode(val, buffer);
+ }
+ encoder.flush(buffer);
+ long e = System.nanoTime();
+ encodeTime += (e - s);
+
+ // test compress time
+ byte[] elems = buffer.toByteArray();
+ s = System.nanoTime();
+ byte[] compressed = compressor.compress(elems);
+ e = System.nanoTime();
+ compressTime += (e - s);
+
+ // test compression ratio and compressed size
+ compressed_size += compressed.length;
+ double ratioTmp =(double) compressed.length / (double)(tmp.size() * Float.BYTES);
+ ratio += ratioTmp;
+
+ // test uncompress time
+ s = System.nanoTime();
+ byte[] x = unCompressor.uncompress(compressed);
+ e = System.nanoTime();
+ uncompressTime += (e - s);
+
+ // test decode time
+ ByteBuffer ebuffer = ByteBuffer.wrap(buffer.toByteArray());
+ while (decoder.hasNext(ebuffer)) {
+ decoder.readFloat(ebuffer);
+ }
+ e = System.nanoTime();
+ decodeTime += (e - s);
+
+ buffer.close();
+ }
+ ratio /= repeatTime;
+ compressed_size /= repeatTime;
+ encodeTime /= repeatTime;
+ decodeTime /= repeatTime;
+ compressTime /= repeatTime;
+ uncompressTime /= repeatTime;
+
+ // write info to file
+ String[] record = {
+ f.toString(),
+ String.valueOf(index),
+ encoding.toString(),
+ comp.toString(),
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(compressTime),
+ String.valueOf(uncompressTime),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ }
+ }
+ break;
+ }
+ }
+
+ if (fileRepeat > repeatTime) break;
+ }
+
+ }
+ writer.close();
+ }
+}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/EncodeTestBuff.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/EncodeTestBuff.java
new file mode 100644
index 0000000000..e807a1add5
--- /dev/null
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/EncodeTestBuff.java
@@ -0,0 +1,72 @@
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+//import cn.edu.thu.diq.compression.encoderbuff.BuffEncoder;
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.apache.iotdb.tsfile.compress.ICompressor;
+import org.apache.iotdb.tsfile.encoding.encoderbuff.BuffEncoder;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+
+public class EncodeTestBuff {
+
+ public static void main(@org.jetbrains.annotations.NotNull String[] args) throws IOException {
+ BuffEncoder buffEncoder = new BuffEncoder(18);
+ String inputPath = "C:\\Users\\xiaoj\\Desktop\\bufftest";
+ String Output = "C:\\Users\\xiaoj\\Desktop\\compressedResultBUFF.csv"; // the direction of output compression ratio and speed
+
+
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+ String[] head = {"compressed size","encoded size"};
+ writer.writeRecord(head); // write header to output file
+
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+ for(int columnIndex=0;columnIndex<71;columnIndex++){
+ assert tempList != null;
+ for (File f : tempList) {
+ InputStream inputStream = new FileInputStream(f);
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ String fileName = f.getAbsolutePath();
+ ArrayList<String> data = new ArrayList<>();
+ loader.readHeaders();
+ while (loader.readRecord()) {
+ data.add(loader.getValues()[columnIndex]);
+ }
+ loader.close();
+ inputStream.close();
+ int len = data.size();
+ double[] tmp = new double[len];
+ for (int i=0;i<len;i++) {
+ tmp[i] = Double.parseDouble(data.get(i));
+ }
+ System.out.println(len);
+ // System.out.println(tmp[0]);
+ // System.out.println(tmp[1]);
+ // System.out.println(tmp[2]);
+ byte[] encodedBytes = buffEncoder.encode(tmp);
+ ICompressor.GZIPCompressor compressor = new ICompressor.GZIPCompressor();
+ System.out.println(encodedBytes.length);
+ byte[] compressed = compressor.compress(encodedBytes);
+ System.out.println(compressed.length);
+
+ // double[] decoded = buffEncoder.decode(encodedBytes);
+ // for (int i=0;i<len;i++) {
+ // String a_str = String.format("%.5f", decoded[i]);
+ // if(tmp[i] != Double.parseDouble(a_str)) System.out.println(i);
+ // }
+ String[] record = {String.valueOf(compressed.length),String.valueOf(encodedBytes.length)};
+ writer.writeRecord(record);
+ }
+
+ }
+
+ writer.close();
+ }
+}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/FloatDecoderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/FloatDecoderTest.java
index cbbdc8e6ef..4f5c524746 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/FloatDecoderTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/FloatDecoderTest.java
@@ -40,8 +40,8 @@ public class FloatDecoderTest {
private static final Logger logger = LoggerFactory.getLogger(FloatDecoderTest.class);
private final double delta = 0.0000001;
- private final int floatMaxPointValue = 10000;
- private final int floatMaxPointNumber = 4;
+ private final int floatMaxPointValue = 100000;
+ private final int floatMaxPointNumber = 5;
private final long doubleMaxPointValue = 1000000000000000L;
private final int doubleMaxPointNumber = 15;
private List<Float> floatList;
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/HuffmanV2DecoderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/HuffmanV2DecoderTest.java
new file mode 100644
index 0000000000..5b4b803a04
--- /dev/null
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/HuffmanV2DecoderTest.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import org.apache.iotdb.tsfile.encoding.encoder.HuffmanEncoder;
+import org.apache.iotdb.tsfile.encoding.encoder.HuffmanEncoderV2;
+import org.apache.iotdb.tsfile.utils.Binary;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.nio.ByteBuffer;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class HuffmanV2DecoderTest {
+ private HuffmanEncoderV2 encoder = new HuffmanEncoderV2(1024);
+ private HuffmanDecoderV2 decoder = new HuffmanDecoderV2();
+ private ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ @Test
+ public void testSingle() {
+ int a[] = {1};
+ testAll(a);
+ a[0] = 2;
+ testAll(a);
+ a[0] = 1023;
+ testAll(a);
+ }
+
+ @Test
+ public void testAllUnique() {
+ int a[] = {1, 2, 3};
+ int b[] = {52, 123, 432};
+ int c[] = {54, 76, 42, 27, 35};
+ testAll(a);
+ testAll(b);
+ testAll(c);
+ }
+
+ @Test
+ public void testAllSame() {
+ int a[] = {20, 20, 20};
+ int b[] = {166, 166, 166};
+ testAll(a);
+ testAll(b);
+ }
+
+ @Test
+ public void testMixed() {
+ // all characters
+ int[] allChars = new int[700];
+ for (int i = 0; i < 700; i++) {
+ allChars[i] = i;
+ }
+ testAll(allChars);
+ }
+
+ private void testAll(int[] all) {
+ for (int s : all) {
+ encoder.encode(s, baos);
+ }
+ encoder.flush(baos);
+
+ ByteBuffer out = ByteBuffer.wrap(baos.toByteArray());
+
+ for (int s : all) {
+ assertTrue(decoder.hasNext(out));
+ int b = decoder.readInt(out);
+ assertEquals(s, b);
+ }
+
+ decoder.reset();
+ baos.reset();
+ }
+}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/RLBEDecoderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/RLBEDecoderTest.java
new file mode 100644
index 0000000000..5a7547ef99
--- /dev/null
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/RLBEDecoderTest.java
@@ -0,0 +1,257 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import org.apache.iotdb.tsfile.encoding.encoder.*;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public class RLBEDecoderTest {
+
+ private static final Logger logger = LoggerFactory.getLogger(RLBEDecoderTest.class);
+ private final double delta = 0.0000001;
+ private final int floatMaxPointValue = 10000;
+ private final long doubleMaxPointValue = 1000000000000000L;
+ private List<Float> floatList;
+ private List<Double> doubleList;
+
+ @Before
+ public void setUp() {
+ floatList = new ArrayList<Float>();
+ int hybridCount = 11;
+ int hybridNum = 50;
+ int hybridStart = 2000;
+ for (int i = 0; i < hybridNum; i++) {
+ for (int j = 0; j < hybridCount; j++) {
+ floatList.add((float) hybridStart / floatMaxPointValue);
+ }
+ for (int j = 0; j < hybridCount; j++) {
+ floatList.add((float) hybridStart / floatMaxPointValue);
+ hybridStart += 3;
+ }
+
+ hybridCount += 2;
+ }
+
+ doubleList = new ArrayList<Double>();
+ int hybridCountDouble = 11;
+ int hybridNumDouble = 50;
+ long hybridStartDouble = 2000;
+
+ for (int i = 0; i < hybridNumDouble; i++) {
+ for (int j = 0; j < hybridCountDouble; j++) {
+ doubleList.add((double) hybridStartDouble / doubleMaxPointValue);
+ }
+ for (int j = 0; j < hybridCountDouble; j++) {
+ doubleList.add((double) hybridStartDouble / doubleMaxPointValue);
+ hybridStart += 3;
+ }
+
+ hybridCountDouble += 2;
+ }
+ }
+
+ @After
+ public void tearDown() {}
+
+ @Test
+ public void testNegativeNumber() throws IOException {
+ Encoder encoder = new DoubleRLBE();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ double value = -7.101f;
+ encoder.encode(value, baos);
+ encoder.encode(value - 2, baos);
+ encoder.encode(value - 4, baos);
+ encoder.flush(baos);
+ encoder.encode(value, baos);
+ encoder.encode(value - 2, baos);
+ encoder.encode(value - 4, baos);
+ encoder.flush(baos);
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+ for (int i = 0; i < 2; i++) {
+ Decoder decoder = new DoubleRLBEDecoder();
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readDouble(buffer), delta);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value - 2, decoder.readDouble(buffer), delta);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value - 4, decoder.readDouble(buffer), delta);
+ }
+ }
+ }
+
+ @Test
+ public void testZeroNumber() throws IOException {
+ Encoder encoder = new DoubleRLBE();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ double value = 0f;
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.flush(baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.flush(baos);
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+ for (int i = 0; i < 2; i++) {
+ Decoder decoder = new DoubleRLBEDecoder();
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readDouble(buffer), delta);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readDouble(buffer), delta);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readDouble(buffer), delta);
+ }
+ }
+ }
+
+ @Test
+ public void testFloatRepeat() throws Exception {
+ for (int i = 1; i <= 10; i++) {
+ testFloatLength(floatList, false, i);
+ }
+ }
+
+ @Test
+ public void testDoubleRepeat() throws Exception {
+ for (int i = 1; i <= 10; i++) {
+ testDoubleLength(doubleList, false, i);
+ }
+ }
+
+ @Test
+ public void testFloat() throws IOException {
+ Encoder encoder = new FloatRLBE();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ float value = 7.101f;
+ int num = 10000;
+ for (int i = 0; i < num; i++) {
+ encoder.encode(value + 2 * i, baos);
+ }
+ encoder.flush(baos);
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+ Decoder decoder = new FloatRLBEDecoder();
+ for (int i = 0; i < num; i++) {
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value + 2 * i, decoder.readFloat(buffer), delta);
+ continue;
+ }
+ fail();
+ }
+ }
+
+ @Test
+ public void testDouble() throws IOException {
+ Encoder encoder = new DoublePrecisionEncoderV1();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ double value = 7.101f;
+ int num = 1000;
+ for (int i = 0; i < num; i++) {
+ encoder.encode(value + 2 * i, baos);
+ }
+ encoder.flush(baos);
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+ Decoder decoder = new DoublePrecisionDecoderV1();
+ for (int i = 0; i < num; i++) {
+ if (decoder.hasNext(buffer)) {
+ // System.out.println("turn "+i);
+ assertEquals(value + 2 * i, decoder.readDouble(buffer), delta);
+ continue;
+ }
+ fail();
+ }
+ }
+
+ private void testFloatLength(List<Float> valueList, boolean isDebug, int repeatCount)
+ throws Exception {
+ Encoder encoder = new FloatRLBE();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ for (int i = 0; i < repeatCount; i++) {
+ for (float value : valueList) {
+ encoder.encode(value, baos);
+ }
+ encoder.flush(baos);
+ }
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+ for (int i = 0; i < repeatCount; i++) {
+
+ Decoder decoder = new FloatRLBEDecoder();
+ for (float value : valueList) {
+ // System.out.println("Repeat: "+i+" value: "+value);
+ if (decoder.hasNext(buffer)) {
+ float value_ = decoder.readFloat(buffer);
+ if (isDebug) {
+ logger.debug("{} // {}", value_, value);
+ }
+ assertEquals(value, value_, delta);
+ continue;
+ }
+ fail();
+ }
+ }
+ }
+
+ private void testDoubleLength(List<Double> valueList, boolean isDebug, int repeatCount)
+ throws Exception {
+ Encoder encoder = new DoublePrecisionEncoderV1();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ for (int i = 0; i < repeatCount; i++) {
+ for (double value : valueList) {
+ encoder.encode(value, baos);
+ }
+ encoder.flush(baos);
+ }
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ for (int i = 0; i < repeatCount; i++) {
+ Decoder decoder = new DoublePrecisionDecoderV1();
+ for (double value : valueList) {
+ if (decoder.hasNext(buffer)) {
+ double value_ = decoder.readDouble(buffer);
+ if (isDebug) {
+ logger.debug("{} // {}", value_, value);
+ }
+ assertEquals(value, value_, delta);
+ continue;
+ }
+ fail();
+ }
+ }
+ }
+}
+
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/SprintzDecoderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/SprintzDecoderTest.java
new file mode 100644
index 0000000000..a7d7947ff1
--- /dev/null
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/SprintzDecoderTest.java
@@ -0,0 +1,589 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import org.apache.iotdb.tsfile.encoding.encoder.*;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public class SprintzDecoderTest {
+ private static final double DELTA = 0;
+ private static final int FLOAT_MAX_POINT_VALUE = 10000;
+ private static final long DOUBLE_MAX_POINT_VALUE = 1000000000000000L;
+
+ private static final List<Float> floatList = new ArrayList<>();
+ private static final List<Double> doubleList = new ArrayList<>();
+ private static final List<Integer> intList = new ArrayList<>();
+ private static final List<Long> longList = new ArrayList<>();
+
+ private static final List<Integer> iterations = new ArrayList<>();
+
+ @BeforeClass
+ public static void setUp() {
+ int hybridCount = 11;
+ int hybridNum = 50;
+ int hybridStart = 2000;
+ for (int i = 0; i < hybridNum; i++) {
+ for (int j = 0; j < hybridCount; j++) {
+ floatList.add((float) hybridStart / FLOAT_MAX_POINT_VALUE);
+ doubleList.add((double) hybridStart / DOUBLE_MAX_POINT_VALUE);
+ intList.add(hybridStart);
+ longList.add((long) hybridStart);
+ }
+ for (int j = 0; j < hybridCount; j++) {
+ floatList.add((float) hybridStart / FLOAT_MAX_POINT_VALUE);
+ doubleList.add((double) hybridStart / DOUBLE_MAX_POINT_VALUE);
+ intList.add(hybridStart);
+ longList.add((long) hybridStart);
+ hybridStart += 3;
+ }
+ hybridCount += 2;
+ }
+
+ iterations.add(1);
+ iterations.add(3);
+ iterations.add(8);
+ iterations.add(16);
+ iterations.add(1000);
+ iterations.add(10000);
+ }
+
+ @Test
+ public void testIntSingleValue() throws IOException {
+ Encoder encoder = new IntSprintzEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ encoder.encode(777, baos);
+ encoder.flush(baos);
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ Decoder decoder = new IntSprintzDecoder();
+ if (decoder.hasNext(buffer)) {
+ assertEquals(777, decoder.readInt(buffer));
+ }
+ if (decoder.hasNext(buffer)) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testLongSingleValue() throws IOException {
+ Encoder encoder = new LongSprintzEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ encoder.encode((long) Integer.MAX_VALUE + 10, baos);
+ encoder.flush(baos);
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ Decoder decoder = new LongSprintzDecoder();
+ if (decoder.hasNext(buffer)) {
+ assertEquals((long) Integer.MAX_VALUE + 10, decoder.readLong(buffer));
+ }
+ if (decoder.hasNext(buffer)) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testIntZeroNumber() throws IOException {
+ Encoder encoder = new IntSprintzEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ int value = 0;
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.flush(baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.flush(baos);
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+ for (int i = 0; i < 2; i++) {
+ Decoder decoder = new IntSprintzDecoder();
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readInt(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readInt(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readInt(buffer), DELTA);
+ }
+ }
+ }
+
+ @Test
+ public void testLongZeroNumber() throws IOException {
+ Encoder encoder = new LongSprintzEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ long value = 0;
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.flush(baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.flush(baos);
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+ for (int i = 0; i < 2; i++) {
+ Decoder decoder = new LongSprintzDecoder();
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readLong(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readLong(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readLong(buffer), DELTA);
+ }
+ }
+ }
+
+ @Test
+ public void testInteger() throws IOException {
+ for (Integer num : iterations) {
+ Encoder encoder = new IntSprintzEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ int value = 7;
+ for (int i = 0; i < num; i++) {
+ encoder.encode(value + 2 * i, baos);
+ }
+ encoder.flush(baos);
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ Decoder decoder = new IntSprintzDecoder();
+ for (int i = 0; i < num; i++) {
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value + 2 * i, decoder.readInt(buffer));
+ continue;
+ }
+ fail();
+ }
+ if (decoder.hasNext(buffer)) {
+ fail();
+ }
+ }
+ }
+
+ @Test
+ public void testLong() throws IOException {
+ for (Integer num : iterations) {
+ Encoder encoder = new LongSprintzEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ long value = 7;
+ for (int i = 0; i < num; i++) {
+ encoder.encode(value + 2 * i, baos);
+ }
+ encoder.flush(baos);
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ Decoder decoder = new LongSprintzDecoder();
+ for (int i = 0; i < num; i++) {
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value + 2 * i, decoder.readLong(buffer));
+ continue;
+ }
+ fail();
+ }
+ if (decoder.hasNext(buffer)) {
+ fail();
+ }
+ }
+ }
+
+ @Test
+ public void testIntegerRepeat() throws Exception {
+ for (int i = 1; i <= 10; i++) {
+ testInteger(i);
+ }
+ }
+
+ private void testInteger(int repeatCount) throws Exception {
+ Encoder encoder = new IntSprintzEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ for (int i = 0; i < repeatCount; i++) {
+ for (int value : SprintzDecoderTest.intList) {
+ encoder.encode(value, baos);
+ }
+ encoder.flush(baos);
+ }
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ for (int i = 0; i < repeatCount; i++) {
+ Decoder decoder = new IntSprintzDecoder();
+ for (int expected : SprintzDecoderTest.intList) {
+ if (decoder.hasNext(buffer)) {
+ int actual = decoder.readInt(buffer);
+ assertEquals(expected, actual);
+ continue;
+ }
+ fail();
+ }
+ }
+
+ encoder = new IntSprintzEncoder();
+ baos = new ByteArrayOutputStream();
+ for (int i = 0; i < repeatCount; i++) {
+ for (int value : SprintzDecoderTest.intList) {
+ encoder.encode(-value, baos);
+ }
+ encoder.flush(baos);
+ }
+
+ buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ for (int i = 0; i < repeatCount; i++) {
+ Decoder decoder = new IntSprintzDecoder();
+ for (int expected : SprintzDecoderTest.intList) {
+ if (decoder.hasNext(buffer)) {
+ int actual = decoder.readInt(buffer);
+ assertEquals(expected, -actual);
+ continue;
+ }
+ fail();
+ }
+ }
+ }
+
+ @Test
+ public void testLongRepeat() throws Exception {
+ for (int i = 1; i <= 10; i++) {
+ testLong(i);
+ }
+ }
+
+ private void testLong(int repeatCount) throws Exception {
+ Encoder encoder = new LongSprintzEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ for (int i = 0; i < repeatCount; i++) {
+ for (long value : SprintzDecoderTest.longList) {
+ encoder.encode(value, baos);
+ }
+ encoder.flush(baos);
+ }
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ for (int i = 0; i < repeatCount; i++) {
+ Decoder decoder = new LongSprintzDecoder();
+ for (long expected : SprintzDecoderTest.longList) {
+ if (decoder.hasNext(buffer)) {
+ long actual = decoder.readLong(buffer);
+ assertEquals(expected, actual);
+ continue;
+ }
+ fail();
+ }
+ }
+
+ encoder = new LongSprintzEncoder();
+ baos = new ByteArrayOutputStream();
+ for (int i = 0; i < repeatCount; i++) {
+ for (long value : SprintzDecoderTest.longList) {
+ encoder.encode(-value, baos);
+ }
+ encoder.flush(baos);
+ }
+
+ buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ for (int i = 0; i < repeatCount; i++) {
+ Decoder decoder = new LongSprintzDecoder();
+ for (long expected : SprintzDecoderTest.longList) {
+ if (decoder.hasNext(buffer)) {
+ long actual = decoder.readLong(buffer);
+ assertEquals(expected, -actual);
+ continue;
+ }
+ fail();
+ }
+ }
+ }
+
+ @Test
+ public void testFloatSingleValue() throws IOException {
+ Encoder encoder = new FloatSprintzEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ encoder.encode(Float.MAX_VALUE, baos);
+ encoder.flush(baos);
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ Decoder decoder = new FloatSprintzDecoder();
+ if (decoder.hasNext(buffer)) {
+ assertEquals(Float.MAX_VALUE, decoder.readFloat(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testDoubleSingleValue() throws IOException {
+ Encoder encoder = new DoubleSprintzEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ encoder.encode(Double.MAX_VALUE, baos);
+ encoder.flush(baos);
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ Decoder decoder = new DoubleSprintzDecoder();
+ if (decoder.hasNext(buffer)) {
+ assertEquals(Double.MAX_VALUE, decoder.readDouble(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testFloatZeroNumber() throws IOException {
+ Encoder encoder = new FloatSprintzEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ float value = 0f;
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.flush(baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.flush(baos);
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+ for (int i = 0; i < 2; i++) {
+ Decoder decoder = new FloatSprintzDecoder();
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readFloat(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readFloat(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readFloat(buffer), DELTA);
+ }
+ }
+ }
+
+ @Test
+ public void testDoubleZeroNumber() throws IOException {
+ Encoder encoder = new DoubleSprintzEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ double value = 0f;
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.flush(baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.flush(baos);
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+ for (int i = 0; i < 2; i++) {
+ Decoder decoder = new DoubleSprintzDecoder();
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readDouble(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readDouble(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readDouble(buffer), DELTA);
+ }
+ }
+ }
+
+ @Test
+ public void testFloat() throws IOException {
+ for (Integer num : iterations) {
+ Encoder encoder = new FloatSprintzEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ float value = 7.101f;
+ for (int i = 0; i < num; i++) {
+ encoder.encode(value + 2 * i, baos);
+ }
+ encoder.flush(baos);
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ Decoder decoder = new FloatSprintzDecoder();
+ for (int i = 0; i < num; i++) {
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value + 2 * i, decoder.readFloat(buffer), DELTA);
+ continue;
+ }
+ fail();
+ }
+ if (decoder.hasNext(buffer)) {
+ fail();
+ }
+ }
+ }
+
+ @Test
+ public void testDouble() throws IOException {
+ for (Integer num : iterations) {
+ Encoder encoder = new DoubleSprintzEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ double value = 7.101f;
+ for (int i = 0; i < num; i++) {
+ encoder.encode(value + 2 * i, baos);
+ }
+ encoder.flush(baos);
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ Decoder decoder = new DoubleSprintzDecoder();
+ for (int i = 0; i < num; i++) {
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value + 2 * i, decoder.readDouble(buffer), DELTA);
+ continue;
+ }
+ fail();
+ }
+ if (decoder.hasNext(buffer)) {
+ fail();
+ }
+ }
+ }
+
+ @Test
+ public void testFloatRepeat() throws Exception {
+ for (int i = 1; i <= 10; i++) {
+ testFloat(i);
+ }
+ }
+
+ private void testFloat(int repeatCount) throws Exception {
+ Encoder encoder = new FloatSprintzEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ for (int i = 0; i < repeatCount; i++) {
+ for (float value : SprintzDecoderTest.floatList) {
+ encoder.encode(value, baos);
+ }
+ encoder.flush(baos);
+ }
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ for (int i = 0; i < repeatCount; i++) {
+ Decoder decoder = new FloatSprintzDecoder();
+ for (float expected : SprintzDecoderTest.floatList) {
+ if (decoder.hasNext(buffer)) {
+ float actual = decoder.readFloat(buffer);
+ assertEquals(expected, actual, DELTA);
+ continue;
+ }
+ fail();
+ }
+ }
+
+ encoder = new FloatSprintzEncoder();
+ baos = new ByteArrayOutputStream();
+ for (int i = 0; i < repeatCount; i++) {
+ for (float value : SprintzDecoderTest.floatList) {
+ encoder.encode(-value, baos);
+ }
+ encoder.flush(baos);
+ }
+
+ buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ for (int i = 0; i < repeatCount; i++) {
+ Decoder decoder = new FloatSprintzDecoder();
+ for (float expected : SprintzDecoderTest.floatList) {
+ if (decoder.hasNext(buffer)) {
+ float actual = decoder.readFloat(buffer);
+ assertEquals(expected, -actual, DELTA);
+ continue;
+ }
+ fail();
+ }
+ }
+ }
+
+ @Test
+ public void testDoubleRepeat() throws Exception {
+ for (int i = 1; i <= 10; i++) {
+ testDouble(i);
+ }
+ }
+
+ private void testDouble(int repeatCount) throws Exception {
+ Encoder encoder = new DoubleSprintzEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ for (int i = 0; i < repeatCount; i++) {
+ for (double value : SprintzDecoderTest.doubleList) {
+ encoder.encode(value, baos);
+ }
+ encoder.flush(baos);
+ }
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ for (int i = 0; i < repeatCount; i++) {
+ Decoder decoder = new DoubleSprintzDecoder();
+ for (double expected : SprintzDecoderTest.doubleList) {
+ if (decoder.hasNext(buffer)) {
+ double actual = decoder.readDouble(buffer);
+ assertEquals(expected, actual, DELTA);
+ continue;
+ }
+ fail();
+ }
+ }
+
+ encoder = new DoubleSprintzEncoder();
+ baos = new ByteArrayOutputStream();
+ for (int i = 0; i < repeatCount; i++) {
+ for (double value : SprintzDecoderTest.doubleList) {
+ encoder.encode(-value, baos);
+ }
+ encoder.flush(baos);
+ }
+
+ buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ for (int i = 0; i < repeatCount; i++) {
+ Decoder decoder = new DoubleSprintzDecoder();
+ for (double expected : SprintzDecoderTest.doubleList) {
+ if (decoder.hasNext(buffer)) {
+ double actual = decoder.readDouble(buffer);
+ assertEquals(expected, -actual, DELTA);
+ continue;
+ }
+ fail();
+ }
+ }
+ }
+}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/delta/DeltaBinaryEncoderIntegerTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/delta/DeltaBinaryEncoderIntegerTest.java
index 5043a23a3c..2a289596af 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/delta/DeltaBinaryEncoderIntegerTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/delta/DeltaBinaryEncoderIntegerTest.java
@@ -29,6 +29,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Random;
+import static java.lang.Math.pow;
import static org.junit.Assert.assertEquals;
public class DeltaBinaryEncoderIntegerTest {
@@ -46,6 +47,15 @@ public class DeltaBinaryEncoderIntegerTest {
reader = new DeltaBinaryDecoder.IntDeltaDecoder();
}
+ @Test
+ public void testtest() throws IOException{
+ int[] data = new int[20];
+ for (int i = 0; i < 20; i++) {
+ data[i] = i * (int)pow(-1, i);
+ }
+ shouldReadAndWrite(data, data.length);
+ }
+
@Test
public void testBasic() throws IOException {
int[] data = new int[ROW_NUM];
@@ -65,7 +75,7 @@ public class DeltaBinaryEncoderIntegerTest {
private void boundInt(int power, int[] data) throws IOException {
for (int i = 0; i < ROW_NUM; i++) {
- data[i] = ran.nextInt((int) Math.pow(2, power));
+ data[i] = ran.nextInt((int) pow(2, power));
}
shouldReadAndWrite(data, ROW_NUM);
}
@@ -106,5 +116,6 @@ public class DeltaBinaryEncoderIntegerTest {
while (reader.hasNext(buffer)) {
assertEquals(data[i++], reader.readInt(buffer));
}
+
}
}