You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hu...@apache.org on 2023/12/04 14:29:23 UTC
(iotdb) 20/42: Update EncodeTest.java
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch research/encoding-reorder
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 6693b83bdf5b5a01a617f61423def6370781da3a
Author: xjz17 <67...@users.noreply.github.com>
AuthorDate: Tue Nov 14 00:00:24 2023 +0800
Update EncodeTest.java
---
.../apache/iotdb/tsfile/encoding/EncodeTest.java | 713 ++++++++++++++-------
1 file changed, 470 insertions(+), 243 deletions(-)
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/EncodeTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/EncodeTest.java
index 816b20fab04..6e802aabd76 100644
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/EncodeTest.java
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/EncodeTest.java
@@ -4,7 +4,6 @@ import org.apache.iotdb.tsfile.compress.ICompressor;
import org.apache.iotdb.tsfile.compress.IUnCompressor;
import org.apache.iotdb.tsfile.encoding.decoder.Decoder;
import org.apache.iotdb.tsfile.encoding.encoder.Encoder;
-import org.apache.iotdb.tsfile.encoding.encoder.FloatEncoder;
import org.apache.iotdb.tsfile.encoding.encoder.TSEncodingBuilder;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -26,13 +25,17 @@ import java.util.ArrayList;
public class EncodeTest {
public static void main(@NotNull String[] args) throws IOException {
-
- String parent_dir = "/Users/zihanguo/Downloads/outliier_code/encoding-outlier/";
- String output_parent_dir = parent_dir + "vldb/compression_ratio/sota_ratio/";
- String input_parent_dir = parent_dir + "origin_data/";
+// String parent_dir =
+// "C:\\Users\\xiaoj\\Documents\\GitHub\\encoding-reorder\\vldb\\compression_ratio\\sota_ratio";
+//
+ String parent_dir = "C:\\Users\\Jinnsjao Shawl\\Documents\\GitHub\\encoding-reorder\\";
+ String output_parent_dir = parent_dir + "vldb\\compression_ratio\\sota_ratio";
+ String input_parent_dir = parent_dir + "reorder\\iotdb_test_small\\";
ArrayList<String> input_path_list = new ArrayList<>();
ArrayList<String> output_path_list = new ArrayList<>();
ArrayList<String> dataset_name = new ArrayList<>();
+ ArrayList<Integer> dataset_block_size = new ArrayList<>();
+
dataset_name.add("CS-Sensors");
dataset_name.add("Metro-Traffic");
dataset_name.add("USGS-Earthquakes");
@@ -47,311 +50,535 @@ public class EncodeTest {
dataset_name.add("EPM-Education");
- for (String item : dataset_name) {
- input_path_list.add(input_parent_dir + item);
+ for (int i = 0; i < dataset_name.size(); i++) {
+ input_path_list.add(input_parent_dir + dataset_name.get(i));
}
- output_path_list.add(output_parent_dir + "CS-Sensors_ratio.csv"); // 0
-
- output_path_list.add(output_parent_dir + "Metro-Traffic_ratio.csv");// 1
-
- output_path_list.add(output_parent_dir + "USGS-Earthquakes_ratio.csv");// 2
-
- output_path_list.add(output_parent_dir + "YZ-Electricity_ratio.csv"); // 3
-
- output_path_list.add(output_parent_dir + "GW-Magnetic_ratio.csv"); //4
-
- output_path_list.add(output_parent_dir + "TY-Fuel_ratio.csv");//5
-
- output_path_list.add(output_parent_dir + "Cyber-Vehicle_ratio.csv"); //6
-
- output_path_list.add(output_parent_dir + "Vehicle-Charge_ratio.csv");//7
-
- output_path_list.add(output_parent_dir + "Nifty-Stocks_ratio.csv");//8
-
- output_path_list.add(output_parent_dir + "TH-Climate_ratio.csv");//9
-
- output_path_list.add(output_parent_dir + "TY-Transport_ratio.csv");//10
-
- output_path_list.add(output_parent_dir + "EPM-Education_ratio.csv");//11
-
+ output_path_list.add(output_parent_dir + "\\CS-Sensors_ratio.csv"); // 0
+ dataset_block_size.add(1024);
+// dataset_k.add(5);
+ output_path_list.add(output_parent_dir + "\\Metro-Traffic_ratio.csv");// 1
+ dataset_block_size.add(512);
+// dataset_k.add(7);
+ output_path_list.add(output_parent_dir + "\\USGS-Earthquakes_ratio.csv");// 2
+ dataset_block_size.add(512);
+// dataset_k.add(7);
+ output_path_list.add(output_parent_dir + "\\YZ-Electricity_ratio.csv"); // 3
+ dataset_block_size.add(512);
+// dataset_k.add(1);
+ output_path_list.add(output_parent_dir + "\\GW-Magnetic_ratio.csv"); //4
+ dataset_block_size.add(128);
+// dataset_k.add(6);
+ output_path_list.add(output_parent_dir + "\\TY-Fuel_ratio.csv");//5
+ dataset_block_size.add(64);
+// dataset_k.add(5);
+ output_path_list.add(output_parent_dir + "\\Cyber-Vehicle_ratio.csv"); //6
+ dataset_block_size.add(128);
+// dataset_k.add(4);
+ output_path_list.add(output_parent_dir + "\\Vehicle-Charge_ratio.csv");//7
+ dataset_block_size.add(512);
+// dataset_k.add(8);
+ output_path_list.add(output_parent_dir + "\\Nifty-Stocks_ratio.csv");//8
+ dataset_block_size.add(256);
+// dataset_k.add(1);
+ output_path_list.add(output_parent_dir + "\\TH-Climate_ratio.csv");//9
+ dataset_block_size.add(512);
+// dataset_k.add(2);
+ output_path_list.add(output_parent_dir + "\\TY-Transport_ratio.csv");//10
+ dataset_block_size.add(512);
+// dataset_k.add(9);
+ output_path_list.add(output_parent_dir + "\\EPM-Education_ratio.csv");//11
+ dataset_block_size.add(512);
+
+// for(int file_i=3;file_i<4;file_i++){
for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
String inputPath = input_path_list.get(file_i);
String Output = output_path_list.get(file_i);
+ // String Output = "C:\\Users\\xiaoj\\Desktop\\test_ratio_ts_2diff.csv";
// speed
int repeatTime = 1; // set repeat time
String dataTypeName = "int"; // set dataType
+ // if (args.length >= 2) inputPath = args[1];
+ // if (args.length >= 3) Output = args[2];
File file = new File(inputPath);
File[] tempList = file.listFiles();
// select encoding algorithms
TSEncoding[] encodingList = {
- TSEncoding.PLAIN ,
- TSEncoding.TS_2DIFF,
- TSEncoding.RLE,
- TSEncoding.SPRINTZ,
- TSEncoding.GORILLA,
- TSEncoding.RLBE,
+ TSEncoding.PLAIN ,
+ TSEncoding.TS_2DIFF,
+ TSEncoding.RLE,
+ TSEncoding.SPRINTZ,
+ TSEncoding.GORILLA,
+ TSEncoding.RLBE,
+// TSEncoding.RAKE,
TSEncoding.CHIMP,
// TSEncoding.BUFF
};
-
+ // select compression algorithms
CompressionType[] compressList = {
- CompressionType.UNCOMPRESSED,
-
+ CompressionType.UNCOMPRESSED,
+ // CompressionType.LZ4,
+ // CompressionType.GZIP,
+ // CompressionType.SNAPPY
};
CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
String[] head = {
- "Input Direction",
- "Column Index",
- "Encoding Algorithm",
- "Compress Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Compress Time",
- "Uncompress Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
+ "Input Direction",
+ "Column Index",
+ "Encoding Algorithm",
+ "Compress Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+// "Compress Time",
+// "Uncompress Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
};
writer.writeRecord(head); // write header to output file
assert tempList != null;
+ int fileRepeat = 0;
ArrayList<Integer> columnIndexes = new ArrayList<>(); // set the column indexes of compressed
for (int i = 0; i < 2; i++) {
columnIndexes.add(i, i);
}
for (File f : tempList) {
System.out.println(f);
+ fileRepeat += 1;
InputStream inputStream = Files.newInputStream(f.toPath());
CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ String fileName = f.getAbsolutePath();
ArrayList<String> data = new ArrayList<>();
for (int index : columnIndexes) {
- if (index == 0){
- continue;
- }
- int max_precision = 2;
- if (file_i == 3){
- max_precision = 4;
- } else if (file_i == 4 ||file_i == 5) {
- max_precision = 3;
- }
+ // add a column to "data"
+ // System.out.println(index);
+ int max_precision = 0;
loader.readHeaders();
data.clear();
while (loader.readRecord()) {
String v = loader.getValues()[index];
-
+ // int ind = v.indexOf(".");
+ // if (ind > -1) {
+ // int len = v.substring(ind + 1).length();
+ // if (len > max_precision) {
+ // max_precision = len;
+ // }
+ // }
data.add(v);
- if (!v.matches("-?\\d+")){
- dataTypeName = "float";
- }
}
+ // System.out.println(max_precision);
inputStream.close();
switch (dataTypeName) {
case "int":
- {
- TSDataType dataType = TSDataType.INT32; // set TSDataType
- ArrayList<Integer> tmp = new ArrayList<>();
- for (String value : data) {
- tmp.add(Integer.valueOf(value));
- }
- // Iterate over each encoding algorithm
- for (TSEncoding encoding : encodingList) {
- Encoder encoder =
- TSEncodingBuilder.getEncodingBuilder(encoding).getEncoder(dataType);
- Decoder decoder = Decoder.getDecoderByType(encoding, dataType);
- long encodeTime = 0;
- long decodeTime = 0;
-
- // Iterate over each compression algorithm
- for (CompressionType comp : compressList) {
- ICompressor compressor = ICompressor.getCompressor(comp);
- IUnCompressor unCompressor = IUnCompressor.getUnCompressor(comp);
-
- double ratio = 0;
- double compressed_size = 0;
-
- long compressTime = 0;
- long uncompressTime = 0;
-
- // repeat many times to test time
- for (int i = 0; i < repeatTime; i++) {
- ByteArrayOutputStream buffer = new ByteArrayOutputStream();
-
- // test encode time
- long s = System.nanoTime();
- for (int val : tmp) {
- encoder.encode(val, buffer);
- }
-
- encoder.flush(buffer);
- long e = System.nanoTime();
- encodeTime += (e - s);
-
- // test compress time
- byte[] elems = buffer.toByteArray();
- s = System.nanoTime();
- byte[] compressed = compressor.compress(elems);
- e = System.nanoTime();
- compressTime += (e - s);
-
- // test compression ratio and compressed size
- compressed_size += compressed.length;
- double ratioTmp =
- (double) compressed.length / (double) (tmp.size() * Integer.BYTES);
- ratio += ratioTmp;
-
- // test uncompress time
- s = System.nanoTime();
- unCompressor.uncompress(compressed);
- e = System.nanoTime();
- uncompressTime += (e - s);
-
- // test decode time
- ByteBuffer ebuffer = ByteBuffer.wrap(buffer.toByteArray());
- s = System.nanoTime();
- while (decoder.hasNext(ebuffer)) {
- decoder.readInt(ebuffer);
- }
- e = System.nanoTime();
- decodeTime += (e - s);
-
- buffer.close();
+ {
+ TSDataType dataType = TSDataType.INT32; // set TSDataType
+ ArrayList<Integer> tmp = new ArrayList<>();
+ for (String value : data) {
+ tmp.add(Integer.valueOf(value));
+ }
+ // Iterate over each encoding algorithm
+ for (TSEncoding encoding : encodingList) {
+ Encoder encoder =
+ TSEncodingBuilder.getEncodingBuilder(encoding).getEncoder(dataType);
+ Decoder decoder = Decoder.getDecoderByType(encoding, dataType);
+ long encodeTime = 0;
+ long decodeTime = 0;
+
+ // Iterate over each compression algorithm
+ for (CompressionType comp : compressList) {
+ ICompressor compressor = ICompressor.getCompressor(comp);
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(comp);
+
+ double ratio = 0;
+ double compressed_size = 0;
+
+ long compressTime = 0;
+ long uncompressTime = 0;
+
+ // repeat many times to test time
+ for (int i = 0; i < repeatTime; i++) {
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+
+ // test encode time
+ long s = System.nanoTime();
+ for (int val : tmp) {
+ encoder.encode(val, buffer);
}
- ratio /= repeatTime;
- compressed_size /= repeatTime;
-
- String[] record = {
- f.toString(),
- String.valueOf(index),
- encoding.toString(),
- comp.toString(),
- String.valueOf(encodeTime),
- String.valueOf(decodeTime),
- String.valueOf(compressTime),
- String.valueOf(uncompressTime),
- String.valueOf(data.size()),
- String.valueOf(compressed_size),
- String.valueOf(ratio)
- };
- System.out.println(ratio);
- writer.writeRecord(record);
+ // byte[] elems = buffer.toByteArray();
+ encoder.flush(buffer);
+ long e = System.nanoTime();
+ encodeTime += (e - s);
+
+ // test compress time
+ byte[] elems = buffer.toByteArray();
+ s = System.nanoTime();
+ byte[] compressed = compressor.compress(elems);
+ e = System.nanoTime();
+ compressTime += (e - s);
+
+ // test compression ratio and compressed size
+ compressed_size += compressed.length;
+ double ratioTmp =
+ (double) compressed.length / (double) (tmp.size() * Integer.BYTES);
+ ratio += ratioTmp;
+
+ // test uncompress time
+ s = System.nanoTime();
+ byte[] x = unCompressor.uncompress(compressed);
+ e = System.nanoTime();
+ uncompressTime += (e - s);
+
+ // test decode time
+ ByteBuffer ebuffer = ByteBuffer.wrap(buffer.toByteArray());
+ s = System.nanoTime();
+ int i_tmp = 0;
+ while (decoder.hasNext(ebuffer)) {
+ // decoder.readInt(ebuffer);
+ decoder.readInt(ebuffer);
+ // if(tmp.get(i_tmp) == tmp_tmp)
+ // System.out.println("equal");
+ // i_tmp += 1;
+ }
+ e = System.nanoTime();
+ decodeTime += (e - s);
+
+ buffer.close();
}
+
+ ratio /= repeatTime;
+ compressed_size /= repeatTime;
+ encodeTime /= repeatTime;
+ decodeTime /= repeatTime;
+ compressTime /= repeatTime;
+ uncompressTime /= repeatTime;
+
+ String[] record = {
+ f.toString(),
+ String.valueOf(index),
+ encoding.toString(),
+ comp.toString(),
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+// String.valueOf(compressTime),
+// String.valueOf(uncompressTime),
+ String.valueOf(data.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ System.out.println(ratio);
+ writer.writeRecord(record);
}
- tmp.clear();
- break;
}
+ tmp.clear();
+ break;
+ }
+ case "long":
+ {
+ TSDataType dataType = TSDataType.INT64;
+ ArrayList<Long> tmp = new ArrayList<>();
+ for (String value : data) {
+ tmp.add(Long.valueOf(value));
+ }
+ // Iterate over each encoding algorithm
+ for (TSEncoding encoding : encodingList) {
+ Encoder encoder =
+ TSEncodingBuilder.getEncodingBuilder(encoding).getEncoder(dataType);
+ Decoder decoder = Decoder.getDecoderByType(encoding, dataType);
+ long encodeTime = 0;
+ long decodeTime = 0;
+
+ // Iterate over each compression algorithm
+ for (CompressionType comp : compressList) {
+ ICompressor compressor = ICompressor.getCompressor(comp);
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(comp);
+ double ratio = 0;
+ double compressed_size = 0;
+
+ long compressTime = 0;
+ long uncompressTime = 0;
+ for (int i = 0; i < repeatTime; i++) {
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+
+ // test encode time
+ long s = System.nanoTime();
+ for (long val : tmp) encoder.encode(val, buffer);
+ encoder.flush(buffer);
+ long e = System.nanoTime();
+ encodeTime += (e - s);
+
+ // test compress time
+ byte[] elems = buffer.toByteArray();
+ s = System.nanoTime();
+ byte[] compressed = compressor.compress(elems);
+ e = System.nanoTime();
+ compressTime += (e - s);
+
+ // test compression ratio and compressed size
+ compressed_size = compressed.length;
+ double ratioTmp =
+ (double) compressed.length / (double) (tmp.size() * Long.BYTES);
+ ratio += ratioTmp;
+
+ // test uncompress time
+ s = System.nanoTime();
+ byte[] x = unCompressor.uncompress(compressed);
+ e = System.nanoTime();
+ uncompressTime += (e - s);
+
+ // test decode time
+ ByteBuffer ebuffer = ByteBuffer.wrap(buffer.toByteArray());
+ s = System.nanoTime();
+ while (decoder.hasNext(ebuffer)) {
+ decoder.readInt(ebuffer);
+ }
+ e = System.nanoTime();
+ decodeTime += (e - s);
- // write info to file
- case "float":
- {
- System.out.println("get float");
- TSDataType dataType = TSDataType.FLOAT;
- ArrayList<Float> tmp = new ArrayList<>();
- data.removeIf(String::isEmpty);
- for (String datum : data) {
- tmp.add(Float.valueOf(datum));
+ buffer.close();
+ }
+
+ ratio /= repeatTime;
+ compressed_size /= repeatTime;
+ encodeTime /= repeatTime;
+ decodeTime /= repeatTime;
+ compressTime /= repeatTime;
+ uncompressTime /= repeatTime;
+
+ // write info to file
+ String[] record = {
+ f.toString(),
+ String.valueOf(index),
+ encoding.toString(),
+ comp.toString(),
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(compressTime),
+ String.valueOf(uncompressTime),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
}
+ }
+ break;
+ }
+ case "double":
+ {
+ TSDataType dataType = TSDataType.DOUBLE;
+ ArrayList<Double> tmp = new ArrayList<>();
+ data.removeIf(String::isEmpty);
+ for (String value : data) {
+ tmp.add(Double.valueOf(value));
+ }
+ // Iterate over each encoding algorithm
+ for (TSEncoding encoding : encodingList) {
+ Encoder encoder =
+ TSEncodingBuilder.getEncodingBuilder(encoding).getEncoder(dataType);
+ Decoder decoder = Decoder.getDecoderByType(encoding, dataType);
+ long encodeTime = 0;
+ long decodeTime = 0;
+
+ // Iterate over each compression algorithm
+ for (CompressionType comp : compressList) {
+ ICompressor compressor = ICompressor.getCompressor(comp);
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(comp);
+ long compressTime = 0;
+ long uncompressTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ // repeat many times to test time
+ for (int i = 0; i < repeatTime; i++) {
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+
+ // test encode time
+ long s = System.nanoTime();
+ for (double val : tmp) encoder.encode(val, buffer);
+ encoder.flush(buffer);
+ long e = System.nanoTime();
+ encodeTime += (e - s);
+
+ // test compress time
+ byte[] elems = buffer.toByteArray();
+ s = System.nanoTime();
+ byte[] compressed = compressor.compress(elems);
+ e = System.nanoTime();
+ compressTime += (e - s);
+
+ // test compression ratio and compressed size
+ compressed_size = compressed.length;
+ double ratioTmp =
+ (double) compressed.length / (double) (tmp.size() * Double.BYTES);
+ ratio += ratioTmp;
+
+ // test uncompress time
+ s = System.nanoTime();
+ byte[] x = unCompressor.uncompress(compressed);
+ e = System.nanoTime();
+ uncompressTime += (e - s);
+
+ // test decode time
+ ByteBuffer ebuffer = ByteBuffer.wrap(buffer.toByteArray());
+ s = System.nanoTime();
+ int i_de = 0;
+ while (decoder.hasNext(ebuffer)) {
+ double v = decoder.readDouble(ebuffer);
+ // if(v!=Double.parseDouble(data.get(i_de))){
+ // System.out.println(v);
+ // System.out.println(data.get(i_de));
+ // System.out.println("noequal");
+ // System.out.println(encoding);
+ // };
+ // i_de++;
+ }
+ e = System.nanoTime();
+ decodeTime += (e - s);
- // Iterate over each encoding algorithm
- for (TSEncoding encoding : encodingList) {
- Encoder encoder;
- encoder = TSEncodingBuilder.getEncodingBuilder(encoding).getEncoder(dataType);
- if (encoding == TSEncoding.TS_2DIFF){
- encoder = new FloatEncoder(TSEncoding.TS_2DIFF, dataType, max_precision);
- } else if (encoding == TSEncoding.RLE){
- encoder = new FloatEncoder(TSEncoding.RLE, dataType, max_precision);
+ buffer.close();
}
- Decoder decoder = Decoder.getDecoderByType(encoding, dataType);
-
- long encodeTime = 0;
- long decodeTime = 0;
- // Iterate over each compression algorithm
- for (CompressionType comp : compressList) {
- ICompressor compressor = ICompressor.getCompressor(comp);
- IUnCompressor unCompressor = IUnCompressor.getUnCompressor(comp);
- long compressTime = 0;
- long uncompressTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
- // repeat many times to test time
- for (int i = 0; i < repeatTime; i++) {
- ByteArrayOutputStream buffer = new ByteArrayOutputStream();
-
- // test encode time
- long s = System.nanoTime();
- for (float val : tmp) {
- encoder.encode(val, buffer);
- }
- encoder.flush(buffer);
- long e = System.nanoTime();
- encodeTime += (e - s);
-
- // test compress time
- byte[] elems = buffer.toByteArray();
- s = System.nanoTime();
- byte[] compressed = compressor.compress(elems);
- e = System.nanoTime();
- compressTime += (e - s);
-
- // test compression ratio and compressed size
- compressed_size += compressed.length;
- double ratioTmp =
- (double) compressed.length / (double) (tmp.size() * Float.BYTES);
- ratio += ratioTmp;
-
- // test uncompress time
- s = System.nanoTime();
- unCompressor.uncompress(compressed);
- e = System.nanoTime();
- uncompressTime += (e - s);
-
- // test decode time
- ByteBuffer ebuffer = ByteBuffer.wrap(buffer.toByteArray());
- while (decoder.hasNext(ebuffer)) {
- decoder.readFloat(ebuffer);
- }
- e = System.nanoTime();
- decodeTime += (e - s);
-
- buffer.close();
+ ratio /= repeatTime;
+ compressed_size /= repeatTime;
+ encodeTime /= repeatTime;
+ decodeTime /= repeatTime;
+ compressTime /= repeatTime;
+ uncompressTime /= repeatTime;
+
+ // write info to file
+ String[] record = {
+ f.toString(),
+ String.valueOf(index),
+ encoding.toString(),
+ comp.toString(),
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(compressTime),
+ String.valueOf(uncompressTime),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+ }
+ break;
+ }
+ case "float":
+ {
+ TSDataType dataType = TSDataType.FLOAT;
+ ArrayList<Float> tmp = new ArrayList<>();
+ data.removeIf(String::isEmpty);
+ for (int i = 0; i < data.size(); i++) {
+ tmp.add(Float.valueOf(data.get(i)));
+ }
+
+ // Iterate over each encoding algorithm
+ for (TSEncoding encoding : encodingList) {
+ Encoder encoder;
+ // if(encoding == TSEncoding.TS_2DIFF){
+ // Map<String, String> props = null;
+ // props.put(Encoder.MAX_POINT_NUMBER,
+ // String.valueOf(max_precision));
+ // TSEncodingBuilder.Ts2Diff.setMaxPointNumber(max_precision);
+ // }
+ encoder = TSEncodingBuilder.getEncodingBuilder(encoding).getEncoder(dataType);
+
+ Decoder decoder = Decoder.getDecoderByType(encoding, dataType);
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ // Iterate over each compression algorithm
+ for (CompressionType comp : compressList) {
+ ICompressor compressor = ICompressor.getCompressor(comp);
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(comp);
+ long compressTime = 0;
+ long uncompressTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ // repeat many times to test time
+ for (int i = 0; i < repeatTime; i++) {
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+
+ // test encode time
+ long s = System.nanoTime();
+ for (float val : tmp) {
+ encoder.encode(val, buffer);
+ }
+ encoder.flush(buffer);
+ long e = System.nanoTime();
+ encodeTime += (e - s);
+
+ // test compress time
+ byte[] elems = buffer.toByteArray();
+ s = System.nanoTime();
+ byte[] compressed = compressor.compress(elems);
+ e = System.nanoTime();
+ compressTime += (e - s);
+
+ // test compression ratio and compressed size
+ compressed_size += compressed.length;
+ double ratioTmp =
+ (double) compressed.length / (double) (tmp.size() * Float.BYTES);
+ ratio += ratioTmp;
+
+ // test uncompress time
+ s = System.nanoTime();
+ byte[] x = unCompressor.uncompress(compressed);
+ e = System.nanoTime();
+ uncompressTime += (e - s);
+
+ // test decode time
+ ByteBuffer ebuffer = ByteBuffer.wrap(buffer.toByteArray());
+ while (decoder.hasNext(ebuffer)) {
+ decoder.readFloat(ebuffer);
}
- ratio /= repeatTime;
- compressed_size /= repeatTime;
-
- // write info to file
- String[] record = {
- f.toString(),
- String.valueOf(index),
- encoding.toString(),
- comp.toString(),
- String.valueOf(encodeTime),
- String.valueOf(decodeTime),
- String.valueOf(compressTime),
- String.valueOf(uncompressTime),
- String.valueOf(data.size()),
- String.valueOf(compressed_size),
- String.valueOf(ratio)
- };
- writer.writeRecord(record);
+ e = System.nanoTime();
+ decodeTime += (e - s);
+
+ buffer.close();
}
+ ratio /= repeatTime;
+ compressed_size /= repeatTime;
+ encodeTime /= repeatTime;
+ decodeTime /= repeatTime;
+ compressTime /= repeatTime;
+ uncompressTime /= repeatTime;
+
+ // write info to file
+ String[] record = {
+ f.toString(),
+ String.valueOf(index),
+ encoding.toString(),
+ comp.toString(),
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(compressTime),
+ String.valueOf(uncompressTime),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
}
- break;
}
+ break;
+ }
}
inputStream = Files.newInputStream(f.toPath());
loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
}
+ // if (fileRepeat > repeatTime) break;
+ // break;
}
writer.close();
}
}
-}
+}
\ No newline at end of file