You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hu...@apache.org on 2023/12/04 14:29:45 UTC
(iotdb) 42/42: Update RegerPFloatTest.java
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch research/encoding-reorder
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 253d02f1e23edbba3161016769814bd2f1f83242
Author: xjz17 <67...@users.noreply.github.com>
AuthorDate: Mon Dec 4 21:37:41 2023 +0800
Update RegerPFloatTest.java
---
.../iotdb/tsfile/encoding/RegerPFloatTest.java | 957 ++-------------------
1 file changed, 58 insertions(+), 899 deletions(-)
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RegerPFloatTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RegerPFloatTest.java
index 14edb0a50b2..65944c63eee 100644
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RegerPFloatTest.java
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RegerPFloatTest.java
@@ -2,8 +2,6 @@ package org.apache.iotdb.tsfile.encoding;
import com.csvreader.CsvReader;
import com.csvreader.CsvWriter;
-import org.apache.commons.math3.stat.descriptive.moment.StandardDeviation;
-import org.apache.commons.math3.stat.regression.OLSMultipleLinearRegression;
import org.apache.commons.math3.linear.*;
import org.junit.Test;
@@ -79,75 +77,6 @@ public class RegerPFloatTest {
return newArray;
}
- public static int bytes2Integer(byte[] encoded, int start, int num) {
- int value = 0;
- if (num > 4) {
- System.out.println("bytes2Integer error");
- return 0;
- }
- for (int i = 0; i < num; i++) {
- value <<= 8;
- int b = encoded[i + start] & 0xFF;
- value |= b;
- }
- return value;
- }
-
- private static int byte2Integer(byte[] encoded, int decode_pos) {
- int value = 0;
- int b = encoded[decode_pos] & 0xFF;
- value |= b;
- if (value == 0) return 256;
- return value % 256;
- }
-
- public static void pack8Values(
- ArrayList<Integer> values, int offset, int width, int encode_pos, byte[] encoded_result) {
- int bufIdx = 0;
- int valueIdx = offset;
- // remaining bits for the current unfinished Integer
- int leftBit = 0;
-
- while (valueIdx < 8 + offset) {
- // buffer is used for saving 32 bits as a part of result
- int buffer = 0;
- // remaining size of bits in the 'buffer'
- int leftSize = 32;
-
- // encode the left bits of current Integer to 'buffer'
- if (leftBit > 0) {
- buffer |= (values.get(valueIdx) << (32 - leftBit));
- leftSize -= leftBit;
- leftBit = 0;
- valueIdx++;
- }
-
- while (leftSize >= width && valueIdx < 8 + offset) {
- // encode one Integer to the 'buffer'
- buffer |= (values.get(valueIdx) << (leftSize - width));
- leftSize -= width;
- valueIdx++;
- }
- // If the remaining space of the buffer can not save the bits for one Integer,
- if (leftSize > 0 && valueIdx < 8 + offset) {
- // put the first 'leftSize' bits of the Integer into remaining space of the
- // buffer
- buffer |= (values.get(valueIdx) >>> (width - leftSize));
- leftBit = width - leftSize;
- }
-
- // put the buffer into the final result
- for (int j = 0; j < 4; j++) {
- encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
- encode_pos++;
- bufIdx++;
- if (bufIdx >= width) {
- return;
- }
- }
- }
- // return encode_pos;
- }
public static void pack8Values(
int[][] values, int index, int offset, int width, int encode_pos, byte[] encoded_result) {
@@ -197,38 +126,6 @@ public class RegerPFloatTest {
// return encode_pos;
}
- public static int unpack8Values(
- byte[] encoded, int offset, int width, int value_pos, int[] result_list) {
- int byteIdx = offset;
- // int pos_encode = 0;
- long buffer = 0;
- // total bits which have read from 'buf' to 'buffer'. i.e.,
- // number of available bits to be decoded.
- int totalBits = 0;
- int valueIdx = 0;
-
- while (valueIdx < 8) {
- // If current available bits are not enough to decode one Integer,
- // then add next byte from buf to 'buffer' until totalBits >= width
- while (totalBits < width) {
- buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
- byteIdx++;
- totalBits += 8;
- }
-
- // If current available bits are enough to decode one Integer,
- // then decode one Integer one by one until left bits in 'buffer' is
- // not enough to decode one Integer.
- while (totalBits >= width && valueIdx < 8) {
- result_list[value_pos] = (int) (buffer >>> (totalBits - width));
- value_pos++;
- valueIdx++;
- totalBits -= width;
- buffer = buffer & ((1L << totalBits) - 1);
- }
- }
- return value_pos;
- }
public static int bitPacking(
int[][] numbers,
@@ -247,19 +144,6 @@ public class RegerPFloatTest {
return encode_pos;
}
- public static int decodeBitPacking(
- byte[] encoded, int decode_pos, int bit_width, int block_size, int[] result_list) {
- // int[] result_list = new int[];
- int block_num = block_size / 8;
- int value_pos = 0;
-
- for (int i = 0; i < block_num; i++) { // bitpacking
- value_pos = unpack8Values(encoded, decode_pos, bit_width, value_pos, result_list);
- decode_pos += bit_width;
- }
- // decode_pos_result.add(decode_pos);
- return decode_pos;
- }
public static void float2bytes(float f, int pos_encode, byte[] encode_result) {
int fbit = Float.floatToIntBits(f);
@@ -278,106 +162,6 @@ public class RegerPFloatTest {
}
}
- public static float bytes2float(ArrayList<Byte> b, int index) {
- int l;
- l = b.get(index);
- l &= 0xff;
- l |= ((long) b.get(index + 1) << 8);
- l &= 0xffff;
- l |= ((long) b.get(index + 2) << 16);
- l &= 0xffffff;
- l |= ((long) b.get(index + 3) << 24);
- return Float.intBitsToFloat(l);
- }
-
- public static float bytes2float(byte[] b, int index) {
- int l;
- l = b[index];
- l &= 0xff;
- l |= ((long) b[index + 1] << 8);
- l &= 0xffff;
- l |= ((long) b[index + 2] << 16);
- l &= 0xffffff;
- l |= ((long) b[index + 3] << 24);
- return Float.intBitsToFloat(l);
- }
-
- public static byte[] int2Bytes(int integer) {
- byte[] bytes = new byte[4];
- bytes[0] = (byte) (integer >> 24);
- bytes[1] = (byte) (integer >> 16);
- bytes[2] = (byte) (integer >> 8);
- bytes[3] = (byte) integer;
- return bytes;
- }
-
- public static byte[] bitWidth2Bytes(int integer) {
- byte[] bytes = new byte[1];
- bytes[0] = (byte) integer;
- return bytes;
- }
-
- public static byte[] float2bytes(float f) {
- int fbit = Float.floatToIntBits(f);
- byte[] b = new byte[4];
- for (int i = 0; i < 4; i++) {
- b[i] = (byte) (fbit >> (24 - i * 8));
- }
- int len = b.length;
- byte[] dest = new byte[len];
- System.arraycopy(b, 0, dest, 0, len);
- byte temp;
- for (int i = 0; i < len / 2; ++i) {
- temp = dest[i];
- dest[i] = dest[len - i - 1];
- dest[len - i - 1] = temp;
- }
- return dest;
- }
-
- public static byte[] bitPacking(
- ArrayList<ArrayList<Integer>> numbers, int index, int start, int block_num, int bit_width) {
- block_num = block_num / 8;
- byte[] result = new byte[bit_width * block_num];
-
- for (int i = 0; i < block_num; i++) {
- for (int j = 0; j < bit_width; j++) {
- int tmp_int = 0;
- for (int k = 0; k < 8; k++) {
- tmp_int += (((numbers.get(start + i * 8 + k).get(index) >> j) % 2) << k);
- }
- result[i * bit_width + j] = (byte) tmp_int;
- }
- }
- return result;
- }
-
- public static ArrayList<Integer> decodebitPacking(
- ArrayList<Byte> encoded, int decode_pos, int bit_width, int min_delta, int block_size) {
- ArrayList<Integer> result_list = new ArrayList<>();
- for (int i = 0; i < (block_size - 1) / 8; i++) {
- int[] val8 = new int[8];
- for (int j = 0; j < 8; j++) {
- val8[j] = 0;
- }
- for (int j = 0; j < bit_width; j++) {
- byte tmp_byte = encoded.get(decode_pos + bit_width - 1 - j);
- byte[] bit8 = new byte[8];
- for (int k = 0; k < 8; k++) {
- bit8[k] = (byte) (tmp_byte & 1);
- tmp_byte = (byte) (tmp_byte >> 1);
- }
- for (int k = 0; k < 8; k++) {
- val8[k] = val8[k] * 2 + bit8[k];
- }
- }
- for (int j = 0; j < 8; j++) {
- result_list.add(val8[j] + min_delta);
- }
- decode_pos += bit_width;
- }
- return result_list;
- }
public static int part(ArrayList<ArrayList<Integer>> arr, int index, int low, int high) {
ArrayList<Integer> tmp = arr.get(low);
@@ -414,7 +198,7 @@ public class RegerPFloatTest {
stack.push(low);
stack.push(mid - 1);
}
- while (stack.empty() == false) {
+ while (!stack.empty()) {
high = stack.pop();
low = stack.pop();
mid = part(arr, index, low, high);
@@ -448,10 +232,8 @@ public class RegerPFloatTest {
if (td_common == 0) {
if (time_diffi != 0) {
td_common = time_diffi;
- continue;
- } else {
- continue;
}
+ continue;
}
if (time_diffi != 0) {
td_common = getCommon(time_diffi, td_common);
@@ -483,124 +265,11 @@ public class RegerPFloatTest {
result.add(t0);
}
- public static void terminate(
- ArrayList<ArrayList<Integer>> ts_block, ArrayList<Float> coefficient, int p) {
- int length = ts_block.size();
- assert length > p;
- int size = length - p;
-
- double[] param;
- try {
- OLSMultipleLinearRegression ols1 = new OLSMultipleLinearRegression();
- double[][] X1 = new double[size][p];
- double[] Y1 = new double[size];
- for (int i = 0; i < size; i++) {
- X1[i] = new double[p];
- for (int j = 0; j < p; j++) {
- X1[i][j] = ts_block.get(i + j).get(0);
- }
- Y1[i] = ts_block.get(i + p).get(0);
- }
- ols1.newSampleData(Y1, X1);
- param = ols1.estimateRegressionParameters(); // 结果的第1项是常数项, 之后依次序为各个特征的系数
- // System.out.println(Arrays.toString(param));
- } catch (Exception e) {
- param = new double[p + 1];
- for (int i = 0; i <= p; i++) {
- param[i] = 0;
- }
- }
-
- double[] param2;
- try {
- OLSMultipleLinearRegression ols2 = new OLSMultipleLinearRegression();
- double[][] X2 = new double[size][p];
- double[] Y2 = new double[size];
- for (int i = 0; i < size; i++) {
- X2[i] = new double[p];
- for (int j = 0; j < p; j++) {
- X2[i][j] = ts_block.get(i + j).get(1);
- }
- Y2[i] = ts_block.get(i + p).get(1);
- }
- ols2.newSampleData(Y2, X2);
- param2 = ols2.estimateRegressionParameters(); // 结果的第1项是常数项, 之后依次序为各个特征的系数
- // System.out.println(Arrays.toString(param2));
- } catch (Exception exception) {
- param2 = new double[p + 1];
- for (int i = 0; i <= p; i++) {
- param2[i] = 0;
- }
- }
-
- for (int i = 0; i <= p; i++) {
- coefficient.add((float) param[i]);
- coefficient.add((float) param2[i]);
- }
- }
-
-// public static void terminate(int[][] ts_block, float[] coefficient, int p) {
-// int length = ts_block.length;
-// assert length > p;
-// int size = length - p;
-//
-// double[] param;
-// try {
-// OLSMultipleLinearRegression ols1 = new OLSMultipleLinearRegression();
-// double[][] X1 = new double[size][p];
-// double[] Y1 = new double[size];
-// for (int i = 0; i < size; i++) {
-// X1[i] = new double[p];
-// for (int j = 0; j < p; j++) {
-// X1[i][j] = ts_block[i + j][0];
-// }
-// Y1[i] = ts_block[i + p][0];
-// }
-// ols1.newSampleData(Y1, X1);
-// param = ols1.estimateRegressionParameters(); // 结果的第1项是常数项, 之后依次序为各个特征的系数
-// // System.out.println(Arrays.toString(param));
-// } catch (Exception e) {
-// param = new double[p + 1];
-// for (int i = 0; i <= p; i++) {
-// param[i] = 0;
-// }
-// }
-//
-// double[] param2;
-// try {
-// OLSMultipleLinearRegression ols2 = new OLSMultipleLinearRegression();
-// double[][] X2 = new double[size][p];
-// double[] Y2 = new double[size];
-// for (int i = 0; i < size; i++) {
-// X2[i] = new double[p];
-// for (int j = 0; j < p; j++) {
-// X2[i][j] = ts_block[i + j][1];
-// }
-// Y2[i] = ts_block[i + p][1];
-// }
-// ols2.newSampleData(Y2, X2);
-// param2 = ols2.estimateRegressionParameters(); // 结果的第1项是常数项, 之后依次序为各个特征的系数
-// // System.out.println(Arrays.toString(param2));
-// } catch (Exception exception) {
-// param2 = new double[p + 1];
-// for (int i = 0; i <= p; i++) {
-// param2[i] = 0;
-// }
-// }
-//
-// for (int i = 0; i <= p; i++) {
-// coefficient[2 * i] = (float) param[i];
-// coefficient[2 * i + 1] = (float) param2[i];
-// }
-// }
-
public static void terminate(int[][] ts_block, float[] coefficient, int p){
int length = ts_block.length;
assert length > p;
int size = length - p;
- OLSMultipleLinearRegression regression = new OLSMultipleLinearRegression();
-
RealMatrix matrix = new Array2DRowRealMatrix(size, p+1);
RealVector vector = new ArrayRealVector(size);
@@ -631,12 +300,6 @@ public class RegerPFloatTest {
DecompositionSolver solver1 = new SingularValueDecomposition(matrix).getSolver();
RealVector solution2 = solver1.solve(vector);
-// // 输出系数
-// System.out.println("自回归模型系数:");
-// for (int i = 0; i < solution.getDimension(); i++) {
-// System.out.println("AR(" + (i + 1) + "): " + solution.getEntry(i));
-// }
-
for (int i = 0; i <= p; i++) {
coefficient[2 * i] = (float) solution1.getEntry(p-i);
@@ -722,19 +385,13 @@ public class RegerPFloatTest {
public static int getBetaP(
int[][] ts_block, int alpha, int block_size, int[] raw_length, float[] theta, int p) {
- int timestamp_delta_min = Integer.MAX_VALUE;
- int value_delta_min = Integer.MAX_VALUE;
- int max_timestamp = Integer.MIN_VALUE;
- int max_value = Integer.MIN_VALUE;
- int raw_timestamp_delta_max_index = -1;
- int raw_value_delta_max_index = -1;
int raw_abs_sum = raw_length[0];
int range = block_size / 32;
ArrayList<Integer> j_star_list = new ArrayList<>(); // beta list of min b phi alpha to j
-// ArrayList<Integer> max_index = new ArrayList<>();
+
int j_star = -1;
@@ -1000,7 +657,6 @@ public class RegerPFloatTest {
private static int[] adjustCase1(int[][] ts_block, int alpha, int j_star, float[] theta, int p) {
int[][] tmp_ts_block = new int[ts_block.length][2];
cloneTsblock(ts_block,tmp_ts_block);
- int block_size = ts_block.length;
int[] delta_index = new int[4];
int[] delta = new int[4];
@@ -1036,7 +692,6 @@ public class RegerPFloatTest {
private static int[] adjustCase2(int[][] ts_block, int alpha, int j_star, float[] theta, int p) {
int[][] tmp_ts_block = new int[ts_block.length][2];
cloneTsblock(ts_block,tmp_ts_block);
- int block_size = ts_block.length;
int[] tmp_tv = tmp_ts_block[alpha].clone();
for (int u = alpha - 1; u >= j_star; u--) {
tmp_ts_block[u + 1][0] = tmp_ts_block[u][0];
@@ -1047,7 +702,7 @@ public class RegerPFloatTest {
int[] delta_index = new int[4];
int[] delta = new int[4];
- int[][] ts_block_delta = new int[block_size - p][2];
+ int[][] ts_block_delta;
ts_block_delta =deltaTSBlock(tmp_ts_block, delta_index,delta, theta, p);
int timestamp_delta_min = delta[2];
@@ -1067,7 +722,6 @@ public class RegerPFloatTest {
private static int[] adjustCase3(int[][] ts_block, int alpha, int j_star, float[] theta, int p) {
int[][] tmp_ts_block = new int[ts_block.length][2];
cloneTsblock(ts_block,tmp_ts_block);
- int block_size = ts_block.length;
int[] tmp_tv = tmp_ts_block[alpha].clone();
for (int u = alpha + 1; u < j_star; u++) {
tmp_ts_block[u - 1][0] = tmp_ts_block[u][0];
@@ -1079,7 +733,7 @@ public class RegerPFloatTest {
int[] delta_index = new int[4];
int[] delta = new int[4];
- int[][] ts_block_delta = new int[block_size - p][2];
+ int[][] ts_block_delta;
ts_block_delta =deltaTSBlock(tmp_ts_block, delta_index,delta, theta, p);
int timestamp_delta_min = delta[2];
@@ -1099,7 +753,6 @@ public class RegerPFloatTest {
private static int[] adjustCase4(int[][] ts_block, int alpha, int j_star, float[] theta, int p) {
int[][] tmp_ts_block = new int[ts_block.length][2];
cloneTsblock(ts_block,tmp_ts_block);
- int block_size = ts_block.length;
int[] tmp_tv = tmp_ts_block[alpha].clone();
for (int u = alpha + 1; u < j_star; u++) {
tmp_ts_block[u - 1][0] = tmp_ts_block[u][0];
@@ -1111,7 +764,7 @@ public class RegerPFloatTest {
int[] delta_index = new int[4];
int[] delta = new int[4];
- int[][] ts_block_delta = new int[block_size - p][2];
+ int[][] ts_block_delta;
ts_block_delta =deltaTSBlock(tmp_ts_block, delta_index,delta, theta, p);
int timestamp_delta_min = delta[2];
@@ -1141,7 +794,7 @@ public class RegerPFloatTest {
tmp_ts_block[block_size - 1][1] = tmp_tv[1];
int[] delta_index = new int[4];
int[] delta = new int[4];
- int[][] ts_block_delta = new int[block_size - p][2];
+ int[][] ts_block_delta;
ts_block_delta =deltaTSBlock(tmp_ts_block, delta_index,delta, theta, p);
int timestamp_delta_min = delta[2];
@@ -1162,16 +815,11 @@ public class RegerPFloatTest {
public static int encodeRLEBitWidth2Bytes(int[][] bit_width_segments) {
int encoded_result = 0;
- // ArrayList<ArrayList<Integer>> run_length_time = new ArrayList<>();
- // ArrayList<ArrayList<Integer>> run_length_value = new ArrayList<>();
-
int count_of_time = 1;
int count_of_value = 1;
int pre_time = bit_width_segments[0][0];
int pre_value = bit_width_segments[0][1];
int size = bit_width_segments.length;
- int[][] run_length_time = new int[size][2];
- int[][] run_length_value = new int[size][2];
int pos_time = 0;
int pos_value = 0;
@@ -1180,8 +828,7 @@ public class RegerPFloatTest {
int cur_time = bit_width_segments[i][0];
int cur_value = bit_width_segments[i][1];
if (cur_time != pre_time && count_of_time != 0) {
- run_length_time[pos_time][0] = count_of_time;
- run_length_time[pos_time][1] = pre_time;
+
pos_time++;
pre_time = cur_time;
count_of_time = 1;
@@ -1189,16 +836,14 @@ public class RegerPFloatTest {
count_of_time++;
pre_time = cur_time;
if (count_of_time == 256) {
- run_length_time[pos_time][0] = count_of_time;
- run_length_time[pos_time][1] = pre_time;
+
pos_time++;
count_of_time = 1;
}
}
if (cur_value != pre_value && count_of_value != 0) {
- run_length_value[pos_value][0] = count_of_value;
- run_length_value[pos_value][1] = pre_value;
+
pos_value++;
pre_value = cur_value;
@@ -1207,21 +852,15 @@ public class RegerPFloatTest {
count_of_value++;
pre_value = cur_value;
if (count_of_value == 256) {
- run_length_value[pos_value][0] = count_of_value;
- run_length_value[pos_value][1] = pre_value;
pos_value++;
count_of_value = 0;
}
}
}
if (count_of_time != 0) {
- run_length_time[pos_time][0] = count_of_time;
- run_length_time[pos_time][1] = pre_time;
pos_time++;
}
if (count_of_value != 0) {
- run_length_value[pos_value][0] = count_of_value;
- run_length_value[pos_value][1] = pre_value;
pos_value++;
}
@@ -1297,11 +936,7 @@ public class RegerPFloatTest {
pos_encode += 2;
intWord2Bytes(pos_value, pos_encode, encoded_result);
pos_encode += 2;
- // System.out.println("pos_time="+pos_time);
- // System.out.println("pos_value="+pos_value);
- // System.out.println(Arrays.deepToString(run_length_time));
- // System.out.println(Arrays.deepToString(run_length_value));
for (int i = 0; i < pos_time; i++) {
int[] bit_width_time = run_length_time[i];
intByte2Bytes(bit_width_time[0], pos_encode, encoded_result);
@@ -1309,8 +944,6 @@ public class RegerPFloatTest {
intByte2Bytes(bit_width_time[1], pos_encode, encoded_result);
pos_encode++;
- // System.out.println("bit_width_time[0]="+bit_width_time[0]);
- // System.out.println("bit_width_time[1]="+bit_width_time[1]);
}
for (int i = 0; i < pos_value; i++) {
int[] bit_width_value = run_length_value[i];
@@ -1319,8 +952,6 @@ public class RegerPFloatTest {
intByte2Bytes(bit_width_value[1], pos_encode, encoded_result);
pos_encode++;
- // System.out.println("bit_width_value[0]="+bit_width_value[0]);
- // System.out.println("bit_width_value[1]="+bit_width_value[1]);
}
return pos_encode;
@@ -1331,7 +962,6 @@ public class RegerPFloatTest {
int[][] bit_width_segments,
int[] raw_length,
int segment_size,
- int p,
float[] theta,
int pos_encode,
byte[] encoded_result) {
@@ -1357,12 +987,6 @@ public class RegerPFloatTest {
int2Bytes(raw_length[4], pos_encode, encoded_result);
pos_encode += 4;
- // System.out.println(delta_segments[0][0]);
- // System.out.println(delta_segments[0][1]);
- // System.out.println(theta[0] + raw_length[3]);
- // System.out.println(theta[1]);
- // System.out.println(theta[2] + raw_length[4]);
- // System.out.println(theta[3]);
pos_encode = encodeRLEBitWidth2Bytes(bit_width_segments, pos_encode, encoded_result);
for (int segment_i = 0; segment_i < segment_n; segment_i++) {
@@ -1393,7 +1017,7 @@ public class RegerPFloatTest {
public static int[][] getEncodeBitsRegressionNoTrain(
- int[][] ts_block, int block_size, int[] raw_length, float[] theta, int segment_size, int p) {
+ int[][] ts_block, int block_size, int[] raw_length, float[] theta, int p) {
int timestamp_delta_min = Integer.MAX_VALUE;
int value_delta_min = Integer.MAX_VALUE;
int[][] ts_block_delta = new int[ts_block.length][2];
@@ -1473,7 +1097,6 @@ public class RegerPFloatTest {
int[][] bit_width_segments,
int segment_size,
int p) {
- ArrayList<Byte> encoded_result = new ArrayList<>();
int block_size = delta_segments.length;
int segment_n = (block_size - p) / segment_size;
int result = 0;
@@ -1543,8 +1166,8 @@ public class RegerPFloatTest {
float epsilon_v = (float) ts_block[j][1] - theta[1];
for (int pi = 1; pi <= p; pi++) {
- epsilon_r -= (float) (theta[2 * pi] * (float) ts_block[j - pi][0]);
- epsilon_v -= (float) (theta[2 * pi + 1] * (float) ts_block[j - pi][1]);
+ epsilon_r -= theta[2 * pi] * (float) ts_block[j - pi][0];
+ epsilon_v -= theta[2 * pi + 1] * (float) ts_block[j - pi][1];
}
ts_block_delta[j][0] = (int) epsilon_r;
ts_block_delta[j][1] = (int) epsilon_v;
@@ -1582,7 +1205,6 @@ public class RegerPFloatTest {
public static int[] getIStar(
int[][] ts_block,
ArrayList<Integer> min_index,
- int block_size,
int index,
float[] theta,
int p) {
@@ -1606,7 +1228,7 @@ public class RegerPFloatTest {
}
public static int[] getIStar(
- int[][] ts_block, ArrayList<Integer> min_index, int block_size, float[] theta, int p, int k) {
+ int[][] ts_block, ArrayList<Integer> min_index, float[] theta, int p) {
int[] alpha_list = new int[4];
@@ -1647,7 +1269,6 @@ public class RegerPFloatTest {
int supply_length,
int[] third_value,
int segment_size,
- int k,
int p,
int encode_pos,
byte[] cur_byte) {
@@ -1659,11 +1280,8 @@ public class RegerPFloatTest {
ts_block = new int[block_size][2];
ts_block_partition = new int[block_size][2];
for (int j = 0; j < block_size; j++) {
-// data[j + i * block_size][0] -= min_time;
ts_block[j][0] = data[j + i * block_size][0]-min_time;
ts_block[j][1] = data[j + i * block_size][1];
- // ts_block_reorder[j][0] = data[j + i * block_size][0];
- // ts_block_reorder[j][1] = data[j + i * block_size][1];
}
} else {
ts_block = new int[supply_length][2];
@@ -1682,8 +1300,6 @@ public class RegerPFloatTest {
}
int2Bytes(min_time,encode_pos,cur_byte);
encode_pos += 4;
- // System.out.println(Arrays.deepToString(data));
- // System.out.println(Arrays.deepToString(ts_block));
int[] reorder_length = new int[5];
float[] theta_reorder = new float[2 * p + 2];
@@ -1693,28 +1309,16 @@ public class RegerPFloatTest {
int[] raw_length =
new int[5]; // length,max_bit_width_interval,max_bit_width_value,max_bit_width_deviation
float[] theta = new float[2 * p + 2];
- int[][] ts_block_delta_reorder = new int[block_size][2];
- int[][] bit_width_segments_value;
- int[][] ts_block_delta_time = new int[block_size][2];
- int[][] bit_width_segments_time;
- int[][] ts_block_delta = new int[block_size][2];
- int[][] bit_width_segments_partition;
+ int[][] ts_block_delta_reorder;
+ int[][] ts_block_delta_time;
+ int[][] ts_block_delta;
terminate(ts_block, theta_time, p);
ts_block_delta_time =
- getEncodeBitsRegressionNoTrain(ts_block, block_size, time_length, theta_time, segment_size, p);
-// System.out.println(Arrays.deepToString(ts_block));
- bit_width_segments_time = segmentBitPacking(ts_block_delta_time, block_size, segment_size);
-// time_length[0] =
-// numberOfEncodeSegment2Bytes(
-// ts_block_delta_time,
-// bit_width_segments_time,
-// segment_size,
-// p
-// );
-// System.out.println((Arrays.toString(theta_time)));
-//// System.out.println((Arrays.deepToString(ts_block_delta)));
-// System.out.println((Arrays.toString(time_length)));
+ getEncodeBitsRegressionNoTrain(ts_block, block_size, time_length, theta_time, p);
+
+ segmentBitPacking(ts_block_delta_time, block_size, segment_size);
+
int pos_ts_block_partition = 0;
if (third_value.length > 0) {
@@ -1749,14 +1353,10 @@ public class RegerPFloatTest {
terminate(ts_block_partition, theta, p);
ts_block_delta =
getEncodeBitsRegressionNoTrain(
- ts_block_partition, block_size, raw_length, theta, segment_size, p);
- bit_width_segments_partition = segmentBitPacking(ts_block_delta, block_size, segment_size);
+ ts_block_partition, block_size, raw_length, theta, p);
+ segmentBitPacking(ts_block_delta, block_size, segment_size);
-// raw_length[0] =
-// numberOfEncodeSegment2Bytes(
-// ts_block_delta, bit_width_segments_partition, segment_size, p);
-
Arrays.sort(
ts_block,
(a, b) -> {
@@ -1766,40 +1366,17 @@ public class RegerPFloatTest {
terminate(ts_block, theta_reorder, p);
ts_block_delta_reorder =
getEncodeBitsRegressionNoTrain(
- ts_block, block_size, reorder_length, theta_reorder, segment_size, p);
- bit_width_segments_value =
- segmentBitPacking(ts_block_delta_reorder, block_size, segment_size);
-// reorder_length[0] =
-// numberOfEncodeSegment2Bytes(
-// ts_block_delta_reorder,
-// bit_width_segments_value,
-// segment_size,
-// p
-// );
- // System.out.println(Arrays.deepToString(ts_block_delta));
+ ts_block, block_size, reorder_length, theta_reorder, p);
+ segmentBitPacking(ts_block_delta_reorder, block_size, segment_size);
+
int[] alpha_list;
int choose = min3(time_length[0], raw_length[0], reorder_length[0]);
ArrayList<Integer> min_index = new ArrayList<>();
int index_alpha_list = 0;
-// if (time_length[0] < raw_length[0]) {
-// raw_length = time_length.clone();
-//// System.out.println("time");
-// Arrays.sort(
-// ts_block,
-// (a, b) -> {
-// if (a[0] == b[0]) return Integer.compare(a[1], b[1]);
-// return Integer.compare(a[0], b[0]);
-// });
-// theta = theta_time.clone();
-// } else {
-// ts_block = ts_block_partition.clone();
-//
-// }
if (choose == 0) {
raw_length = time_length.clone();
-// System.out.println("time");
Arrays.sort(
ts_block,
(a, b) -> {
@@ -1809,10 +1386,9 @@ public class RegerPFloatTest {
theta = theta_time.clone();
} else if (choose == 1) {
ts_block = ts_block_partition.clone();
-// System.out.println("partition");
+
} else {
raw_length = reorder_length.clone();
-// System.out.println("value");
theta = theta_reorder.clone();
Arrays.sort(
ts_block,
@@ -1823,16 +1399,13 @@ public class RegerPFloatTest {
index_alpha_list = 1;
}
-// System.out.println(Arrays.toString(theta));
-// System.out.println(Arrays.toString(raw_length));
- alpha_list = getIStar(ts_block, min_index, block_size, index_alpha_list, theta, p);
+ alpha_list = getIStar(ts_block, min_index, index_alpha_list, theta, p);
int[] beta_list = new int[alpha_list.length];
int[][] new_length_list = new int[alpha_list.length][5];
int pos_new_length_list = 0;
- // ArrayList<ArrayList<Integer>> new_length_list = new ArrayList<>();
for (int j = 0; j < alpha_list.length; j++) {
int alpha = alpha_list[j];
@@ -1867,7 +1440,7 @@ public class RegerPFloatTest {
beta_list[all_length.get(0).get(0)]);
int[] new_length = new int[5];
ts_block_delta =
- getEncodeBitsRegressionNoTrain(new_ts_block, block_size, new_length, theta, segment_size, p);
+ getEncodeBitsRegressionNoTrain(new_ts_block, block_size, new_length, theta, p);
int[][] bit_width_segments = segmentBitPacking(ts_block_delta, block_size, segment_size);
new_length[0] =
numberOfEncodeSegment2Bytes(
@@ -1883,7 +1456,7 @@ public class RegerPFloatTest {
} else {
break;
}
- alpha_list = getIStar(ts_block, min_index, block_size, theta, p, k);
+ alpha_list = getIStar(ts_block, min_index, theta, p);
int alpha_size = alpha_list.length;
for (int alpha_i = alpha_size - 1; alpha_i >= 0; alpha_i--) {
@@ -1906,16 +1479,12 @@ public class RegerPFloatTest {
isMoveable = isMovable(alpha_list, beta_list);
}
- // System.out.println("getEncodeBitsRegressionNoTrain before:" +
- // Arrays.deepToString(ts_block_delta));
+
ts_block_delta =
- getEncodeBitsRegressionNoTrain(ts_block, block_size, raw_length, theta, segment_size, p);
+ getEncodeBitsRegressionNoTrain(ts_block, block_size, raw_length, theta, p);
- // System.out.println("getEncodeBitsRegressionNoTrain after:"
- // +Arrays.deepToString(ts_block_delta));
- // ArrayList<ArrayList<Integer>> bit_width_segments = new ArrayList<>();
int segment_n = (block_size - 1) / segment_size;
int[][] bit_width_segments = new int[segment_n][2];
for (int segment_i = 0; segment_i < segment_n; segment_i++) {
@@ -1939,8 +1508,7 @@ public class RegerPFloatTest {
bit_width_segments[segment_i][1] = bit_width_value;
}
- // System.out.println(Arrays.deepToString(ts_block_delta));
- // System.out.println(Arrays.deepToString(bit_width_segments));
+
encode_pos =
encodeSegment2Bytes(
@@ -1948,12 +1516,11 @@ public class RegerPFloatTest {
bit_width_segments,
raw_length,
segment_size,
- p,
- theta,
+ theta,
encode_pos,
cur_byte);
- // System.out.println("encode_pos="+encode_pos);
+
return encode_pos;
}
@@ -1963,9 +1530,8 @@ public class RegerPFloatTest {
int[] third_value,
int segment_size,
int p,
- int k,
byte[] encoded_result) {
-// for (int i = 0; i < p; i++)
+
block_size++;
int length_all = data.length;
@@ -1984,139 +1550,7 @@ public class RegerPFloatTest {
intByte2Bytes(p, encode_pos, encoded_result);
encode_pos += 1;
- // ----------------------- compare data order by time, value and partition
- // ---------------------------
- int length_time = 0;
- int length_value = 0;
- int length_partition = 0;
-// int[][] data_value = data.clone();
-// Arrays.sort(
-// data_value,
-// (a, b) -> {
-// if (a[1] == b[1]) return Integer.compare(a[0], b[0]);
-// return Integer.compare(a[1], b[1]);
-// });
-//
-// int[][] data_partition = new int[length_all][2];
-// int pos_data_partition = 0;
-//
-// for (int[] datum : data) {
-// if (datum[1] > third_value[third_value.length - 1]) {
-// data_partition[pos_data_partition][0] = datum[0];
-// data_partition[pos_data_partition][1] = datum[1];
-// pos_data_partition++;
-// }
-// }
-// for (int third_i = third_value.length - 1; third_i > 0; third_i--) {
-// for (int[] datum : data) {
-// if (datum[1] <= third_value[third_i] && datum[1] > third_value[third_i - 1]) {
-// data_partition[pos_data_partition][0] = datum[0];
-// data_partition[pos_data_partition][1] = datum[1];
-// pos_data_partition++;
-// }
-// }
-// }
-// for (int[] datum : data) {
-// if (datum[1] <= third_value[0]) {
-// data_partition[pos_data_partition][0] = datum[0];
-// data_partition[pos_data_partition][1] = datum[1];
-// pos_data_partition++;
-// }
-// }
-// for (int i = 0; i < block_num; i++) {
-// int[][] ts_block_time = new int[block_size][2];
-// int[][] ts_block_value = new int[block_size][2];
-// int[][] ts_block_partition = new int[block_size][2];
-//
-// for (int j = 0; j < block_size; j++) {
-// ts_block_time[j][0] = data[j + i * block_size][0];
-// ts_block_time[j][1] = data[j + i * block_size][1];
-// ts_block_value[j][0] = data_value[j + i * block_size][0];
-// ts_block_value[j][1] = data_value[j + i * block_size][1];
-// ts_block_partition[j][0] = data_partition[j + i * block_size][0];
-// ts_block_partition[j][1] = data_partition[j + i * block_size][1];
-// }
-//
-// int[] raw_length = new int[5];
-// float[] theta = new float[2 * p + 2];
-// int[][] ts_block_delta =
-// getEncodeBitsRegression(ts_block_time, block_size, raw_length, theta, segment_size, p);
-// int[][] bit_width_segments = segmentBitPacking(ts_block_delta, block_size, segment_size);
-// length_time +=
-// numberOfEncodeSegment2Bytes(
-// ts_block_delta, bit_width_segments, segment_size, p);
-//
-// int[] raw_length_value = new int[5];
-// float[] theta_value = new float[2 * p + 2];
-// int[][] ts_block_delta_value =
-// getEncodeBitsRegression(
-// ts_block_value, block_size, raw_length_value, theta_value, segment_size, p);
-// int[][] bit_width_segments_value =
-// segmentBitPacking(ts_block_delta_value, block_size, segment_size);
-// length_value +=
-// numberOfEncodeSegment2Bytes(
-// ts_block_delta_value,
-// bit_width_segments_value,
-// segment_size,
-// p
-// );
-//
-// int[] raw_length_partition = new int[5];
-// float[] theta_partition = new float[2 * p + 2];
-// int[][] ts_block_delta_partition =
-// getEncodeBitsRegression(
-// ts_block_partition,
-// block_size,
-// raw_length_partition,
-// theta_partition,
-// segment_size,
-// p);
-// int[][] bit_width_segments_partition =
-// segmentBitPacking(ts_block_delta_partition, block_size, segment_size);
-// length_partition +=
-// numberOfEncodeSegment2Bytes(
-// ts_block_delta_partition,
-// bit_width_segments_partition,
-// segment_size,
-// p
-// );
-// }
-
-// if (length_partition < length_time
-// && length_partition < length_value) { // partition performs better
-// data = data_partition;
-// // for (int i = 0; i < 2; i++) {
-// System.out.println("Partition");
-// for (int i = 0; i < block_num; i++) {
-// encode_pos =
-// REGERBlockEncoderPartition(
-// data, i, block_size, segment_size, k, p, encode_pos, encoded_result);
-// }
-// } else {
-// if (length_value < length_time) { // order by value performs better
-// System.out.println("Value");
-// data = data_value;
-//// for (int i = 0; i < 1; i++) {
-// for (int i = 0; i < block_num; i++) {
-//
-// encode_pos =
-// REGERBlockEncoder(
-// data,
-// 1,
-// i,
-// block_size,
-// 0,
-// third_value,
-// segment_size,
-// k,
-// p,
-// encode_pos,
-// encoded_result);
-// }
-// } else {
-// System.out.println("Time");
- // for (int i = 0; i < 2; i++) {
-// for (int i = 0; i < 1; i++) {
+
for (int i = 0; i < block_num; i++) {
encode_pos =
REGERBlockEncoder(
@@ -2126,13 +1560,11 @@ public class RegerPFloatTest {
0,
third_value,
segment_size,
- k,
- p,
+ p,
encode_pos,
encoded_result);
}
-// }
-// }
+
int remaining_length = length_all - block_num * block_size;
if (remaining_length == 1) {
@@ -2163,8 +1595,7 @@ public class RegerPFloatTest {
supple_length + remaining_length,
third_value,
segment_size,
- k,
- p,
+ p,
encode_pos,
encoded_result);
}
@@ -2172,256 +1603,6 @@ public class RegerPFloatTest {
return encode_pos;
}
- public static double bytes2Double(ArrayList<Byte> encoded, int start, int num) {
- if (num > 8) {
- System.out.println("bytes2Doubleerror");
- return 0;
- }
- long value = 0;
- for (int i = 0; i < 8; i++) {
- value |= ((long) (encoded.get(i + start) & 0xff)) << (8 * i);
- }
- return Double.longBitsToDouble(value);
- }
-
- public static float byte2float2(ArrayList<Byte> b, int index) {
- int l;
- l = b.get(index);
- l &= 0xff;
- l |= ((long) b.get(index + 1) << 8);
- l &= 0xffff;
- l |= ((long) b.get(index + 2) << 16);
- l &= 0xffffff;
- l |= ((long) b.get(index + 3) << 24);
- return Float.intBitsToFloat(l);
- }
-
- public static int bytes2Integer(ArrayList<Byte> encoded, int start, int num) {
- int value = 0;
- if (num > 4) {
- System.out.println("bytes2Integer error");
- return 0;
- }
- for (int i = 0; i < num; i++) {
- value <<= 8;
- int b = encoded.get(i + start) & 0xFF;
- value |= b;
- }
- return value;
- }
-
- public static double[] dataStandardization(double[] array) {
- StandardDeviation deviation = new StandardDeviation();
- for (int i = 0; i < array.length; i++) {
- array[i] = array[i];
- }
- return array;
- }
-
- public static double[][] dataStandardizationDouble(double[][] arrays) {
- double[][] result = new double[arrays[0].length][arrays.length];
- for (int i = 0; i < arrays.length; i++) {
- double[] doubles = dataStandardization(arrays[i]);
- for (int k = 0; k < result.length; k++) {
- result[k][i] = doubles[k];
- }
- }
- return result;
- }
-
-// @Test
-// public void REGERPFloat() throws IOException {
-//
-// String parent_dir =
-// "/Users/xiaojinzhao/Documents/GitHub/iotdb/iotdb-core/tsfile/src/test/resources/";
-// String output_parent_dir =
-// "/Users/xiaojinzhao/Documents/GitHub/encoding-reorder/compression_ratio/p_float/";
-// String input_parent_dir = parent_dir + "trans_data/";
-//
-// ArrayList<String> input_path_list = new ArrayList<>();
-// ArrayList<String> output_path_list = new ArrayList<>();
-// ArrayList<String> dataset_name = new ArrayList<>();
-// ArrayList<Integer> dataset_block_size = new ArrayList<>();
-//
-// ArrayList<int[]> dataset_third = new ArrayList<>();
-//
-// dataset_name.add("CS-Sensors");
-// dataset_name.add("Metro-Traffic");
-// dataset_name.add("USGS-Earthquakes");
-// dataset_name.add("YZ-Electricity");
-// dataset_name.add("GW-Magnetic");
-// dataset_name.add("TY-Fuel");
-// dataset_name.add("Cyber-Vehicle");
-// dataset_name.add("Vehicle-Charge");
-// dataset_name.add("Nifty-Stocks");
-// dataset_name.add("TH-Climate");
-// dataset_name.add("TY-Transport");
-// dataset_name.add("EPM-Education");
-//
-// int[] dataset_0 = {547, 2816};
-// int[] dataset_1 = {1719, 3731};
-// int[] dataset_2 = {-48, -11, 6, 25, 52};
-// int[] dataset_3 = {8681, 13584};
-// int[] dataset_4 = {79, 184, 274};
-// int[] dataset_5 = {17, 68};
-// int[] dataset_6 = {677};
-// int[] dataset_7 = {1047, 1725};
-// int[] dataset_8 = {227, 499, 614, 1013};
-// int[] dataset_9 = {474, 678};
-// int[] dataset_10 = {4, 30, 38, 49, 58};
-// int[] dataset_11 = {5182, 8206};
-//
-// dataset_third.add(dataset_0);
-// dataset_third.add(dataset_1);
-// dataset_third.add(dataset_2);
-// dataset_third.add(dataset_3);
-// dataset_third.add(dataset_4);
-// dataset_third.add(dataset_5);
-// dataset_third.add(dataset_6);
-// dataset_third.add(dataset_7);
-// dataset_third.add(dataset_8);
-// dataset_third.add(dataset_9);
-// dataset_third.add(dataset_10);
-// dataset_third.add(dataset_11);
-//
-// for (int i = 0; i < dataset_name.size(); i++) {
-// input_path_list.add(input_parent_dir + dataset_name.get(i));
-// dataset_block_size.add(1024);
-// }
-//
-// output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
-// // dataset_block_size.add(1024);
-//
-// output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv"); // 1
-// // dataset_block_size.add(512);
-// output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv"); // 2
-// // dataset_block_size.add(512);
-// output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
-// // dataset_block_size.add(256);
-// output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); // 4
-// // dataset_block_size.add(128);
-// output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv"); // 5
-// // dataset_block_size.add(64);
-// output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); // 6
-// // dataset_block_size.add(128);
-// output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv"); // 7
-// // dataset_block_size.add(512);
-// output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv"); // 8
-// // dataset_block_size.add(256);
-// output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv"); // 9
-// // dataset_block_size.add(512);
-// output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv"); // 10
-// // dataset_block_size.add(512);
-// output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv"); // 11
-// // dataset_block_size.add(512);
-//
-// for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
-// // for (int file_i = 3; file_i < 4; file_i++) {
-//
-// String inputPath = input_path_list.get(file_i);
-// String Output = output_path_list.get(file_i);
-// System.out.println(inputPath);
-//
-// // speed
-// int repeatTime = 1; // set repeat time
-//
-// File file = new File(inputPath);
-// File[] tempList = file.listFiles();
-//
-// CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
-//
-// String[] head = {
-// "Input Direction",
-// "Encoding Algorithm",
-// "Encoding Time",
-// "Decoding Time",
-// "Points",
-// "p",
-// "Compressed Size",
-// "Compression Ratio"
-// };
-// writer.writeRecord(head); // write header to output file
-//
-// assert tempList != null;
-//
-// for (File f : tempList) {
-// System.out.println(f);
-// for (int p = 2; p < 3; p++) {
-// // for (int p = 1; p < 10; p++) {
-// System.out.println("p=" + p);
-//
-// InputStream inputStream = Files.newInputStream(f.toPath());
-// CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
-// ArrayList<ArrayList<Integer>> data = new ArrayList<>();
-// ArrayList<ArrayList<Integer>> data_decoded = new ArrayList<>();
-//
-// // add a column to "data"
-// loader.readHeaders();
-// data.clear();
-// while (loader.readRecord()) {
-// ArrayList<Integer> tmp = new ArrayList<>();
-// tmp.add(Integer.valueOf(loader.getValues()[0]));
-// tmp.add(Integer.valueOf(loader.getValues()[1]));
-// data.add(tmp);
-// }
-// inputStream.close();
-// ArrayList<Integer> result2 = new ArrayList<>();
-// splitTimeStamp3(data, result2);
-// int[][] data2_arr = new int[data.size()][2];
-// int min_time = data.get(0).get(0);
-// for (int i = 0; i < data.size(); i++) {
-// data2_arr[i][0] = data.get(i).get(0) - min_time;
-// data2_arr[i][1] = data.get(i).get(1);
-// }
-// System.out.println(data2_arr[0][0]);
-// byte[] encoded_result = new byte[data2_arr.length * 8];
-//
-// long encodeTime = 0;
-// long decodeTime = 0;
-// double ratio = 0;
-// double compressed_size = 0;
-// int length = 0;
-//
-// long s = System.nanoTime();
-//
-// for (int repeat_i = 0; repeat_i < 1; repeat_i++)
-// length =
-// ReorderingRegressionEncoder(
-// data2_arr,
-// dataset_block_size.get(file_i),
-// dataset_third.get(file_i),
-// 8,
-// p,
-// 1,
-// encoded_result);
-//
-// long e = System.nanoTime();
-// encodeTime += ((e - s));
-// compressed_size += length;
-// double ratioTmp = (double) compressed_size / (double) (data.size() * Integer.BYTES * 2);
-// ratio += ratioTmp;
-// s = System.nanoTime();
-// e = System.nanoTime();
-// decodeTime += ((e - s));
-//
-// String[] record = {
-// f.toString(),
-// "REGER-32-FLOAT",
-// String.valueOf(encodeTime),
-// String.valueOf(decodeTime),
-// String.valueOf(data.size()),
-// String.valueOf(p),
-// String.valueOf(compressed_size),
-// String.valueOf(ratio)
-// };
-//
-// writer.writeRecord(record);
-// System.out.println(ratio);
-// }
-// }
-// writer.close();
-// }
-// }
@Test
public void REGERPFloatVaryP() throws IOException {
@@ -2485,52 +1666,33 @@ public class RegerPFloatTest {
dataset_third.add(dataset_12);
dataset_third.add(dataset_13);
- for (int i = 0; i < dataset_name.size(); i++) {
- input_path_list.add(input_parent_dir + dataset_name.get(i));
+ for (String value : dataset_name) {
+ input_path_list.add(input_parent_dir + value);
dataset_block_size.add(512);
}
output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
- // dataset_block_size.add(1024);
-
- output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv"); // 1
- // dataset_block_size.add(512);
- output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv"); // 2
- // dataset_block_size.add(512);
+ output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
+ output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
- // dataset_block_size.add(256);
- output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); // 4
- // dataset_block_size.add(128);
- output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv"); // 5
- // dataset_block_size.add(64);
- output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); // 6
- // dataset_block_size.add(128);
- output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv"); // 7
- // dataset_block_size.add(512);
- output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv"); // 8
- // dataset_block_size.add(256);
- output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv"); // 9
- // dataset_block_size.add(512);
- output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv"); // 10
- // dataset_block_size.add(512);
- output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv"); // 11
- // dataset_block_size.add(512);
+ output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
+ output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
+ output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
+ output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
+ output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
+ output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
+ output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
+ output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
output_path_list.add(output_parent_dir + "/FANYP-Sensors_ratio.csv"); // 12
output_path_list.add(output_parent_dir + "/TRAJET-Transport_ratio.csv"); // 13
- // 0 2 6 7
-// int[] file_lists = {1,9};
-// for (int file_i : file_lists) {
for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
-// for (int file_i = 12; file_i < 14; file_i++) {
-//
+
String inputPath = input_path_list.get(file_i);
String Output = output_path_list.get(file_i);
System.out.println(inputPath);
- // speed
- int repeatTime = 1; // set repeat time
File file = new File(inputPath);
File[] tempList = file.listFiles();
@@ -2556,18 +1718,16 @@ public class RegerPFloatTest {
System.out.println(count_csv);
count_csv ++;
System.out.println(f);
-// for (int p = 1; p < 2; p++) {
+
for (int p = 1; p < 10; p++) {
System.out.println("p=" + p);
InputStream inputStream = Files.newInputStream(f.toPath());
CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
ArrayList<ArrayList<Integer>> data = new ArrayList<>();
- ArrayList<ArrayList<Integer>> data_decoded = new ArrayList<>();
// add a column to "data"
loader.readHeaders();
- data.clear();
while (loader.readRecord()) {
ArrayList<Integer> tmp = new ArrayList<>();
tmp.add(Integer.valueOf(loader.getValues()[0]));
@@ -2583,7 +1743,7 @@ public class RegerPFloatTest {
data2_arr[i][0] = data.get(i).get(0) - min_time;
data2_arr[i][1] = data.get(i).get(1);
}
-// System.out.println(data2_arr[0][0]);
+
byte[] encoded_result = new byte[data2_arr.length * 8];
long encodeTime = 0;
@@ -2602,13 +1762,12 @@ public class RegerPFloatTest {
dataset_third.get(file_i),
16,
p,
- 1,
- encoded_result);
+ encoded_result);
long e = System.nanoTime();
encodeTime += ((e - s));
compressed_size += length;
- double ratioTmp = (double) compressed_size / (double) (data.size() * Integer.BYTES * 2);
+ double ratioTmp = compressed_size / (double) (data.size() * Integer.BYTES * 2);
ratio += ratioTmp;
s = System.nanoTime();
e = System.nanoTime();
@@ -2628,7 +1787,7 @@ public class RegerPFloatTest {
writer.writeRecord(record);
System.out.println(ratio);
}
-// break;
+
}
writer.close();
}