You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hu...@apache.org on 2023/12/04 14:29:22 UTC
(iotdb) 19/42: update quicker
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch research/encoding-reorder
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5c96f9ba3dabe8dd3565ff667f43be6248aa9580
Author: xjz17 <67...@users.noreply.github.com>
AuthorDate: Mon Nov 13 23:27:45 2023 +0800
update quicker
---
.../tsfile/encoding/KernelDensityEstimation.java | 29 +-
.../iotdb/tsfile/encoding/REGERCompressTest.java | 831 ++++++++++-----------
.../apache/iotdb/tsfile/encoding/REGERTest.java | 8 +-
3 files changed, 423 insertions(+), 445 deletions(-)
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/KernelDensityEstimation.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/KernelDensityEstimation.java
index bdf3628baf9..ed1f0855560 100644
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/KernelDensityEstimation.java
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/KernelDensityEstimation.java
@@ -53,19 +53,36 @@ public class KernelDensityEstimation {
static int[] findMinIndex(double[] array) {
int[] minIndex = new int[array.length];
int final_min_count = 0;
+ int pre_value = 0;
// double preValue = array[0];
for (int i = 1; i < array.length-1; i++) {
if (array[i] < array[i-1] && array[i] < array[i+1]) {
- minIndex[final_min_count] = i;
- final_min_count ++;
+ if(final_min_count != 0){
+ if(i>pre_value+32){
+ minIndex[final_min_count] = i;
+ final_min_count ++;
+ pre_value = i;
+ }
+ }else{
+ minIndex[final_min_count] = i;
+ final_min_count ++;
+ pre_value = i;
+ }
}
}
int[] final_minIndex = new int[final_min_count];
- if(final_min_count>0){
- final_minIndex[0] = minIndex[0];
- System.arraycopy(minIndex, 0, final_minIndex, 0, final_min_count);
- }
+// if(final_min_count>0){
+// final_minIndex[0] = minIndex[0];
+// int pre_value = minIndex[0];
+// for(int mv = 1; mv<final_min_count;mv++){
+// if(minIndex[mv]-pre_value>16){
+// pre_value = minIndex[mv];
+//
+// }
+// }
+ System.arraycopy(minIndex, 0, final_minIndex, 0, final_min_count);
+// }
return final_minIndex;
}
// public static void main(String[] args) {
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/REGERCompressTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/REGERCompressTest.java
index 0c3f2ebe6f5..7f784dbeee9 100644
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/REGERCompressTest.java
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/REGERCompressTest.java
@@ -1547,6 +1547,7 @@ public class REGERCompressTest {
int raw_abs_sum = raw_length[0];
int[][] new_length_list = new int[block_size][3];
int pos_new_length_list = 0;
+ int range = block_size/16;
ArrayList<Integer> j_star_list = new ArrayList<>(); // beta list of min b phi alpha to j
int j_star = -1;
@@ -1563,7 +1564,8 @@ public class REGERCompressTest {
adjust1ValueCost(ts_block, 1, raw_length, min_index, theta);
}
- for (int j = 2; j < block_size - 1; j++) {
+ for (int j = 2; j < range - 1; j++) {
+// for (int j = 2; j < block_size - 1; j++) {
// if j, alpha+1, alpha points are min residuals, need to recalculate min residuals
if (min_index.contains(j)) { //|| min_index.contains(1)
b = adjust0MinChange(ts_block, j, theta);
@@ -1610,7 +1612,8 @@ public class REGERCompressTest {
if (min_index.get(1) == block_size - 1) {
adjust1ValueCost(ts_block, block_size - 1, raw_length, min_index, theta);
}
- for (int j = 1; j < block_size - 1; j++) {
+ for (int j = block_size - range; j < block_size - 1; j++) {
+// for (int j = 1; j < block_size - 1; j++) {
if (min_index.contains(j)) { //min_index.contains(block_size - 1) ||
b = adjustnMinChange(ts_block, j, theta);
} else {
@@ -1662,7 +1665,10 @@ public class REGERCompressTest {
if (min_index.get(1) == alpha + 1) {
adjust1ValueCost(ts_block, alpha + 1, raw_length, min_index, theta);
}
- for (int j = 1; j < block_size - 1; j++) {
+ int start_j = Math.max(alpha-range/2,1);
+ int end_j = Math.min(alpha+range/2,block_size-1);
+ for (int j = start_j; j < end_j; j++) {
+// for (int j = 1; j < block_size - 1; j++) {
if (alpha != j && (alpha + 1) != j) {
if (min_index.contains(j)) { //|| min_index.contains(alpha) || min_index.contains(alpha + 1)
b = adjustAlphaToJMinChange(ts_block, alpha, j, theta);
@@ -1683,6 +1689,7 @@ public class REGERCompressTest {
}
}
}
+
if (min_index.contains(0)) {//|| min_index.contains(alpha) || min_index.contains(alpha + 1)
b = adjustTo0MinChange(ts_block, alpha, theta);
} else {
@@ -1738,13 +1745,16 @@ public class REGERCompressTest {
int value_delta_min = Integer.MAX_VALUE;
int timestamp_delta_min_index = -1;
int value_delta_min_index = -1;
-
+ int timestamp_delta_max = Integer.MIN_VALUE;
+ int value_delta_max = Integer.MIN_VALUE;
+ int timestamp_delta_max_index = -1;
+ int value_delta_max_index = -1;
float theta0_t = theta[0];
float theta1_t = theta[1];
float theta0_v = theta[2];
float theta1_v = theta[3];
- int[][] ts_block_delta = new int[block_size - 1][2];
+// int[][] ts_block_delta = new int[block_size - 1][2];
for (int j = 1; j < block_size; j++) {
int epsilon_v_j =
@@ -1753,13 +1763,13 @@ public class REGERCompressTest {
int epsilon_r_j =
ts_block[j][0]
- (int) (theta0_t + theta1_t * (float) ts_block[j - 1][0]);
- if (index == 0) {
- ts_block_delta[j - 1][0] = j;
- ts_block_delta[j - 1][1] = epsilon_v_j;
- } else if (index == 1) {
- ts_block_delta[j - 1][0] = j;
- ts_block_delta[j - 1][1] = epsilon_r_j;
- }
+// if (index == 0) {
+// ts_block_delta[j - 1][0] = j;
+// ts_block_delta[j - 1][1] = epsilon_v_j;
+// } else if (index == 1) {
+// ts_block_delta[j - 1][0] = j;
+// ts_block_delta[j - 1][1] = epsilon_r_j;
+// }
// ts_block_delta.add(tmp);
if (epsilon_r_j < timestamp_delta_min) {
timestamp_delta_min = epsilon_r_j;
@@ -1769,21 +1779,37 @@ public class REGERCompressTest {
value_delta_min = epsilon_v_j;
value_delta_min_index = j;
}
+ if (epsilon_r_j > timestamp_delta_max) {
+ timestamp_delta_max = epsilon_r_j;
+ timestamp_delta_max_index = j;
+ }
+ if (epsilon_v_j > value_delta_max) {
+ value_delta_max = epsilon_v_j;
+ value_delta_max_index = j;
+ }
}
min_index.add(timestamp_delta_min_index);
min_index.add(value_delta_min_index);
- Arrays.sort(ts_block_delta, (a, b) -> {
- if (a[1] == b[1])
- return Integer.compare(a[0], b[0]);
- return Integer.compare(a[1], b[1]);
- });
-
- int[] alpha_list = new int[k + 1];
+// Arrays.sort(ts_block_delta, (a, b) -> {
+// if (a[1] == b[1])
+// return Integer.compare(a[0], b[0]);
+// return Integer.compare(a[1], b[1]);
+// });
- alpha_list[0] = ts_block_delta[0][0];
- for (int i = 0; i < k; i++) {
- alpha_list[i + 1] = ts_block_delta[block_size - 2 - k][0];
+ int[] alpha_list = new int[2];
+ if(index==0){
+ alpha_list[0] = value_delta_min_index;
+ alpha_list[1] = value_delta_max_index;
+ }else{
+ alpha_list[0] = timestamp_delta_min_index;
+ alpha_list[1] = timestamp_delta_max_index;
}
+
+
+// alpha_list[0] = ts_block_delta[0][0];
+// for (int i = 0; i < k; i++) {
+// alpha_list[i + 1] = ts_block_delta[block_size - 2 - k][0];
+// }
return alpha_list;
}
@@ -1797,10 +1823,15 @@ public class REGERCompressTest {
int value_delta_max = Integer.MIN_VALUE;
int timestamp_delta_min = Integer.MAX_VALUE;
int value_delta_min = Integer.MAX_VALUE;
+ int timestamp_delta_min_index = -1;
+ int value_delta_min_index = -1;
+ int timestamp_delta_max_index = -1;
+ int value_delta_max_index = -1;
+
int[] alpha_list = new int[2 * k + 2];
- int[][] ts_block_delta_time = new int[block_size - 1][2];
- int[][] ts_block_delta_value = new int[block_size - 1][2];
+// int[][] ts_block_delta_time = new int[block_size - 1][2];
+// int[][] ts_block_delta_value = new int[block_size - 1][2];
float theta0_t = theta[0];
float theta1_t = theta[1];
@@ -1811,56 +1842,67 @@ public class REGERCompressTest {
for (int j = 1; j < block_size; j++) {
int epsilon_r_j = ts_block[j][0] - (int) (theta0_t + theta1_t * (float) ts_block[j - 1][0]);
int epsilon_v_j = ts_block[j][1] - (int) (theta0_v + theta1_v * (float) ts_block[j - 1][1]);
- ts_block_delta_time[j - 1][0] = j;
- ts_block_delta_time[j - 1][1] = epsilon_r_j;
- ts_block_delta_value[j - 1][0] = j;
- ts_block_delta_value[j - 1][1] = epsilon_v_j;
+// ts_block_delta_time[j - 1][0] = j;
+// ts_block_delta_time[j - 1][1] = epsilon_r_j;
+// ts_block_delta_value[j - 1][0] = j;
+// ts_block_delta_value[j - 1][1] = epsilon_v_j;
if (epsilon_v_j > value_delta_max) {
value_delta_max = epsilon_v_j;
+ value_delta_max_index = j;
}
if (epsilon_v_j < value_delta_min) {
value_delta_min = epsilon_v_j;
+ value_delta_min_index = j;
}
if (epsilon_r_j > timestamp_delta_max) {
timestamp_delta_max = epsilon_r_j;
+ timestamp_delta_max_index = j;
}
if (epsilon_r_j < timestamp_delta_min) {
timestamp_delta_min = epsilon_r_j;
+ timestamp_delta_min_index = j;
}
}
- Arrays.sort(ts_block_delta_time, (a, b) -> {
- if (a[1] == b[1])
- return Integer.compare(a[0], b[0]);
- return Integer.compare(a[1], b[1]);
- });
+// Arrays.sort(ts_block_delta_time, (a, b) -> {
+// if (a[1] == b[1])
+// return Integer.compare(a[0], b[0]);
+// return Integer.compare(a[1], b[1]);
+// });
- min_index.add(ts_block_delta_time[0][0]);
- alpha_list[0] = ts_block_delta_time[0][0];
- for (int i = 0; i < k; i++) {
- alpha_list[i + 1] = ts_block_delta_time[block_size - 2 - k][0];
- }
+ min_index.add(timestamp_delta_min_index);
+ alpha_list[0] = timestamp_delta_min_index;
+ alpha_list[1] = timestamp_delta_max_index;
+// alpha_list[0] = ts_block_delta_time[0][0];
+// for (int i = 0; i < k; i++) {
+// alpha_list[i + 1] = ts_block_delta_time[block_size - 2 - k][0];
+// }
- Arrays.sort(ts_block_delta_value, (a, b) -> {
- if (a[1] == b[1])
- return Integer.compare(a[0], b[0]);
- return Integer.compare(a[1], b[1]);
- });
+// Arrays.sort(ts_block_delta_value, (a, b) -> {
+// if (a[1] == b[1])
+// return Integer.compare(a[0], b[0]);
+// return Integer.compare(a[1], b[1]);
+// });
int pos_alpha_list = k + 1;
- min_index.add(ts_block_delta_value[0][0]);
- if (!containsValue(alpha_list, ts_block_delta_value[0][0])) {
- alpha_list[pos_alpha_list] = ts_block_delta_value[0][0];
+ min_index.add(value_delta_min_index);
+ if (!containsValue(alpha_list, value_delta_min_index)) {
+ alpha_list[pos_alpha_list] = value_delta_min_index;
pos_alpha_list++;
}
-
- for (int i = 0; i < k; i++) {
- if (!containsValue(alpha_list, ts_block_delta_value[block_size - 2 - k][0])) {
- alpha_list[pos_alpha_list] = ts_block_delta_value[block_size - 2 - k][0];
- pos_alpha_list++;
- }
+ if (!containsValue(alpha_list, value_delta_max_index)) {
+ alpha_list[pos_alpha_list] = value_delta_max_index;
+ pos_alpha_list++;
}
+
+
+// for (int i = 0; i < k; i++) {
+// if (!containsValue(alpha_list, ts_block_delta_value[block_size - 2 - k][0])) {
+// alpha_list[pos_alpha_list] = ts_block_delta_value[block_size - 2 - k][0];
+// pos_alpha_list++;
+// }
+// }
int[] new_alpha_list = new int[pos_alpha_list];
System.arraycopy(alpha_list, 0, new_alpha_list, 0, pos_alpha_list);
@@ -2104,198 +2146,6 @@ public class REGERCompressTest {
return pos_encode;
}
- private static int REGERBlockEncoderPartition(int[][] data, int i, int block_size, int segment_size, int k, int encode_pos, byte[] cur_byte) {
-
-
- int min_time = data[i * block_size][0];
- int[][] ts_block = new int[block_size][2];
- int[][] ts_block_reorder = new int[block_size][2];
- for (int j = 0; j < block_size; j++) {
-// data[j + i * block_size][0] -= min_time;
- ts_block[j][0] = (data[j + i * block_size][0] - min_time);
- ts_block[j][1] = data[j + i * block_size][1];
- ts_block_reorder[j][0] = data[j + i * block_size][0];
- ts_block_reorder[j][1] = data[j + i * block_size][1];
- }
-
-
- // raw-order
- int[] raw_length = new int[5]; // length,max_bit_width_interval,max_bit_width_value,max_bit_width_deviation
- float[] theta = new float[4];
- int[][] ts_block_delta = getEncodeBitsRegression(ts_block, block_size, raw_length, theta);
- int[][] bit_width_segments_partition = segmentBitPacking(ts_block_delta, block_size, segment_size);
- raw_length[0] = numberOfEncodeSegment2Bytes(ts_block_delta, bit_width_segments_partition, segment_size);
-
- Arrays.sort(ts_block, (a, b) -> {
- if (a[0] == b[0])
- return Integer.compare(a[1], b[1]);
- return Integer.compare(a[0], b[0]);
- });
-// quickSort(ts_block, 0, 0, block_size - 1);
- int[] time_length = new int[5];// length,max_bit_width_interval,max_bit_width_value,max_bit_width_deviation
- float[] theta_time = new float[4];
- int[][] ts_block_delta_time = getEncodeBitsRegression(ts_block, block_size, time_length, theta_time);
- int[][] bit_width_segments_time = segmentBitPacking(ts_block_delta_time, block_size, segment_size);
- time_length[0] = numberOfEncodeSegment2Bytes(ts_block_delta_time, bit_width_segments_time, segment_size);
-
-
- Arrays.sort(ts_block, (a, b) -> {
- if (a[1] == b[1])
- return Integer.compare(a[0], b[0]);
- return Integer.compare(a[1], b[1]);
- });
- // value-order
-// quickSort(ts_block, 1, 0, block_size - 1);
-
- int[] reorder_length = new int[5];
- float[] theta_reorder = new float[4];
- int[][] ts_block_delta_reorder = getEncodeBitsRegression(ts_block, block_size, reorder_length, theta_reorder);
- int[][] bit_width_segments_value = segmentBitPacking(ts_block_delta_reorder, block_size, segment_size);
- reorder_length[0] = numberOfEncodeSegment2Bytes(ts_block_delta_reorder, bit_width_segments_value, segment_size);
-
-
- int[] alpha_list;
-
- int choose = min3(time_length[0], raw_length[0], reorder_length[0]);
- ArrayList<Integer> min_index = new ArrayList<>();
-
- if (choose == 0) {
- raw_length = time_length;
- Arrays.sort(ts_block, (a, b) -> {
- if (a[0] == b[0])
- return Integer.compare(a[1], b[1]);
- return Integer.compare(a[0], b[0]);
- });
- theta = theta_time;
- alpha_list = getIStar(ts_block, min_index, block_size, 0, theta, k);
- } else if (choose == 1) {
- ts_block = ts_block_reorder;
- alpha_list = getIStar(ts_block, min_index, block_size, 0, theta, k);
- } else {
- raw_length = reorder_length;
- theta = theta_reorder;
- alpha_list = getIStar(ts_block, min_index, block_size, 1, theta, k);
- }
- int[] beta_list = new int[alpha_list.length];
- int[][] new_length_list = new int[alpha_list.length][5];
- int pos_new_length_list = 0;
- int[] new_alpha_list = new int[alpha_list.length];
-
-
-// ArrayList<ArrayList<Integer>> new_length_list = new ArrayList<>();
-
-
- for (int alpha : alpha_list) {
- if (alpha == -1)
- continue;
- new_alpha_list[pos_new_length_list] = alpha;
-
- ArrayList<Integer> new_min_index = (ArrayList<Integer>) min_index.clone();
- int[] new_length = raw_length.clone();
-// beta_list[j] = 0;
- beta_list[pos_new_length_list] = getBeta(ts_block, alpha, new_min_index, block_size, new_length, theta);
- if (beta_list[pos_new_length_list] == -1)
- continue;
- System.arraycopy(new_length, 0, new_length_list[pos_new_length_list], 0, 5);
- pos_new_length_list++;
- }
-
- int adjust_count = 0;
- int[][] all_length;
- while (pos_new_length_list != 0) {
- if (adjust_count < block_size / 2 && adjust_count < 20) {
- adjust_count++;
- } else {
- break;
- }
-
- all_length = new int[pos_new_length_list][2];
- for (int pos_new_length_list_j = 0; pos_new_length_list_j < pos_new_length_list; pos_new_length_list_j++) {
- all_length[pos_new_length_list_j][0] = pos_new_length_list_j;
- all_length[pos_new_length_list_j][1] = new_length_list[pos_new_length_list_j][0];
- }
- Arrays.sort(all_length, (a, b) -> {
- if (a[1] == b[1])
- return Integer.compare(a[0], b[0]);
- return Integer.compare(a[1], b[1]);
- });
-
- if (all_length[0][1] <= raw_length[0]) {
- int[][] new_ts_block = ts_block.clone();
- moveAlphaToBeta(new_ts_block, new_alpha_list[all_length[0][0]], beta_list[all_length[0][0]]);
- int[] new_length = new int[5];
- ts_block_delta = getEncodeBitsRegression(new_ts_block, block_size, new_length, theta);
- int[][] bit_width_segments = segmentBitPacking(ts_block_delta, block_size, segment_size);
- new_length[0] = numberOfEncodeSegment2Bytes(ts_block_delta, bit_width_segments, segment_size);
-
- if (new_length[0] <= raw_length[0]) {
- raw_length = new_length;
- ts_block = new_ts_block.clone();
- } else {
- break;
- }
- } else {
- break;
- }
- alpha_list = getIStar(ts_block, min_index, block_size, theta, k);
-
- int alpha_size = alpha_list.length;
- for (int alpha_i = alpha_size - 1; alpha_i >= 0; alpha_i--) {
- if (containsValue(beta_list, alpha_list[alpha_i])) {
- alpha_list = removeElement(alpha_list, alpha_i);
- }
- }
- beta_list = new int[alpha_list.length];
- new_length_list = new int[alpha_list.length][5];
- pos_new_length_list = 0;
- new_alpha_list = new int[alpha_list.length];
-
- for (int alpha : alpha_list) {
- if (alpha == -1)
- continue;
- new_alpha_list[pos_new_length_list] = alpha;
-
- int[] new_length = raw_length.clone();
-// beta_list[j] = 0;
- beta_list[pos_new_length_list] = getBeta(ts_block, alpha, min_index, block_size, new_length, theta);
- if (beta_list[pos_new_length_list] == -1)
- continue;
- System.arraycopy(new_length, 0, new_length_list[pos_new_length_list], 0, 5);
- pos_new_length_list++;
- }
-
- }
-
-
- ts_block_delta = getEncodeBitsRegressionNoTrain(ts_block, block_size, raw_length, theta, segment_size);
-
- int segment_n = (block_size - 1) / segment_size;
- int[][] bit_width_segments = new int[segment_n][2];
- for (int segment_i = 0; segment_i < segment_n; segment_i++) {
- int bit_width_time = Integer.MIN_VALUE;
- int bit_width_value = Integer.MIN_VALUE;
-
- for (int data_i = segment_i * segment_size + 1; data_i < (segment_i + 1) * segment_size + 1; data_i++) {
- int cur_bit_width_time = getBitWith(ts_block_delta[data_i][0]);
- int cur_bit_width_value = getBitWith(ts_block_delta[data_i][1]);
- if (cur_bit_width_time > bit_width_time) {
- bit_width_time = cur_bit_width_time;
- }
- if (cur_bit_width_value > bit_width_value) {
- bit_width_value = cur_bit_width_value;
- }
- }
-
- bit_width_segments[segment_i][0] = bit_width_time;
- bit_width_segments[segment_i][1] = bit_width_value;
- }
-
-
- encode_pos = encodeSegment2Bytes(ts_block_delta, bit_width_segments, raw_length, segment_size, theta, encode_pos, cur_byte);
-
- return encode_pos;
- }
-
private static int[][] ReorderingTimeSeries(int[][] ts_block, int[] raw_length, float[] theta, int segment_size,int k) {
@@ -2306,7 +2156,7 @@ public class REGERCompressTest {
int index_alpha_list = 0;
getEncodeBitsRegression(ts_block, block_size, raw_length, theta);
- int[] alpha_list = getIStar(ts_block, min_index, block_size, index_alpha_list, theta, k);
+ int[] alpha_list = getIStar(ts_block, min_index, block_size, 0, theta, k);
int[] beta_list = new int[alpha_list.length];
int[][] new_length_list = new int[alpha_list.length][5];
int pos_new_length_list = 0;
@@ -2433,61 +2283,93 @@ public class REGERCompressTest {
int[][] ts_block_partition;
if (supply_length == 0) {
- int max_value = Integer.MIN_VALUE;
- int min_value = Integer.MAX_VALUE;
-
-// int[] unique = new int[block_size];
ts_block = new int[block_size][2];
ts_block_value = new int[block_size][2];
ts_block_partition = new int[block_size][2];
+// Map<Integer, Integer> data_map = new HashMap<>();
+// int min_value = Integer.MAX_VALUE;
+
for (int j = 0; j < block_size; j++) {
ts_block[j][0] = (data[j + i * block_size][0] - min_time);
ts_block[j][1] = data[j + i * block_size][1];
- if(ts_block[j][1]>max_value){
- max_value = ts_block[j][1];
- }
- if(ts_block[j][1]<min_value){
- min_value = ts_block[j][1];
- }
- ts_block_value[j][0] =ts_block[j][0];
- ts_block_value[j][1] =ts_block[j][1];
+// if(ts_block[j][1]<min_value){
+// min_value = ts_block[j][1];
+// }
+// if(data_map.containsKey(ts_block[j][1])){
+// int tmp = data_map.get(ts_block[j][1]);
+// tmp++;
+// data_map.put(ts_block[j][1],tmp);
+// }else{
+// data_map.put(ts_block[j][1],1);
+// }
+// ts_block_value[j][0] =ts_block[j][0];
+// ts_block_value[j][1] =ts_block[j][1];
}
- Map<Integer, Integer> data_map = new HashMap<>();
-// for(int mv=min_value;mv<=max_value;mv++){
-// data_map.put(mv,0);
+// for (int j = 0; j < block_size; j++) {
+// ts_block[j][0] = (data[j + i * block_size][0] - min_time);
+// ts_block[j][1] = data[j + i * block_size][1];
+// if(ts_block[j][1]<min_value){
+// min_value = ts_block[j][1];
+// }
+// int tmp_value = ts_block[j][1]-min_value;
+// if(data_map.containsKey(tmp_value)){
+// int tmp = data_map.get(tmp_value);
+// tmp++;
+// data_map.put(tmp_value,tmp);
+// }else{
+// data_map.put(tmp_value,1);
+// }
+// ts_block_value[j][0] =ts_block[j][0];
+// ts_block_value[j][1] =ts_block[j][1];
// }
- for (int j = 0; j < block_size; j++) {
- if(data_map.containsKey(ts_block[j][1])){
- int tmp = data_map.get(ts_block[j][1]);
- tmp++;
- data_map.put(ts_block[j][1],tmp);
- }else{
- data_map.put(ts_block[j][1],1);
- }
- }
-// System.out.println(data_map);
- double[] kernelDensity = calculateKernelDensity(data_map);
-
- third_value= findMinIndex(kernelDensity);
- System.out.println("Minimum point: x=" + (Arrays.toString(third_value)));
+// double[] kernelDensity = calculateKernelDensity(data_map);
+//
+// third_value= findMinIndex(kernelDensity);
+// for(int j=0;j<third_value.length;j++){
+// third_value[j] += min_value;
+// }
+// System.out.println("Minimum point: x=" + (Arrays.toString(third_value)));
} else {
ts_block = new int[supply_length][2];
ts_block_value = new int[supply_length][2];
ts_block_partition = new int[supply_length][2];
int end = data.length - i * block_size;
+// Map<Integer, Integer> data_map = new HashMap<>();
+// int min_value = Integer.MAX_VALUE;
+
for (int j = 0; j < end; j++) {
// data[j + i * block_size][0] -= min_time;
ts_block[j][0] = (data[j + i * block_size][0] - min_time);
ts_block[j][1] = data[j + i * block_size][1];
- ts_block_value[j][0] =ts_block[j][0];
- ts_block_value[j][1] =ts_block[j][1];
+// if(ts_block[j][1]<min_value){
+// min_value = ts_block[j][1];
+// }
+// if(data_map.containsKey(ts_block[j][1])){
+// int tmp = data_map.get(ts_block[j][1]);
+// tmp++;
+// data_map.put(ts_block[j][1],tmp);
+// }else{
+// data_map.put(ts_block[j][1],1);
+// }
+// ts_block_value[j][0] =ts_block[j][0];
+// ts_block_value[j][1] =ts_block[j][1];
}
for (int j = end; j < supply_length; j++) {
ts_block[j][0] = 0;
ts_block[j][1] = 0;
- ts_block_value[j][0] =ts_block[j][0];
- ts_block_value[j][1] =ts_block[j][1];
+// if(data_map.containsKey(ts_block[j][1])){
+// int tmp = data_map.get(ts_block[j][1]);
+// tmp++;
+// data_map.put(ts_block[j][1],tmp);
+// }else{
+// data_map.put(ts_block[j][1],1);
+// }
+// ts_block_value[j][0] =ts_block[j][0];
+// ts_block_value[j][1] =ts_block[j][1];
}
+// double[] kernelDensity = calculateKernelDensity(data_map);
+
+// third_value= findMinIndex(kernelDensity);
block_size = supply_length;
}
@@ -2506,62 +2388,71 @@ public class REGERCompressTest {
ts_block_delta_time = ReorderingTimeSeries(ts_block, time_length, theta_time, segment_size, k);
- int pos_ts_block_partition = 0;
- if(third_value.length>0){
- for (int[] datum : ts_block) {
- if (datum[1] > third_value[third_value.length - 1]) {
- ts_block_partition[pos_ts_block_partition][0] = datum[0];
- ts_block_partition[pos_ts_block_partition][1] = datum[1];
- pos_ts_block_partition++;
- }
- }
- for (int third_i = third_value.length - 1; third_i > 0; third_i--) {
- for (int[] datum : ts_block) {
- if (datum[1] <= third_value[third_i] && datum[1] > third_value[third_i - 1]) {
- ts_block_partition[pos_ts_block_partition][0] = datum[0];
- ts_block_partition[pos_ts_block_partition][1] = datum[1];
- pos_ts_block_partition++;
- }
- }
- }
- for (int[] datum : ts_block) {
- if (datum[1] <= third_value[0]) {
- ts_block_partition[pos_ts_block_partition][0] = datum[0];
- ts_block_partition[pos_ts_block_partition][1] = datum[1];
- pos_ts_block_partition++;
- }
- }
- }
-
-
- ts_block_delta_partition = ReorderingTimeSeries(ts_block_partition, raw_length, theta, segment_size, k);
-
- Arrays.sort(ts_block_value, (a, b) -> {
- if (a[1] == b[1])
- return Integer.compare(a[0], b[0]);
- return Integer.compare(a[1], b[1]);
- });
+// int pos_ts_block_partition = 0;
+// if(third_value.length>0){
+// for (int[] datum : ts_block) {
+// if (datum[1] > third_value[third_value.length - 1]) {
+// ts_block_partition[pos_ts_block_partition][0] = datum[0];
+// ts_block_partition[pos_ts_block_partition][1] = datum[1];
+// pos_ts_block_partition++;
+// }
+// }
+// for (int third_i = third_value.length - 1; third_i > 0; third_i--) {
+// for (int[] datum : ts_block) {
+// if (datum[1] <= third_value[third_i] && datum[1] > third_value[third_i - 1]) {
+// ts_block_partition[pos_ts_block_partition][0] = datum[0];
+// ts_block_partition[pos_ts_block_partition][1] = datum[1];
+// pos_ts_block_partition++;
+// }
+// }
+// }
+// for (int[] datum : ts_block) {
+// if (datum[1] <= third_value[0]) {
+// ts_block_partition[pos_ts_block_partition][0] = datum[0];
+// ts_block_partition[pos_ts_block_partition][1] = datum[1];
+// pos_ts_block_partition++;
+// }
+// }
+// }
+//
+//
+// ts_block_delta_partition = ReorderingTimeSeries(ts_block_partition, raw_length, theta, segment_size, k);
- ts_block_delta_reorder = ReorderingTimeSeries(ts_block_value, reorder_length, theta_reorder, segment_size, k);
+// Arrays.sort(ts_block_value, (a, b) -> {
+// if (a[1] == b[1])
+// return Integer.compare(a[0], b[0]);
+// return Integer.compare(a[1], b[1]);
+// });
+//
+// ts_block_delta_reorder = ReorderingTimeSeries(ts_block_value, reorder_length, theta_reorder, segment_size, k);
- int choose = min3(time_length[0], raw_length[0], reorder_length[0]);
+// int choose = min3(time_length[0], raw_length[0], reorder_length[0]);
int segment_n = (block_size - 1) / segment_size;
int[][] bit_width_segments = new int[segment_n][2];
- if (choose == 0) {
+// if(time_length[0] > raw_length[0]){
+// bit_width_segments = segmentBitPacking(ts_block_delta_partition, block_size, segment_size);
+// encode_pos = encodeSegment2Bytes(ts_block_delta_partition, bit_width_segments, raw_length, segment_size, theta, encode_pos, cur_byte);
+// }else{
bit_width_segments = segmentBitPacking(ts_block_delta_time, block_size, segment_size);
encode_pos = encodeSegment2Bytes(ts_block_delta_time, bit_width_segments, time_length, segment_size, theta_time, encode_pos, cur_byte);
- best_order[0] ++;
- } else if (choose == 1) {
- bit_width_segments = segmentBitPacking(ts_block_delta_partition, block_size, segment_size);
- encode_pos = encodeSegment2Bytes(ts_block_delta_partition, bit_width_segments, raw_length, segment_size, theta, encode_pos, cur_byte);
- best_order[2] ++;
- } else {
- bit_width_segments = segmentBitPacking(ts_block_delta_reorder, block_size, segment_size);
- encode_pos = encodeSegment2Bytes(ts_block_delta_reorder, bit_width_segments, reorder_length, segment_size, theta_reorder, encode_pos, cur_byte);
- best_order[1] ++;
- }
+
+// }
+
+// if (choose == 0) {
+// bit_width_segments = segmentBitPacking(ts_block_delta_time, block_size, segment_size);
+// encode_pos = encodeSegment2Bytes(ts_block_delta_time, bit_width_segments, time_length, segment_size, theta_time, encode_pos, cur_byte);
+// best_order[0] ++;
+// } else if (choose == 1) {
+// bit_width_segments = segmentBitPacking(ts_block_delta_partition, block_size, segment_size);
+// encode_pos = encodeSegment2Bytes(ts_block_delta_partition, bit_width_segments, raw_length, segment_size, theta, encode_pos, cur_byte);
+// best_order[2] ++;
+// } else {
+// bit_width_segments = segmentBitPacking(ts_block_delta_reorder, block_size, segment_size);
+// encode_pos = encodeSegment2Bytes(ts_block_delta_reorder, bit_width_segments, reorder_length, segment_size, theta_reorder, encode_pos, cur_byte);
+// best_order[1] ++;
+// }
@@ -2585,118 +2476,10 @@ public class REGERCompressTest {
int2Bytes(segment_size, encode_pos, encoded_result);
encode_pos += 4;
-// // ----------------------- compare the whole time series order by time, value and partition ---------------------------
-// int length_time = 0;
-// int length_value = 0;
-// int length_partition = 0;
-// int[][] data_value = data.clone();
-//// System.out.println(Arrays.deepToString(data_value));
-//// quickSort(data_value,1,0,length_all-1);
-//// Arrays.sort(data_value, new Comparator<int[]>() {
-//// @Override
-//// public int compare(int[] a, int[] b) {
-//// if (a[1] == b[1])
-//// return Integer.compare(a[0], b[0]);
-//// return Integer.compare(a[1], b[1]);
-//// }
-//// });
-// Arrays.sort(data_value, (a, b) -> {
-// if (a[1] == b[1])
-// return Integer.compare(a[0], b[0]);
-// return Integer.compare(a[1], b[1]);
-// });
-//// System.out.println(Arrays.deepToString(data_value));
-//
-// int[][] data_partition = new int[length_all][2];
-// int pos_data_partition = 0;
-//
-// for (int[] datum : data) {
-// if (datum[1] > third_value[third_value.length - 1]) {
-// data_partition[pos_data_partition][0] = datum[0];
-// data_partition[pos_data_partition][1] = datum[1];
-// pos_data_partition++;
-// }
-// }
-// for (int third_i = third_value.length - 1; third_i > 0; third_i--) {
-// for (int[] datum : data) {
-// if (datum[1] <= third_value[third_i] && datum[1] > third_value[third_i - 1]) {
-// data_partition[pos_data_partition][0] = datum[0];
-// data_partition[pos_data_partition][1] = datum[1];
-// pos_data_partition++;
-// }
-// }
-// }
-// for (int[] datum : data) {
-// if (datum[1] <= third_value[0]) {
-// data_partition[pos_data_partition][0] = datum[0];
-// data_partition[pos_data_partition][1] = datum[1];
-// pos_data_partition++;
-// }
-// }
-// for (int i = 0; i < block_num; i++) {
-// int[][] ts_block_time = new int[block_size][2];
-// int[][] ts_block_value = new int[block_size][2];
-// int[][] ts_block_partition = new int[block_size][2];
-//
-// for (int j = 0; j < block_size; j++) {
-// ts_block_time[j][0] = data[j + i * block_size][0];
-// ts_block_time[j][1] = data[j + i * block_size][1];
-// ts_block_value[j][0] = data_value[j + i * block_size][0];
-// ts_block_value[j][1] = data_value[j + i * block_size][1];
-// ts_block_partition[j][0] = data_partition[j + i * block_size][0];
-// ts_block_partition[j][1] = data_partition[j + i * block_size][1];
-// }
-//
-// int[] raw_length = new int[5];
-// float[] theta = new float[4];
-// int[][] ts_block_delta = getEncodeBitsRegression(ts_block_time, block_size, raw_length, theta, segment_size);
-// int[][] bit_width_segments = segmentBitPacking(ts_block_delta, block_size, segment_size);
-// length_time += numberOfEncodeSegment2Bytes(ts_block_delta, bit_width_segments, raw_length, segment_size, theta);
-//
-//
-// int[] raw_length_value = new int[5];
-// float[] theta_value = new float[4];
-// int[][] ts_block_delta_value = getEncodeBitsRegression(ts_block_value, block_size, raw_length_value, theta_value, segment_size);
-// int[][] bit_width_segments_value = segmentBitPacking(ts_block_delta_value, block_size, segment_size);
-// length_value += numberOfEncodeSegment2Bytes(ts_block_delta_value, bit_width_segments_value, raw_length_value, segment_size, theta_value);
-//
-// int[] raw_length_partition = new int[5];
-// float[] theta_partition = new float[4];
-// int[][] ts_block_delta_partition = getEncodeBitsRegression(ts_block_partition, block_size, raw_length_partition, theta_partition, segment_size);
-// int[][] bit_width_segments_partition = segmentBitPacking(ts_block_delta_partition, block_size, segment_size);
-// length_partition += numberOfEncodeSegment2Bytes(ts_block_delta_partition, bit_width_segments_partition, raw_length_partition, segment_size, theta_partition);
-//
-// }
-//
-
-
-// if (length_partition < length_time && length_partition < length_value) { // partition performs better
-// data = data_partition;
-//
-// System.out.println("Partition");
-// for (int i = 0; i < block_num; i++) {
-// encode_pos = REGERBlockEncoderPartition(data, i, block_size, segment_size, k, encode_pos, encoded_result);
-// }
-// } else {
-// if (length_value < length_time) { // order by value performs better
-// System.out.println("Value");
-// data = data_value;
-//// for (int i = 0; i < 2; i++) {
-// for (int i = 0; i < block_num; i++) {
-//
-// encode_pos = REGERBlockEncoder(data, 1, i, block_size, 0, third_value, segment_size, k, encode_pos, encoded_result);
-// }
-// } else {
-// System.out.println("Time");
-// for (int i = 0; i < 1; i++) {
for (int i = 0; i < block_num; i++) {
encode_pos = REGERBlockEncoder(data, i, block_size, 0, third_value, segment_size, k, encode_pos, encoded_result,best_order);
}
-// }
-//
-// }
-//
int remaining_length = length_all - block_num * block_size;
if (remaining_length == 1) {
int2Bytes(data[data.length - 1][0], encode_pos, encoded_result);
@@ -2853,7 +2636,7 @@ public class REGERCompressTest {
}
@Test
- public void REGER() throws IOException {
+ public void REGERBestOrder() throws IOException {
// String parent_dir = "C:/Users/xiaoj/Desktop/test";
String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/iotdb/iotdb-core/tsfile/src/test/resources/";
String output_parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-reorder/compression_ratio/reger_best_order";
@@ -3032,6 +2815,184 @@ public class REGERCompressTest {
}
}
+ @Test
+ public void REGERRemoveOrderByValue() throws IOException {
+// String parent_dir = "C:/Users/xiaoj/Desktop/test";
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/iotdb/iotdb-core/tsfile/src/test/resources/";
+ String output_parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-reorder/compression_ratio/reger_remove_value";
+
+
+ String input_parent_dir = parent_dir + "trans_data/";
+ ArrayList<String> input_path_list = new ArrayList<>();
+ ArrayList<String> output_path_list = new ArrayList<>();
+ ArrayList<String> dataset_name = new ArrayList<>();
+ ArrayList<Integer> dataset_block_size = new ArrayList<>();
+ ArrayList<int[]> dataset_third = new ArrayList<>();
+ ArrayList<Integer> dataset_k = new ArrayList<>();
+ dataset_name.add("CS-Sensors");
+ dataset_name.add("Metro-Traffic");
+ dataset_name.add("USGS-Earthquakes");
+ dataset_name.add("YZ-Electricity");
+ dataset_name.add("GW-Magnetic");
+ dataset_name.add("TY-Fuel");
+ dataset_name.add("Cyber-Vehicle");
+ dataset_name.add("Vehicle-Charge");
+ dataset_name.add("Nifty-Stocks");
+ dataset_name.add("TH-Climate");
+ dataset_name.add("TY-Transport");
+ dataset_name.add("EPM-Education");
+
+ int[] dataset_0 = {547, 2816};
+ int[] dataset_1 = {1719, 3731};
+ int[] dataset_2 = {-48, -11, 6, 25, 52};
+ int[] dataset_3 = {8681, 13584};
+ int[] dataset_4 = {79, 184, 274};
+ int[] dataset_5 = {17, 68};
+ int[] dataset_6 = {677};
+ int[] dataset_7 = {1047, 1725};
+ int[] dataset_8 = {227, 499, 614, 1013};
+ int[] dataset_9 = {474, 678};
+ int[] dataset_10 = {4, 30, 38, 49, 58};
+ int[] dataset_11 = {5182, 8206};
+
+ dataset_third.add(dataset_0);
+ dataset_third.add(dataset_1);
+ dataset_third.add(dataset_2);
+ dataset_third.add(dataset_3);
+ dataset_third.add(dataset_4);
+ dataset_third.add(dataset_5);
+ dataset_third.add(dataset_6);
+ dataset_third.add(dataset_7);
+ dataset_third.add(dataset_8);
+ dataset_third.add(dataset_9);
+ dataset_third.add(dataset_10);
+ dataset_third.add(dataset_11);
+
+ for (String value : dataset_name) {
+ input_path_list.add(input_parent_dir + value);
+ dataset_k.add(1);
+ dataset_block_size.add(128);
+ }
+
+ output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
+// dataset_block_size.add(128);
+
+ output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
+// dataset_block_size.add(4096);
+ output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
+// dataset_block_size.add(8192);
+ output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
+ output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
+ output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
+// dataset_block_size.add(8192);
+ output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
+// dataset_block_size.add(128);
+ output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
+// dataset_block_size.add(64);
+ output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
+// dataset_block_size.add(64);
+ output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
+// dataset_block_size.add(256);
+
+ for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
+// for (int file_i = 0; file_i < 1; file_i++) {
+ String inputPath = input_path_list.get(file_i);
+ String Output = output_path_list.get(file_i);
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio",
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+
+ for (File f : tempList) {
+// f = tempList[1];
+ System.out.println(f);
+ InputStream inputStream = Files.newInputStream(f.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<ArrayList<Integer>> data = new ArrayList<>();
+
+ // add a column to "data"
+ loader.readHeaders();
+ while (loader.readRecord()) {
+ ArrayList<Integer> tmp = new ArrayList<>();
+ tmp.add(Integer.valueOf(loader.getValues()[0]));
+ tmp.add(Integer.valueOf(loader.getValues()[1]));
+ data.add(tmp);
+ }
+ inputStream.close();
+ ArrayList<Integer> result2 = new ArrayList<>();
+ splitTimeStamp3(data, result2);
+
+ int[][] data2_arr = new int[data.size()][2];
+ int min_time = data.get(0).get(0);
+ for (int i = 0; i < data.size(); i++) {
+ data2_arr[i][0] = data.get(i).get(0) - min_time;
+ data2_arr[i][1] = data.get(i).get(1);
+ }
+// System.out.println(data2_arr[0][0]);
+ byte[] encoded_result = new byte[data2_arr.length * 8];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+ int repeatTime2 = 100;
+ long s = System.nanoTime();
+ int[] best_order = new int[3];
+ int length = 0;
+ for (int repeat = 0; repeat < repeatTime2; repeat++)
+ length = ReorderingRegressionEncoder(data2_arr, dataset_block_size.get(file_i), dataset_third.get(file_i), 8, dataset_k.get(file_i), encoded_result,best_order);
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime2);
+ compressed_size += length;
+ double ratioTmp = compressed_size / (double) (data.size() * Integer.BYTES * 2);
+ ratio += ratioTmp;
+ s = System.nanoTime();
+// for (int repeat = 0; repeat < repeatTime2; repeat++)
+// REGERDecoder(encoded_result);
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime2);
+
+
+ String[] record = {
+ f.toString(),
+ "REGER-Remove-Value",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio),
+// String.valueOf(best_order[0]),
+// String.valueOf(best_order[1]),
+// String.valueOf(best_order[2])
+ };
+ writer.writeRecord(record);
+// System.out.println(Arrays.toString(best_order));
+ System.out.println(ratio);
+
+// break;
+ }
+ writer.close();
+ }
+ }
+
+
// @Test
// public void REGERTime() throws IOException {
//// String parent_dir = "C:/Users/xiaoj/Desktop/test";
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/REGERTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/REGERTest.java
index 1766f95e74c..1c5d3b40b3f 100644
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/REGERTest.java
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/REGERTest.java
@@ -2543,7 +2543,7 @@ public class REGERTest {
}
}
- System.out.println(adjust_count);
+// System.out.println(adjust_count);
ts_block_delta = getEncodeBitsRegressionNoTrain(ts_block, block_size, raw_length, theta, segment_size);
int segment_n = (block_size - 1) / segment_size;
@@ -2939,8 +2939,8 @@ public class REGERTest {
output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
// dataset_block_size.add(256);
-// for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
- for (int file_i = 0; file_i < 1; file_i++) {
+ for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
+// for (int file_i = 0; file_i < 1; file_i++) {
String inputPath = input_path_list.get(file_i);
String Output = output_path_list.get(file_i);
@@ -2993,7 +2993,7 @@ public class REGERTest {
long decodeTime = 0;
double ratio = 0;
double compressed_size = 0;
- int repeatTime2 = 1;
+ int repeatTime2 = 100;
long s = System.nanoTime();
int length = 0;
for (int repeat = 0; repeat < repeatTime2; repeat++)