You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by le...@apache.org on 2022/10/10 08:25:21 UTC
[iotdb] branch research/M4-visualization updated (7f315219ab -> c50b731e0f)
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a change to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 7f315219ab todo
new 082283ef4d width 8 multiple faster, otherwise no
new 724295aee6 width 8 multiple faster, otherwise no
new c50b731e0f width 8 multiple faster, otherwise no
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../iotdb/session/MyBasicOperationTest2.java | 16 +-
...rationTest2.java => MyBasicOperationTest3.java} | 55 ++---
.../session/MyRealDataTest1_WriteAndQuery.java | 94 ++++----
.../tsfile/common/constant/TsFileConstant.java | 26 ++-
.../encoding/decoder/DeltaBinaryDecoder.java | 42 ++--
.../org/apache/iotdb/tsfile/utils/BytesUtils.java | 237 ++++++++++++++-------
6 files changed, 297 insertions(+), 173 deletions(-)
copy session/src/test/java/org/apache/iotdb/session/{MyBasicOperationTest2.java => MyBasicOperationTest3.java} (67%)
[iotdb] 02/03: width 8 multiple faster, otherwise no
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 724295aee605d5f94add92d8ad0f6c608fe84f72
Author: Lei Rui <10...@qq.com>
AuthorDate: Mon Oct 10 15:55:06 2022 +0800
width 8 multiple faster, otherwise no
---
.../test/java/org/apache/iotdb/session/MyBasicOperationTest3.java | 6 +++---
.../src/main/java/org/apache/iotdb/tsfile/utils/BytesUtils.java | 8 +++++---
2 files changed, 8 insertions(+), 6 deletions(-)
diff --git a/session/src/test/java/org/apache/iotdb/session/MyBasicOperationTest3.java b/session/src/test/java/org/apache/iotdb/session/MyBasicOperationTest3.java
index a281b1c117..bcbceb4d61 100644
--- a/session/src/test/java/org/apache/iotdb/session/MyBasicOperationTest3.java
+++ b/session/src/test/java/org/apache/iotdb/session/MyBasicOperationTest3.java
@@ -15,7 +15,7 @@ public class MyBasicOperationTest3 {
int repeat = 1000000;
int packNum = 128;
- int packWidth = 2;
+ int packWidth = 13;
DescriptiveStatistics op1 = new DescriptiveStatistics();
DescriptiveStatistics op2 = new DescriptiveStatistics();
for (int k = 0; k < repeat; k++) {
@@ -25,8 +25,8 @@ public class MyBasicOperationTest3 {
int high = 256; // exclusive
byte[] buf = new byte[packNum * 8];
for (int i = 0; i < packNum; i++) {
-// int v = r.nextInt(high - low) + low;
- int v = 187;
+ int v = r.nextInt(high - low) + low;
+// int v = 187;
BytesUtils.longToBytes(v, buf, i * packWidth, packWidth);
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BytesUtils.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BytesUtils.java
index 49dee499e8..94e9d83548 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BytesUtils.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BytesUtils.java
@@ -590,7 +590,8 @@ public class BytesUtils {
TsFileConstant.bytesToLong_byteNum1_smallByte++;
int mask = (int) Math.pow(2, 8 - width) - 1; // TODO consider if this to make static
mask = (~mask & 0xff) >> startPosInByte;
- return (result[startByte] & 0xff & mask) >> (7 - endPosInByte);
+// return (result[startByte] & 0xff & mask) >> (7 - endPosInByte);
+ return (result[startByte] & mask) >> (7 - endPosInByte);
}
}
// TODO if across two bytes
@@ -607,7 +608,7 @@ public class BytesUtils {
TsFileConstant.byteToLong_byteNums_firstByte_smallByte++;
int mask =
(int) Math.pow(2, 8 - startPosInByte) - 1; // TODO consider if this to make static
- value = value | ((result[startByte] & 0xff & mask) << shift);
+ value = value | ((result[startByte] & mask) << shift);
}
// 2. deal with the last byte
@@ -620,7 +621,8 @@ public class BytesUtils {
TsFileConstant.byteToLong_byteNums_lastByte_smallByte++;
int mask =
(int) Math.pow(2, 7 - endPosInByte) - 1; // TODO consider if this to make static
- value = value | ((result[endByte] & 0xff & ~mask) >> (7 - endPosInByte));
+ value = value | ((result[endByte] & 0xff & ~mask) >> (7
+ - endPosInByte)); // here mask is negative so need &0xff
}
// 3. deal with the middle bytes
[iotdb] 03/03: width 8 multiple faster, otherwise no
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit c50b731e0f99aeed3bd17fdbfc80d1c0de79a806
Author: Lei Rui <10...@qq.com>
AuthorDate: Mon Oct 10 16:25:37 2022 +0800
width 8 multiple faster, otherwise no
---
.../iotdb/session/MyBasicOperationTest3.java | 2 +-
.../session/MyRealDataTest1_WriteAndQuery.java | 2 +-
.../org/apache/iotdb/tsfile/utils/BytesUtils.java | 30 +++++++++++++++++++---
3 files changed, 29 insertions(+), 5 deletions(-)
diff --git a/session/src/test/java/org/apache/iotdb/session/MyBasicOperationTest3.java b/session/src/test/java/org/apache/iotdb/session/MyBasicOperationTest3.java
index bcbceb4d61..f03635041f 100644
--- a/session/src/test/java/org/apache/iotdb/session/MyBasicOperationTest3.java
+++ b/session/src/test/java/org/apache/iotdb/session/MyBasicOperationTest3.java
@@ -15,7 +15,7 @@ public class MyBasicOperationTest3 {
int repeat = 1000000;
int packNum = 128;
- int packWidth = 13;
+ int packWidth = 2;
DescriptiveStatistics op1 = new DescriptiveStatistics();
DescriptiveStatistics op2 = new DescriptiveStatistics();
for (int k = 0; k < repeat; k++) {
diff --git a/session/src/test/java/org/apache/iotdb/session/MyRealDataTest1_WriteAndQuery.java b/session/src/test/java/org/apache/iotdb/session/MyRealDataTest1_WriteAndQuery.java
index 6024cd118a..10f8235500 100644
--- a/session/src/test/java/org/apache/iotdb/session/MyRealDataTest1_WriteAndQuery.java
+++ b/session/src/test/java/org/apache/iotdb/session/MyRealDataTest1_WriteAndQuery.java
@@ -73,7 +73,7 @@ public class MyRealDataTest1_WriteAndQuery {
private static int valueIdx = 1; // 值idx，从0开始
private static int w = 2;
private static long range = total_time_length;
- private static boolean enableRegularityTimeDecode = false;
+ private static boolean enableRegularityTimeDecode = true;
private static long regularTimeInterval = 511996L;
private static String approach = "mac"; // 选择查询执行算法: 1: MAC, 2: MOC, 3: CPV
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BytesUtils.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BytesUtils.java
index 94e9d83548..95cb627a68 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BytesUtils.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BytesUtils.java
@@ -588,10 +588,34 @@ public class BytesUtils {
} else {
// put bits in the byte from the global position pos to pos+width-1 into the long value
TsFileConstant.bytesToLong_byteNum1_smallByte++;
- int mask = (int) Math.pow(2, 8 - width) - 1; // TODO consider if this to make static
- mask = (~mask & 0xff) >> startPosInByte;
-// return (result[startByte] & 0xff & mask) >> (7 - endPosInByte);
+ // TODO precompute and reuse masks
+ int mask = 0;
+ if (width == 2) {
+ System.out.println("[RL]2");
+ switch (startPosInByte) {
+ case 0:
+ mask = 0b11000000;
+ break;
+ case 2:
+ mask = 0b00110000;
+ break;
+ case 4:
+ mask = 0b00001100;
+ break;
+ case 6:
+ mask = 0b00000011;
+ break;
+ default:
+ System.out.println("wrong!!!");
+ }
+ } else {
+ System.out.println("[RL]not 2");
+ mask = (int) Math.pow(2, 8 - width) - 1; // TODO consider if this to make static
+ mask = (~mask & 0xff) >> startPosInByte;
+ }
+
return (result[startByte] & mask) >> (7 - endPosInByte);
+ // here mask is positive so no need &0xff
}
}
// TODO if across two bytes
[iotdb] 01/03: width 8 multiple faster, otherwise no
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 082283ef4d16721c27d7a8eb48f3225eb3aabcb2
Author: Lei Rui <10...@qq.com>
AuthorDate: Mon Oct 10 15:39:59 2022 +0800
width 8 multiple faster, otherwise no
---
.../iotdb/session/MyBasicOperationTest2.java | 16 +-
...rationTest2.java => MyBasicOperationTest3.java} | 57 +++---
.../session/MyRealDataTest1_WriteAndQuery.java | 94 ++++-----
.../tsfile/common/constant/TsFileConstant.java | 26 ++-
.../encoding/decoder/DeltaBinaryDecoder.java | 42 ++--
.../org/apache/iotdb/tsfile/utils/BytesUtils.java | 211 +++++++++++++--------
6 files changed, 272 insertions(+), 174 deletions(-)
diff --git a/session/src/test/java/org/apache/iotdb/session/MyBasicOperationTest2.java b/session/src/test/java/org/apache/iotdb/session/MyBasicOperationTest2.java
index eed2b00e6e..819054b5e4 100644
--- a/session/src/test/java/org/apache/iotdb/session/MyBasicOperationTest2.java
+++ b/session/src/test/java/org/apache/iotdb/session/MyBasicOperationTest2.java
@@ -11,7 +11,7 @@ public class MyBasicOperationTest2 {
// op1: long v = BytesUtils.bytesToLong(deltaBuf, packWidth * i, packWidth);
// op2: put bytes as a whole into long
- int repeat = 1000000;
+ int repeat = 1;
DescriptiveStatistics op1 = new DescriptiveStatistics();
DescriptiveStatistics op2 = new DescriptiveStatistics();
for (int k = 0; k < repeat; k++) {
@@ -19,7 +19,7 @@ public class MyBasicOperationTest2 {
Random r = new Random();
int low = 0; // inclusive
int high = 256; // exclusive
- int packNum = 128;
+ int packNum = 2;
int packWidth = 8; // equal to one byte length
byte[] buf = new byte[packNum];
for (int i = 0; i < packNum; i++) {
@@ -32,7 +32,7 @@ public class MyBasicOperationTest2 {
long start = System.nanoTime();
for (int i = 0; i < packNum; i++) {
sum += BytesUtils.bytesToLong(buf, packWidth * i, packWidth);
-// System.out.println(BytesUtils.bytesToLong(buf, packWidth * i, packWidth));
+ System.out.println(BytesUtils.bytesToLong(buf, packWidth * i, packWidth));
}
long elapsedTime = System.nanoTime() - start;
System.out.println(elapsedTime / 1000.0 + "us");
@@ -47,7 +47,7 @@ public class MyBasicOperationTest2 {
sum += (buf[i] & 0xff) << 8; // &0xff is to convert bytes to unsigned bytes
// op2_b: 把一个byte的高位x个比特装到一个long的低位x个比特
// TODO 如何把一个byte一次分成高位x个比特和低位y个比特
-// System.out.println("---");
+ System.out.println("---");
// System.out.println(buf[i]);
// System.out.println((buf[i] & 0xff & 0b11100000) >> 5);
sum += (buf[i] & 0xff & 0b11100000) >> 5;
@@ -55,6 +55,14 @@ public class MyBasicOperationTest2 {
// op2_c: 把一个byte的低位y个比特装到一个long的从低位到高位pos=packWidth-1的开始
// System.out.println((buf[i] & 0xff & ~0b11100000) << 3);
sum += (buf[i] & 0xff & ~0b11100000) << 3;
+
+ // op2_d: 把一个byte里的一部分比特装到一个long里
+ int oneNum = 2;
+ int mask = (int) Math.pow(2, oneNum) - 1;
+ int shift = 4;
+ mask = mask << shift; // 0b00110000
+ System.out.println(mask);
+ System.out.println((buf[i] & 0xff & mask) >> shift);
}
elapsedTime = System.nanoTime() - start;
System.out.println(elapsedTime / 1000.0 + "us");
diff --git a/session/src/test/java/org/apache/iotdb/session/MyBasicOperationTest2.java b/session/src/test/java/org/apache/iotdb/session/MyBasicOperationTest3.java
similarity index 66%
copy from session/src/test/java/org/apache/iotdb/session/MyBasicOperationTest2.java
copy to session/src/test/java/org/apache/iotdb/session/MyBasicOperationTest3.java
index eed2b00e6e..a281b1c117 100644
--- a/session/src/test/java/org/apache/iotdb/session/MyBasicOperationTest2.java
+++ b/session/src/test/java/org/apache/iotdb/session/MyBasicOperationTest3.java
@@ -3,15 +3,19 @@ package org.apache.iotdb.session;
import java.text.DecimalFormat;
import java.util.Random;
import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics;
+import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
import org.apache.iotdb.tsfile.utils.BytesUtils;
+import org.junit.Assert;
-public class MyBasicOperationTest2 {
+public class MyBasicOperationTest3 {
public static void main(String[] args) {
// op1: long v = BytesUtils.bytesToLong(deltaBuf, packWidth * i, packWidth);
- // op2: put bytes as a whole into long
+ // op2: put bytes as a whole into long, i.e., BytesUtils.bytesToLong2(deltaBuf, packWidth * i, packWidth);
int repeat = 1000000;
+ int packNum = 128;
+ int packWidth = 2;
DescriptiveStatistics op1 = new DescriptiveStatistics();
DescriptiveStatistics op2 = new DescriptiveStatistics();
for (int k = 0; k < repeat; k++) {
@@ -19,52 +23,51 @@ public class MyBasicOperationTest2 {
Random r = new Random();
int low = 0; // inclusive
int high = 256; // exclusive
- int packNum = 128;
- int packWidth = 8; // equal to one byte length
- byte[] buf = new byte[packNum];
+ byte[] buf = new byte[packNum * 8];
for (int i = 0; i < packNum; i++) {
- int v = r.nextInt(high - low) + low;
+// int v = r.nextInt(high - low) + low;
+ int v = 187;
BytesUtils.longToBytes(v, buf, i * packWidth, packWidth);
}
// test op1
- long sum = 0;
+ long[] value1 = new long[packNum];
long start = System.nanoTime();
for (int i = 0; i < packNum; i++) {
- sum += BytesUtils.bytesToLong(buf, packWidth * i, packWidth);
+ value1[i] = BytesUtils.bytesToLong(buf, packWidth * i, packWidth);
// System.out.println(BytesUtils.bytesToLong(buf, packWidth * i, packWidth));
}
long elapsedTime = System.nanoTime() - start;
- System.out.println(elapsedTime / 1000.0 + "us");
- System.out.println(sum);
+// System.out.println(elapsedTime / 1000.0 + "us");
+// System.out.println(sum);
op1.addValue(elapsedTime / 1000.0);
// test op2
- sum = 0;
+ long[] value2 = new long[packNum];
start = System.nanoTime();
for (int i = 0; i < packNum; i++) {
- // op2_a: 把一个byte装到一个long里从低位到高位偏移offset，又即byte的最低位在long中的pos（从低到高从0开始数）
- sum += (buf[i] & 0xff) << 8; // &0xff is to convert bytes to unsigned bytes
- // op2_b: 把一个byte的高位x个比特装到一个long的低位x个比特
- // TODO 如何把一个byte一次分成高位x个比特和低位y个比特
-// System.out.println("---");
-// System.out.println(buf[i]);
-// System.out.println((buf[i] & 0xff & 0b11100000) >> 5);
- sum += (buf[i] & 0xff & 0b11100000) >> 5;
-
- // op2_c: 把一个byte的低位y个比特装到一个long的从低位到高位pos=packWidth-1的开始
-// System.out.println((buf[i] & 0xff & ~0b11100000) << 3);
- sum += (buf[i] & 0xff & ~0b11100000) << 3;
+ value2[i] = BytesUtils.bytesToLong2(buf, packWidth * i, packWidth);
+// System.out.println(BytesUtils.bytesToLong2(buf, packWidth * i, packWidth));
+// Assert.assertEquals(value1[i], value2[i]);
}
elapsedTime = System.nanoTime() - start;
- System.out.println(elapsedTime / 1000.0 + "us");
- System.out.println(sum);
+// System.out.println(elapsedTime / 1000.0 + "us");
+// System.out.println(sum2);
op2.addValue(elapsedTime / 1000.0);
+
+ System.out.println("---------------");
+ for (int i = 0; i < packNum; i++) {
+ Assert.assertEquals(value1[i], value2[i]);
+ }
}
- printStat(op1, "op1-convertBitToLong");
- printStat(op2, "op2-compareByte");
+ printStat(op1, "op1-bytesToLong");
+ printStat(op2, "op2-bytesToLong2");
System.out.println("op1/op2=" + op1.getMean() / op2.getMean());
System.out.println("op2/op1=" + op2.getMean() / op1.getMean());
+ System.out.println("repeat=" + repeat);
+ System.out.println("packNum=" + packNum);
+ System.out.println("packWidth=" + packWidth);
+ TsFileConstant.printByteToLongStatistics();
}
private static String printStat(DescriptiveStatistics statistics, String name) {
diff --git a/session/src/test/java/org/apache/iotdb/session/MyRealDataTest1_WriteAndQuery.java b/session/src/test/java/org/apache/iotdb/session/MyRealDataTest1_WriteAndQuery.java
index c6fa88c7a7..6024cd118a 100644
--- a/session/src/test/java/org/apache/iotdb/session/MyRealDataTest1_WriteAndQuery.java
+++ b/session/src/test/java/org/apache/iotdb/session/MyRealDataTest1_WriteAndQuery.java
@@ -53,29 +53,29 @@ public class MyRealDataTest1_WriteAndQuery {
private static final String queryFormat_UDF =
"select M4(%1$s,'tqs'='%3$d','tqe'='%4$d','w'='%5$d') from %2$s where time>=%3$d and time<%4$d";
- // private static String device = "root.game";
- // private static String measurement = "s6";
- // private static TSDataType tsDataType = TSDataType.INT64; // TSDataType.DOUBLE;
- // private static String timestamp_precision = "ns"; // ns, us, ms
- // private static long dataMinTime = 0;
- // private static long dataMaxTime = 617426057626L;
- // private static long total_time_length = dataMaxTime - dataMinTime;
- // private static int total_point_number = 1200000;
- // private static int iotdb_chunk_point_size = 100000;
- // private static long chunkAvgTimeLen = (long) Math
- // .ceil(total_time_length / Math.ceil(total_point_number * 1.0 / iotdb_chunk_point_size));
- // private static String filePath =
- //
- // "D:\\github\\m4-lsm\\M4-visualization-exp\\src\\main\\java\\org\\apache\\iotdb\\datasets\\BallSpeed.csv";
- // private static int deletePercentage = 0; // 0 means no deletes. 0-100
- // private static int deleteLenPercentage = 0; // 0-100 每次删除的时间长度，用chunkAvgTimeLen的百分比表示
- // private static int timeIdx = 0; // 时间戳idx，从0开始
- // private static int valueIdx = 1; // 值idx，从0开始
- // private static int w = 2;
- // private static long range = total_time_length;
- // private static boolean enableRegularityTimeDecode = true;
- // private static long regularTimeInterval = 511996L;
- // private static String approach = "mac"; // 选择查询执行算法: 1: MAC, 2: MOC, 3: CPV
+ private static String device = "root.game";
+ private static String measurement = "s6";
+ private static TSDataType tsDataType = TSDataType.INT64; // TSDataType.DOUBLE;
+ private static String timestamp_precision = "ns"; // ns, us, ms
+ private static long dataMinTime = 0;
+ private static long dataMaxTime = 617426057626L;
+ private static long total_time_length = dataMaxTime - dataMinTime;
+ private static int total_point_number = 1200000;
+ private static int iotdb_chunk_point_size = 100000;
+ private static long chunkAvgTimeLen = (long) Math
+ .ceil(total_time_length / Math.ceil(total_point_number * 1.0 / iotdb_chunk_point_size));
+ private static String filePath =
+
+ "D:\\github\\m4-lsm\\M4-visualization-exp\\src\\main\\java\\org\\apache\\iotdb\\datasets\\BallSpeed.csv";
+ private static int deletePercentage = 0; // 0 means no deletes. 0-100
+ private static int deleteLenPercentage = 0; // 0-100 每次删除的时间长度，用chunkAvgTimeLen的百分比表示
+ private static int timeIdx = 0; // 时间戳idx，从0开始
+ private static int valueIdx = 1; // 值idx，从0开始
+ private static int w = 2;
+ private static long range = total_time_length;
+ private static boolean enableRegularityTimeDecode = false;
+ private static long regularTimeInterval = 511996L;
+ private static String approach = "mac"; // 选择查询执行算法: 1: MAC, 2: MOC, 3: CPV
// private static String device = "root.debs2012";
// private static String measurement = "mf03";
@@ -101,30 +101,30 @@ public class MyRealDataTest1_WriteAndQuery {
// private static long regularTimeInterval = 10000900L;
// private static String approach = "mac"; // 选择查询执行算法: 1: MAC, 2: MOC, 3: CPV
- private static String device = "root.debs2012";
- private static String measurement = "mf03";
- private static TSDataType tsDataType = TSDataType.INT64; // TSDataType.DOUBLE;
- private static String timestamp_precision = "ns"; // ns, us, ms
- private static long dataMinTime = 1329955200008000000L;
- private static long dataMaxTime = 1329965999991000000L;
- private static long total_time_length = dataMaxTime - dataMinTime;
- private static int total_point_number = 1076102;
- private static int iotdb_chunk_point_size = 100000;
- private static long chunkAvgTimeLen =
- (long)
- Math.ceil(
- total_time_length / Math.ceil(total_point_number * 1.0 / iotdb_chunk_point_size));
- private static String filePath =
- "D:\\github\\m4-lsm\\M4-visualization-exp\\src\\main\\java\\org\\apache\\iotdb\\datasets\\MF03_2.csv";
- private static int deletePercentage = 0; // 0 means no deletes. 0-100
- private static int deleteLenPercentage = 0; // 0-100 每次删除的时间长度，用chunkAvgTimeLen的百分比表示
- private static int timeIdx = 0; // 时间戳idx，从0开始
- private static int valueIdx = 1; // 值idx，从0开始
- private static int w = 2;
- private static long range = total_time_length;
- private static boolean enableRegularityTimeDecode = true;
- private static long regularTimeInterval = 10000000L;
- private static String approach = "mac"; // 选择查询执行算法: 1: MAC, 2: MOC, 3: CPV
+// private static String device = "root.debs2012";
+// private static String measurement = "mf03";
+// private static TSDataType tsDataType = TSDataType.INT64; // TSDataType.DOUBLE;
+// private static String timestamp_precision = "ns"; // ns, us, ms
+// private static long dataMinTime = 1329955200008000000L;
+// private static long dataMaxTime = 1329965999991000000L;
+// private static long total_time_length = dataMaxTime - dataMinTime;
+// private static int total_point_number = 1076102;
+// private static int iotdb_chunk_point_size = 100000;
+// private static long chunkAvgTimeLen =
+// (long)
+// Math.ceil(
+// total_time_length / Math.ceil(total_point_number * 1.0 / iotdb_chunk_point_size));
+// private static String filePath =
+// "D:\\github\\m4-lsm\\M4-visualization-exp\\src\\main\\java\\org\\apache\\iotdb\\datasets\\MF03_2.csv";
+// private static int deletePercentage = 0; // 0 means no deletes. 0-100
+// private static int deleteLenPercentage = 0; // 0-100 每次删除的时间长度，用chunkAvgTimeLen的百分比表示
+// private static int timeIdx = 0; // 时间戳idx，从0开始
+// private static int valueIdx = 1; // 值idx，从0开始
+// private static int w = 2;
+// private static long range = total_time_length;
+// private static boolean enableRegularityTimeDecode = true;
+// private static long regularTimeInterval = 10000000L;
+// private static String approach = "mac"; // 选择查询执行算法: 1: MAC, 2: MOC, 3: CPV
// private static String device = "root.kobelco.trans.03.1090001603.2401604";
// private static String measurement = "KOB_0002_00_67";
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/constant/TsFileConstant.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/constant/TsFileConstant.java
index 75be680e75..bb9492fdb2 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/constant/TsFileConstant.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/constant/TsFileConstant.java
@@ -53,5 +53,29 @@ public class TsFileConstant {
public static DescriptiveStatistics prepareAllRegulars = new DescriptiveStatistics();
- private TsFileConstant() {}
+ public static long bytesToLong_byteNum1_wholeByte = 0;
+ public static long bytesToLong_byteNum1_smallByte = 0;
+ public static long byteToLong_byteNums_firstByte_wholeByte = 0;
+ public static long byteToLong_byteNums_firstByte_smallByte = 0;
+ public static long byteToLong_byteNums_lastByte_wholeByte = 0;
+ public static long byteToLong_byteNums_lastByte_smallByte = 0;
+ public static long byteToLong_byteNums_middleWholeByte = 0;
+
+ private TsFileConstant() {
+ }
+
+ public static void printByteToLongStatistics() {
+ System.out.println("bytesToLong_byteNum1_wholeByte=" + bytesToLong_byteNum1_wholeByte);
+ System.out.println("bytesToLong_byteNum1_smallByte=" + bytesToLong_byteNum1_smallByte);
+ System.out.println(
+ "byteToLong_byteNums_firstByte_wholeByte=" + byteToLong_byteNums_firstByte_wholeByte);
+ System.out.println(
+ "byteToLong_byteNums_firstByte_smallByte=" + byteToLong_byteNums_firstByte_smallByte);
+ System.out.println(
+ "byteToLong_byteNums_lastByte_wholeByte=" + byteToLong_byteNums_lastByte_wholeByte);
+ System.out.println(
+ "byteToLong_byteNums_lastByte_smallByte=" + byteToLong_byteNums_lastByte_smallByte);
+ System.out
+ .println("byteToLong_byteNums_middleWholeByte=" + byteToLong_byteNums_middleWholeByte);
+ }
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java
index 4f91f0b589..e799d33268 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java
@@ -19,6 +19,10 @@
package org.apache.iotdb.tsfile.encoding.decoder;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
import org.apache.iotdb.tsfile.encoding.encoder.DeltaBinaryEncoder;
@@ -27,15 +31,9 @@ import org.apache.iotdb.tsfile.utils.BytesUtils;
import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.HashMap;
-import java.util.Map;
-
/**
* This class is a decoder for decoding the byte array that encoded by {@code
- * DeltaBinaryEncoder}.DeltaBinaryDecoder just supports integer and long values.<br>
- * .
+ * DeltaBinaryEncoder}.DeltaBinaryDecoder just supports integer and long values.<br> .
*
* @see DeltaBinaryEncoder
*/
@@ -44,16 +42,24 @@ public abstract class DeltaBinaryDecoder extends Decoder {
protected long count = 0;
protected byte[] deltaBuf;
- /** the first value in one pack. */
+ /**
+ * the first value in one pack.
+ */
protected int readIntTotalCount = 0;
protected int nextReadIndex = 0;
- /** max bit length of all value in a pack. */
+ /**
+ * max bit length of all value in a pack.
+ */
protected int packWidth;
- /** data number in this pack. */
+ /**
+ * data number in this pack.
+ */
protected int packNum;
- /** how many bytes data takes after encoding. */
+ /**
+ * how many bytes data takes after encoding.
+ */
protected int encodingLength;
public DeltaBinaryDecoder() {
@@ -86,7 +92,9 @@ public abstract class DeltaBinaryDecoder extends Decoder {
private int firstValue;
private int[] data;
private int previous;
- /** minimum value for all difference. */
+ /**
+ * minimum value for all difference.
+ */
private int minDeltaBase;
public IntDeltaDecoder() {
@@ -170,7 +178,9 @@ public abstract class DeltaBinaryDecoder extends Decoder {
private long firstValue;
private long[] data;
private long previous;
- /** minimum value for all difference. */
+ /**
+ * minimum value for all difference.
+ */
private long minDeltaBase;
private boolean enableRegularityTimeDecode;
@@ -239,7 +249,7 @@ public abstract class DeltaBinaryDecoder extends Decoder {
}
} else if (newRegularDelta < 0
|| newRegularDelta
- >= Math.pow(2, packWidth)) { // no need to compare equality cause impossible
+ >= Math.pow(2, packWidth)) { // no need to compare equality cause impossible
for (int i = 0; i < packNum; i++) {
long v = BytesUtils.bytesToLong(deltaBuf, packWidth * i, packWidth);
data[i] = previous + minDeltaBase + v;
@@ -267,6 +277,8 @@ public abstract class DeltaBinaryDecoder extends Decoder {
if (newRegularDelta != 0) {
// otherwise newRegularDelta=0 so leave byteArray as initial zeros
+ // TODO consider if these steps can be accelerated
+
// put bit-packed newRegularDelta starting at position i,
// and pad the front and back with newRegularDeltas
// 1. deal with padding the first byte
@@ -345,7 +357,7 @@ public abstract class DeltaBinaryDecoder extends Decoder {
// System.out.println("[RL]equals");
// TsFileConstant.countForRegularEqual++;
} else {
- long v = BytesUtils.bytesToLong(deltaBuf, packWidth * i, packWidth);
+ long v = BytesUtils.bytesToLong2(deltaBuf, packWidth * i, packWidth);
data[i] = previous + minDeltaBase + v;
// System.out.println("[RL]no");
// TsFileConstant.countForRegularNOTEqual++;
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BytesUtils.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BytesUtils.java
index 622b228731..49dee499e8 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BytesUtils.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BytesUtils.java
@@ -18,26 +18,24 @@
*/
package org.apache.iotdb.tsfile.utils;
-import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
+import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
+import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* BytesUtils is a utility class. It provide conversion among byte array and other type including
- * integer, long, float, boolean, double and string. <br>
- * It also provide other usable function as follow:<br>
- * reading function which receives InputStream. <br>
- * concat function to join a list of byte array to one.<br>
- * get and set one bit in a byte array.
+ * integer, long, float, boolean, double and string. <br> It also provide other usable function as
+ * follow:<br> reading function which receives InputStream. <br> concat function to join a list of
+ * byte array to one.<br> get and set one bit in a byte array.
*/
public class BytesUtils {
- private BytesUtils() {}
+ private BytesUtils() {
+ }
private static final Logger LOG = LoggerFactory.getLogger(BytesUtils.class);
@@ -48,19 +46,20 @@ public class BytesUtils {
* @return byte[4] for integer
*/
public static byte[] intToBytes(int i) {
- return new byte[] {
- (byte) ((i >> 24) & 0xFF),
- (byte) ((i >> 16) & 0xFF),
- (byte) ((i >> 8) & 0xFF),
- (byte) (i & 0xFF)
+ return new byte[]{
+ (byte) ((i >> 24) & 0xFF),
+ (byte) ((i >> 16) & 0xFF),
+ (byte) ((i >> 8) & 0xFF),
+ (byte) (i & 0xFF)
};
}
/**
- * integer convert to byte array, then write four bytes to parameter desc start from index:offset.
+ * integer convert to byte array, then write four bytes to parameter desc start from
+ * index:offset.
*
- * @param i integer to convert
- * @param desc byte array be written
+ * @param i integer to convert
+ * @param desc byte array be written
* @param offset position in desc byte array that conversion result should start
* @return byte array
*/
@@ -81,8 +80,8 @@ public class BytesUtils {
*
* @param srcNum input integer variable
* @param result byte array to convert
- * @param pos start position
- * @param width bit-width
+ * @param pos start position
+ * @param width bit-width
*/
public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
int temp = 0;
@@ -160,7 +159,7 @@ public class BytesUtils {
/**
* convert four-bytes byte array cut from parameters to integer.
*
- * @param bytes source bytes which length should be greater than 4
+ * @param bytes source bytes which length should be greater than 4
* @param offset position in parameter byte array that conversion result should start
* @return integer
*/
@@ -182,8 +181,8 @@ public class BytesUtils {
* given a byte array, read width bits from specified position bits and convert it to an integer.
*
* @param result input byte array
- * @param pos bit offset rather than byte offset
- * @param width bit-width
+ * @param pos bit offset rather than byte offset
+ * @param width bit-width
* @return integer variable
*/
public static int bytesToInt(byte[] result, int pos, int width) {
@@ -217,8 +216,8 @@ public class BytesUtils {
/**
* float convert to boolean, then write four bytes to parameter desc start from index:offset.
*
- * @param x float
- * @param desc byte array be written
+ * @param x float
+ * @param desc byte array be written
* @param offset position in desc byte array that conversion result should start
*/
public static void floatToBytes(float x, byte[] desc, int offset) {
@@ -257,7 +256,7 @@ public class BytesUtils {
/**
* convert four-bytes byte array cut from parameters to float.
*
- * @param b source bytes which length should be greater than 4
+ * @param b source bytes which length should be greater than 4
* @param offset position in parameter byte array that conversion result should start
* @return float
*/
@@ -296,8 +295,8 @@ public class BytesUtils {
/**
* convert double to byte into the given byte array started from offset.
*
- * @param d input double
- * @param bytes target byte[]
+ * @param d input double
+ * @param bytes target byte[]
* @param offset start pos
*/
public static void doubleToBytes(double d, byte[] bytes, int offset) {
@@ -340,7 +339,7 @@ public class BytesUtils {
/**
* convert eight-bytes byte array cut from parameters to double.
*
- * @param bytes source bytes which length should be greater than 8
+ * @param bytes source bytes which length should be greater than 8
* @param offset position in parameter byte array that conversion result should start
* @return double
*/
@@ -395,10 +394,11 @@ public class BytesUtils {
}
/**
- * boolean convert to byte array, then write four bytes to parameter desc start from index:offset.
+ * boolean convert to byte array, then write four bytes to parameter desc start from
+ * index:offset.
*
- * @param x input boolean
- * @param desc byte array be written
+ * @param x input boolean
+ * @param desc byte array be written
* @param offset position in desc byte array that conversion result should start
* @return byte[1]
*/
@@ -428,7 +428,7 @@ public class BytesUtils {
/**
* convert one-bytes byte array cut from parameters to boolean.
*
- * @param b source bytes which length should be greater than 1
+ * @param b source bytes which length should be greater than 1
* @param offset position in parameter byte array that conversion result should start
* @return boolean
*/
@@ -452,10 +452,9 @@ public class BytesUtils {
/**
* specify the result array length. then, convert long to Big-Endian byte from low to high. <br>
- * e.g.<br>
- * the binary presentation of long number 1000L is {6 bytes equal 0000000} 00000011 11101000<br>
- * if len = 2, it will return byte array :{00000011 11101000}(Big-Endian) if len = 1, it will
- * return byte array :{11101000}.
+ * e.g.<br> the binary presentation of long number 1000L is {6 bytes equal 0000000} 00000011
+ * 11101000<br> if len = 2, it will return byte array :{00000011 11101000}(Big-Endian) if len = 1,
+ * it will return byte array :{11101000}.
*
* @param num long variable to be converted
* @param len length of result byte array
@@ -472,8 +471,8 @@ public class BytesUtils {
/**
* long convert to byte array, then write four bytes to parameter desc start from index:offset.
*
- * @param num input long variable
- * @param desc byte array be written
+ * @param num input long variable
+ * @param desc byte array be written
* @param offset position in desc byte array that conversion result should start
* @return byte array
*/
@@ -491,8 +490,8 @@ public class BytesUtils {
*
* @param srcNum input long variable
* @param result byte array to convert
- * @param pos start position
- * @param width bit-width
+ * @param pos start position
+ * @param width bit-width
*/
public static void longToBytes(long srcNum, byte[] result, int pos, int width) {
int temp = 0;
@@ -524,12 +523,11 @@ public class BytesUtils {
/**
* specify the input byte array length. then, convert byte array to long value from low to high.
* <br>
- * e.g.<br>
- * the input byte array is {00000011 11101000}. if len = 2, return 1000 if len = 1, return
- * 232(only calculate the low byte).
+ * e.g.<br> the input byte array is {00000011 11101000}. if len = 2, return 1000 if len = 1,
+ * return 232(only calculate the low byte).
*
* @param byteNum byte array to be converted
- * @param len length of input byte array to be converted
+ * @param len length of input byte array to be converted
* @return long
*/
public static long bytesToLong(byte[] byteNum, int len) {
@@ -545,8 +543,8 @@ public class BytesUtils {
* given a byte array, read width bits from specified pos bits and convert it to an long.
*
* @param result input byte array
- * @param pos bit offset rather than byte offset
- * @param width bit-width
+ * @param pos bit offset rather than byte offset
+ * @param width bit-width
* @return long variable
*/
public static long bytesToLong(byte[] result, int pos, int width) {
@@ -563,28 +561,85 @@ public class BytesUtils {
* given a byte array, read width bits from specified pos bits and convert it to an long.
*
* @param result input byte array
- * @param pos bit offset rather than byte offset
- * @param width bit-width
+ * @param pos bit offset rather than byte offset
+ * @param width bit-width
* @return long variable
*/
public static long bytesToLong2(byte[] result, int pos, int width) {
- long value = 0;
-// int temp = 0;
-// for (int i = 0; i < width; i++) {
-// temp = (pos + width - 1 - i) / 8;
-// value = setLongN(value, i, getByteN(result[temp], pos + width - 1 - i));
-// }
- //TODO new implementation
+ // pos is global over the byte array, from low to high, starting from 0
+ // TODO new implementation
+ if (width == 0) {
+ return 0;
+ }
+
+ int endPos = pos + width - 1;
+ int startByte = pos / 8;
+ int endByte = endPos / 8;
+ int byteNum = endByte - startByte + 1;
+ int startPosInByte = pos % 8; // in a byte from high to low bits starting from 0
+ int endPosInByte = endPos % 8;
+
+ // TODO if in the same byte
+ if (byteNum == 1) {
+ if (endPosInByte - startPosInByte == 7) {
+ // put the whole byte into the long value
+ TsFileConstant.bytesToLong_byteNum1_wholeByte++;
+ return result[startByte] & 0xff;
+ } else {
+ // put bits in the byte from the global position pos to pos+width-1 into the long value
+ TsFileConstant.bytesToLong_byteNum1_smallByte++;
+ int mask = (int) Math.pow(2, 8 - width) - 1; // TODO consider if this to make static
+ mask = (~mask & 0xff) >> startPosInByte;
+ return (result[startByte] & 0xff & mask) >> (7 - endPosInByte);
+ }
+ }
+ // TODO if across two bytes
+ else {
+ long value = 0;
+ // 1. deal with the first byte
+ int shift = width - (8 - startPosInByte);
+ if (startPosInByte == 0) {
+ // put the whole byte into the long value's front place among the last width bits
+ TsFileConstant.byteToLong_byteNums_firstByte_wholeByte++;
+ value = value | ((result[startByte] & 0xff) << shift);
+ } else {
+ // put the bits in the first byte from relative position pos%8 to the end into the long value's front place among the last width bits
+ TsFileConstant.byteToLong_byteNums_firstByte_smallByte++;
+ int mask =
+ (int) Math.pow(2, 8 - startPosInByte) - 1; // TODO consider if this to make static
+ value = value | ((result[startByte] & 0xff & mask) << shift);
+ }
+
+ // 2. deal with the last byte
+ if (endPosInByte == 7) {
+ // put the whole byte into the long value's back place among the last width bits
+ TsFileConstant.byteToLong_byteNums_lastByte_wholeByte++;
+ value = value | (result[endByte] & 0xff);
+ } else {
+ // put the bits in the last byte from relative position 0 to (pos+width-1)%8 into the long value's back place among the last width bits
+ TsFileConstant.byteToLong_byteNums_lastByte_smallByte++;
+ int mask =
+ (int) Math.pow(2, 7 - endPosInByte) - 1; // TODO consider if this to make static
+ value = value | ((result[endByte] & 0xff & ~mask) >> (7 - endPosInByte));
+ }
+
+ // 3. deal with the middle bytes
+ for (int k = startByte + 1; k < endByte; k++) {
+ TsFileConstant.byteToLong_byteNums_middleWholeByte++;
+ shift -= 8;
+ value = value | ((result[k] & 0xff) << shift);
+ }
+ return value;
+ }
- return value;
}
/**
* convert eight-bytes byte array cut from parameters to long.
*
* @param byteNum source bytes which length should be greater than 8
- * @param len length of input byte array to be converted
- * @param offset position in parameter byte array that conversion result should start
+ * @param len length of input byte array to be converted
+ * @param offset position in parameter byte array that conversion result should start
* @return long
*/
public static long bytesToLongFromOffset(byte[] byteNum, int len, int offset) {
@@ -658,8 +713,8 @@ public class BytesUtils {
/**
* cut out specified length byte array from parameter start from input byte array src and return.
*
- * @param src input byte array
- * @param start start index of src
+ * @param src input byte array
+ * @param start start index of src
* @param length cut off length
* @return byte array
*/
@@ -676,12 +731,11 @@ public class BytesUtils {
}
/**
- * get one bit in input integer. the offset is from low to high and start with 0<br>
- * e.g.<br>
+ * get one bit in input integer. the offset is from low to high and start with 0<br> e.g.<br>
* data:1000(00000000 00000000 00000011 11101000), if offset is 4, return 0(111 "0" 1000) if
* offset is 9, return 1(00000 "1" 1 11101000).
*
- * @param data input int variable
+ * @param data input int variable
* @param offset bit offset
* @return 0 or 1
*/
@@ -696,14 +750,13 @@ public class BytesUtils {
/**
* set one bit in input integer. the offset is from low to high and start with index 0<br>
- * e.g.<br>
- * data:1000({00000000 00000000 00000011 11101000}), if offset is 4, value is 1, return
+ * e.g.<br> data:1000({00000000 00000000 00000011 11101000}), if offset is 4, value is 1, return
* 1016({00000000 00000000 00000011 111 "1" 1000}) if offset is 9, value is 0 return 488({00000000
* 00000000 000000 "0" 1 11101000}) if offset is 0, value is 0 return 1000(no change).
*
- * @param data input int variable
+ * @param data input int variable
* @param offset bit offset
- * @param value value to set
+ * @param value value to set
* @return int variable
*/
public static int setIntN(int data, int offset, int value) {
@@ -716,12 +769,11 @@ public class BytesUtils {
}
/**
- * get one bit in input byte. the offset is from low to high and start with 0<br>
- * e.g.<br>
+ * get one bit in input byte. the offset is from low to high and start with 0<br> e.g.<br>
* data:16(00010000), if offset is 4, return 1(000 "1" 0000) if offset is 7, return 0("0"
* 0010000).
*
- * @param data input byte variable
+ * @param data input byte variable
* @param offset bit offset
* @return 0/1
*/
@@ -735,14 +787,13 @@ public class BytesUtils {
}
/**
- * set one bit in input byte. the offset is from low to high and start with index 0<br>
- * e.g.<br>
+ * set one bit in input byte. the offset is from low to high and start with index 0<br> e.g.<br>
* data:16(00010000), if offset is 4, value is 0, return 0({000 "0" 0000}) if offset is 1, value
* is 1, return 18({00010010}) if offset is 0, value is 0, return 16(no change).
*
- * @param data input byte variable
+ * @param data input byte variable
* @param offset bit offset
- * @param value value to set
+ * @param value value to set
* @return byte variable
*/
public static byte setByteN(byte data, int offset, int value) {
@@ -757,7 +808,7 @@ public class BytesUtils {
/**
* get one bit in input long. the offset is from low to high and start with 0.
*
- * @param data input long variable
+ * @param data input long variable
* @param offset bit offset
* @return 0/1
*/
@@ -773,9 +824,9 @@ public class BytesUtils {
/**
* set one bit in input long. the offset is from low to high and start with index 0.
*
- * @param data input long variable
+ * @param data input long variable
* @param offset bit offset
- * @param value value to set
+ * @param value value to set
* @return long variable
*/
public static long setLongN(long data, int offset, int value) {
@@ -851,7 +902,7 @@ public class BytesUtils {
* read bytes specified length from InputStream safely.
*
* @param count number of byte to read
- * @param in InputStream
+ * @param in InputStream
* @return byte array
* @throws IOException cannot read from InputStream
*/