You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by le...@apache.org on 2022/10/10 09:25:03 UTC
[iotdb] branch research/M4-visualization updated: 80%
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/research/M4-visualization by this push:
new 7d9d7cf535 80%
7d9d7cf535 is described below
commit 7d9d7cf53567ba4031571eae291456fe145559c2
Author: Lei Rui <10...@qq.com>
AuthorDate: Mon Oct 10 17:25:10 2022 +0800
80%
---
.../iotdb/session/MyBasicOperationTest3.java | 17 ++++-
.../session/MyRealDataTest1_WriteAndQuery.java | 2 +-
.../tsfile/common/constant/TsFileConstant.java | 39 ++++++++++
.../encoding/decoder/DeltaBinaryDecoder.java | 87 ++++++++--------------
.../org/apache/iotdb/tsfile/utils/BytesUtils.java | 82 ++++++++++----------
5 files changed, 129 insertions(+), 98 deletions(-)
diff --git a/session/src/test/java/org/apache/iotdb/session/MyBasicOperationTest3.java b/session/src/test/java/org/apache/iotdb/session/MyBasicOperationTest3.java
index f03635041f..0064940891 100644
--- a/session/src/test/java/org/apache/iotdb/session/MyBasicOperationTest3.java
+++ b/session/src/test/java/org/apache/iotdb/session/MyBasicOperationTest3.java
@@ -1,5 +1,6 @@
package org.apache.iotdb.session;
+import java.io.IOException;
import java.text.DecimalFormat;
import java.util.Random;
import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics;
@@ -9,13 +10,21 @@ import org.junit.Assert;
public class MyBasicOperationTest3 {
- public static void main(String[] args) {
+ public static void main(String[] args) throws IOException {
// op1: long v = BytesUtils.bytesToLong(deltaBuf, packWidth * i, packWidth);
// op2: put bytes as a whole into long, i.e., BytesUtils.bytesToLong2(deltaBuf, packWidth * i, packWidth);
int repeat = 1000000;
int packNum = 128;
- int packWidth = 2;
+ int packWidth = 9;
+
+ int[] fallWithinMasks;
+ if (packWidth < 8) {
+ fallWithinMasks = TsFileConstant.generateFallWithinMasks(packWidth);
+ } else {
+ fallWithinMasks = null;
+ }
+
DescriptiveStatistics op1 = new DescriptiveStatistics();
DescriptiveStatistics op2 = new DescriptiveStatistics();
for (int k = 0; k < repeat; k++) {
@@ -26,7 +35,7 @@ public class MyBasicOperationTest3 {
byte[] buf = new byte[packNum * 8];
for (int i = 0; i < packNum; i++) {
int v = r.nextInt(high - low) + low;
-// int v = 187;
+// int v = 1;
BytesUtils.longToBytes(v, buf, i * packWidth, packWidth);
}
@@ -46,7 +55,7 @@ public class MyBasicOperationTest3 {
long[] value2 = new long[packNum];
start = System.nanoTime();
for (int i = 0; i < packNum; i++) {
- value2[i] = BytesUtils.bytesToLong2(buf, packWidth * i, packWidth);
+ value2[i] = BytesUtils.bytesToLong2(buf, packWidth * i, packWidth, fallWithinMasks);
// System.out.println(BytesUtils.bytesToLong2(buf, packWidth * i, packWidth));
// Assert.assertEquals(value1[i], value2[i]);
}
diff --git a/session/src/test/java/org/apache/iotdb/session/MyRealDataTest1_WriteAndQuery.java b/session/src/test/java/org/apache/iotdb/session/MyRealDataTest1_WriteAndQuery.java
index 10f8235500..6024cd118a 100644
--- a/session/src/test/java/org/apache/iotdb/session/MyRealDataTest1_WriteAndQuery.java
+++ b/session/src/test/java/org/apache/iotdb/session/MyRealDataTest1_WriteAndQuery.java
@@ -73,7 +73,7 @@ public class MyRealDataTest1_WriteAndQuery {
private static int valueIdx = 1; // 值idx，从0开始
private static int w = 2;
private static long range = total_time_length;
- private static boolean enableRegularityTimeDecode = true;
+ private static boolean enableRegularityTimeDecode = false;
private static long regularTimeInterval = 511996L;
private static String approach = "mac"; // 选择查询执行算法: 1: MAC, 2: MOC, 3: CPV
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/constant/TsFileConstant.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/constant/TsFileConstant.java
index bb9492fdb2..1ef5630c1c 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/constant/TsFileConstant.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/constant/TsFileConstant.java
@@ -18,6 +18,7 @@
*/
package org.apache.iotdb.tsfile.common.constant;
+import java.io.IOException;
import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics;
public class TsFileConstant {
@@ -61,6 +62,44 @@ public class TsFileConstant {
public static long byteToLong_byteNums_lastByte_smallByte = 0;
public static long byteToLong_byteNums_middleWholeByte = 0;
+ public static int[] endInByteMasks = {
+ 0b01111111,
+ 0b00111111,
+ 0b00011111,
+ 0b00001111,
+ 0b00000111,
+ 0b00000011,
+ 0b00000001
+ };
+
+ public static int[] startInByteMasks = {
+ 0b10000000,
+ 0b11000000,
+ 0b11100000,
+ 0b11110000,
+ 0b11111000,
+ 0b11111100,
+ 0b11111110
+ };
+
+ /**
+ * example: int[] fallWithinMasks_2 = { 0b11000000, 0b01100000, 0b00110000, 0b00011000,
+ * 0b00001100, 0b00000110, 0b00000011 }; int[] fallWithinMasks_3 = { 0b11100000, 0b01110000,
+ * 0b00111000, 0b00011100, 0b00001110, 0b00000111 };
+ */
+ public static int[] generateFallWithinMasks(int packWidth) throws IOException {
+ if (packWidth >= 8) {
+ throw new IOException("only accept packWidth smaller than 8.");
+ }
+ int num = 9 - packWidth;
+ int[] res = new int[num];
+ int mask = (int) Math.pow(2, packWidth) - 1;
+ for (int i = num - 1; i >= 0; i--) {
+ res[i] = mask << (num - 1 - i);
+ }
+ return res;
+ }
+
private TsFileConstant() {
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java
index e799d33268..efa704e1b6 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java
@@ -191,6 +191,8 @@ public abstract class DeltaBinaryDecoder extends Decoder {
private Map<Pair<Long, Integer>, Map<Integer, byte[]>> allRegularBytes =
new HashMap<>(); // <newRegularDelta,packWidth> -> (relativePos->bytes)
+ private Map<Integer, int[]> allFallWithinMasks = new HashMap<>(); // packWidth -> fallWithinMasks
+
public LongDeltaDecoder() {
super();
this.enableRegularityTimeDecode =
@@ -237,50 +239,50 @@ public abstract class DeltaBinaryDecoder extends Decoder {
if (enableRegularityTimeDecode) {
long newRegularDelta = regularTimeInterval - minDeltaBase;
- // System.out.println("newRegularDelta = " + newRegularDelta);
- // TsFileConstant.regularNewDeltasStatistics.addValue(newRegularDelta);
-
if (packWidth == 0) {
+ // [CASE 1]
for (int i = 0; i < packNum; i++) {
data[i] = previous + minDeltaBase; // v=0
previous = data[i];
- // System.out.println("[RL]0");
- // TsFileConstant.countForRegularZero++;
}
- } else if (newRegularDelta < 0
- || newRegularDelta
- >= Math.pow(2, packWidth)) { // no need to compare equality cause impossible
+ } else if (newRegularDelta < 0 || newRegularDelta >= Math.pow(2, packWidth)) {
+ // [CASE 2] no need to compare equality cause impossible
for (int i = 0; i < packNum; i++) {
long v = BytesUtils.bytesToLong(deltaBuf, packWidth * i, packWidth);
data[i] = previous + minDeltaBase + v;
previous = data[i];
- // System.out.println("[RL]no");
- // TsFileConstant.countForRegularNOTEqual++;
}
} else {
- // long start1 = System.nanoTime();
+ // [CASE 3]
+ // preprocess to get fallWithinMasks and regularBytes
+ int[] fallWithinMasks = null;
+ if (packWidth >= 8) {
+ fallWithinMasks = null;
+ } else if (allFallWithinMasks.containsKey(packWidth)) {
+ fallWithinMasks = allFallWithinMasks.get(packWidth);
+ } else { // packWidth<8 and allFallWithinMasks does not contain it
+ try {
+ fallWithinMasks = TsFileConstant.generateFallWithinMasks(packWidth);
+ allFallWithinMasks.put(packWidth, fallWithinMasks);
+ } catch (Exception ignored) {
+ }
+ }
Map<Integer, byte[]> regularBytes;
if (allRegularBytes.containsKey(new Pair<>(newRegularDelta, packWidth))) {
regularBytes = allRegularBytes.get(new Pair<>(newRegularDelta, packWidth));
- // System.out.println("here");
- // TsFileConstant.countForHitNewDeltas.addValue(1);
- } else {
- // System.out.println("here");
- // TsFileConstant.countForNotHitNewDeltas.addValue(1);
-
+ } else { // TODO consider if the following steps can be accelerated by using bytes instead of bitwise get and set
regularBytes = new HashMap<>();
for (int i = 0; i < 8; i++) {
// i is the starting position in the byte from high to low bits
+
int endPos = i + packWidth - 1; // starting from 0
int byteNum = endPos / 8 + 1;
byte[] byteArray = new byte[byteNum];
if (newRegularDelta != 0) {
- // otherwise newRegularDelta=0 so leave byteArray as initial zeros
-
- // TODO consider if these steps can be accelerated
-
// put bit-packed newRegularDelta starting at position i,
- // and pad the front and back with newRegularDeltas
+ // and pad the front and back with newRegularDeltas.
+ // Otherwise if newRegularDelta=0, just leave byteArray as initial zeros
+
// 1. deal with padding the first byte
for (int x = i - 1; x >= 0; x--) {
// y is the position in the bit-packed newRegularDelta, 0->packWidth-1 from low to
@@ -312,57 +314,37 @@ public abstract class DeltaBinaryDecoder extends Decoder {
}
allRegularBytes.put(new Pair<>(newRegularDelta, packWidth), regularBytes);
}
- // long elapsedTime1 = System.nanoTime() - start1;
- // TsFileConstant.prepareAllRegulars.addValue(elapsedTime1 / 1000.0); // us
+ // Begin decoding each number in this pack
for (int i = 0; i < packNum; i++) {
- // (1) extract bits from deltaBuf,
- // (2) compare bits with encodedRegularTimeInterval,
+ // (1) extract bytes from deltaBuf,
+ // (2) compare bytes with encodedRegularTimeInterval,
// (3) equal to reuse, else to convert
boolean equal = true;
- // the starting position in the byte from high to low bits
- int pos = i * packWidth % 8;
+ int pos = i * packWidth
+ % 8; // the starting relative position in the byte from high to low bits
- byte[] byteArray = regularBytes.get(pos);
+ byte[] byteArray = regularBytes.get(pos); // the regular padded bytes to be compared
+
+ int posByteIdx = i * packWidth / 8; // the start byte of the encoded new delta
- int posByteIdx = i * packWidth / 8;
- // System.out.println("byteArray length=" + byteArray.length);
- // TsFileConstant.byteArrayLengthStatistics.addValue(byteArray.length);
for (int k = 0; k < byteArray.length; k++, posByteIdx++) {
byte regular = byteArray[k];
byte data = deltaBuf[posByteIdx];
if (regular != data) {
equal = false;
- // System.out.println("k=" + k);
break;
}
}
- // int posByteIdx = ((i + 1) * packWidth - 1) / 8;
- // for (int k = byteArray.length - 1; k >= 0; k--, posByteIdx--) {
- // // compare the lower bytes first
- // byte regular = byteArray[k];
- // byte data = deltaBuf[posByteIdx];
- // if (regular != data) {
- // equal = false;
- //// System.out.println("k'=" + (byteArray.length - 1 - k));
- // break;
- // }
- // }
-
if (equal) {
data[i] = previous + regularTimeInterval;
- // System.out.println("[RL]equals");
- // TsFileConstant.countForRegularEqual++;
} else {
- long v = BytesUtils.bytesToLong2(deltaBuf, packWidth * i, packWidth);
+ long v = BytesUtils.bytesToLong2(deltaBuf, packWidth * i, packWidth, fallWithinMasks);
data[i] = previous + minDeltaBase + v;
- // System.out.println("[RL]no");
- // TsFileConstant.countForRegularNOTEqual++;
}
- // data[i] = previous + regularTimeInterval;
previous = data[i];
}
}
@@ -370,9 +352,6 @@ public abstract class DeltaBinaryDecoder extends Decoder {
readPack();
}
- // TsFileConstant.allRegularBytesSize.addValue(allRegularBytes.size());
- // System.out.println("allRegularBytes size=" + allRegularBytes.size());
-
long runTime = System.nanoTime() - start; // ns
TsFileConstant.timeColumnTS2DIFFLoadBatchCost.addValue(runTime / 1000.0); // us
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BytesUtils.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BytesUtils.java
index 95cb627a68..1829b41a62 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BytesUtils.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BytesUtils.java
@@ -560,12 +560,15 @@ public class BytesUtils {
/**
* given a byte array, read width bits from specified pos bits and convert it to an long.
*
- * @param result input byte array
- * @param pos bit offset rather than byte offset
- * @param width bit-width
+ * @param result input byte array
+ * @param pos bit offset rather than byte offset
+ * @param width bit-width
+ * @param fallWithinMasks if width is a multiple of 8 or greater than 8, then this parameter null;
+ * if width<8, need this parameter to be (9-width) masks that are fully
+ * within a byte.
* @return long variable
*/
- public static long bytesToLong2(byte[] result, int pos, int width) {
+ public static long bytesToLong2(byte[] result, int pos, int width, int[] fallWithinMasks) {
// pos is global over the byte array, from low to high, starting from 0
// TODO new implementation
if (width == 0) {
@@ -583,36 +586,35 @@ public class BytesUtils {
if (byteNum == 1) {
if (endPosInByte - startPosInByte == 7) {
// put the whole byte into the long value
- TsFileConstant.bytesToLong_byteNum1_wholeByte++;
+// TsFileConstant.bytesToLong_byteNum1_wholeByte++;
return result[startByte] & 0xff;
} else {
// put bits in the byte from the global position pos to pos+width-1 into the long value
- TsFileConstant.bytesToLong_byteNum1_smallByte++;
+// TsFileConstant.bytesToLong_byteNum1_smallByte++;
// TODO precompute and reuse masks
- int mask = 0;
- if (width == 2) {
- System.out.println("[RL]2");
- switch (startPosInByte) {
- case 0:
- mask = 0b11000000;
- break;
- case 2:
- mask = 0b00110000;
- break;
- case 4:
- mask = 0b00001100;
- break;
- case 6:
- mask = 0b00000011;
- break;
- default:
- System.out.println("wrong!!!");
- }
- } else {
- System.out.println("[RL]not 2");
- mask = (int) Math.pow(2, 8 - width) - 1; // TODO consider if this to make static
- mask = (~mask & 0xff) >> startPosInByte;
- }
+ int mask = fallWithinMasks[startPosInByte];
+// int mask = 0;
+// if (width == 2) {
+// switch (startPosInByte) {
+// case 0:
+// mask = 0b11000000;
+// break;
+// case 2:
+// mask = 0b00110000;
+// break;
+// case 4:
+// mask = 0b00001100;
+// break;
+// case 6:
+// mask = 0b00000011;
+// break;
+// default:
+// System.out.println("wrong!!!");
+// }
+// } else {
+// mask = (int) Math.pow(2, 8 - width) - 1; // TODO consider if this to make static
+// mask = (~mask & 0xff) >> startPosInByte;
+// }
return (result[startByte] & mask) >> (7 - endPosInByte);
// here mask is positive so no need &0xff
@@ -625,33 +627,35 @@ public class BytesUtils {
int shift = width - (8 - startPosInByte);
if (startPosInByte == 0) {
// put the whole byte into the long value's front place among the last width bits
- TsFileConstant.byteToLong_byteNums_firstByte_wholeByte++;
+// TsFileConstant.byteToLong_byteNums_firstByte_wholeByte++;
value = value | ((result[startByte] & 0xff) << shift);
} else {
// put the bits in the first byte from relative position pos%8 to the end into the long value's front place among the last width bits
- TsFileConstant.byteToLong_byteNums_firstByte_smallByte++;
- int mask =
- (int) Math.pow(2, 8 - startPosInByte) - 1; // TODO consider if this to make static
+// TsFileConstant.byteToLong_byteNums_firstByte_smallByte++;
+// int mask =
+// (int) Math.pow(2, 8 - startPosInByte) - 1; // TODO consider if this to make static
+ int mask = TsFileConstant.endInByteMasks[startPosInByte - 1];
value = value | ((result[startByte] & mask) << shift);
}
// 2. deal with the last byte
if (endPosInByte == 7) {
// put the whole byte into the long value's back place among the last width bits
- TsFileConstant.byteToLong_byteNums_lastByte_wholeByte++;
+// TsFileConstant.byteToLong_byteNums_lastByte_wholeByte++;
value = value | (result[endByte] & 0xff);
} else {
// put the bits in the last byte from relative position 0 to (pos+width-1)%8 into the long value's back place among the last width bits
- TsFileConstant.byteToLong_byteNums_lastByte_smallByte++;
- int mask =
- (int) Math.pow(2, 7 - endPosInByte) - 1; // TODO consider if this to make static
+// TsFileConstant.byteToLong_byteNums_lastByte_smallByte++;
+// int mask =
+// (int) Math.pow(2, 7 - endPosInByte) - 1; // TODO consider if this to make static
+ int mask = TsFileConstant.endInByteMasks[endPosInByte];
value = value | ((result[endByte] & 0xff & ~mask) >> (7
- endPosInByte)); // here mask is negative so need &0xff
}
// 3. deal with the middle bytes
for (int k = startByte + 1; k < endByte; k++) {
- TsFileConstant.byteToLong_byteNums_middleWholeByte++;
+// TsFileConstant.byteToLong_byteNums_middleWholeByte++;
shift -= 8;
value = value | ((result[k] & 0xff) << shift);
}