You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2021/03/23 07:57:01 UTC
[iotdb] branch NullableVector updated: [To Vector] add bitmap in
tablet (#2858)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch NullableVector
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/NullableVector by this push:
new 23407b3 [To Vector] add bitmap in tablet (#2858)
23407b3 is described below
commit 23407b35a94c805929cec17715069de830966a36
Author: CRZbulabula <33...@users.noreply.github.com>
AuthorDate: Tue Mar 23 15:56:44 2021 +0800
[To Vector] add bitmap in tablet (#2858)
Use bitmap written by jixuan1989
Co-authored-by: CRZbulabula <cr...@gmail.com>
Co-authored-by: Haonan <hh...@outlook.com>
---
.../org/apache/iotdb/session/SessionUtils.java | 47 ++++++--
.../apache/iotdb/session/IoTDBSessionSimpleIT.java | 12 +-
.../apache/iotdb/tsfile/write/record/BitMap.java | 128 +++++++++++++++++++++
.../apache/iotdb/tsfile/write/record/Tablet.java | 24 +++-
4 files changed, 199 insertions(+), 12 deletions(-)
diff --git a/session/src/main/java/org/apache/iotdb/session/SessionUtils.java b/session/src/main/java/org/apache/iotdb/session/SessionUtils.java
index cd35017..c0aa710 100644
--- a/session/src/main/java/org/apache/iotdb/session/SessionUtils.java
+++ b/session/src/main/java/org/apache/iotdb/session/SessionUtils.java
@@ -22,6 +22,7 @@ import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.utils.Binary;
import org.apache.iotdb.tsfile.utils.BytesUtils;
+import org.apache.iotdb.tsfile.write.record.BitMap;
import org.apache.iotdb.tsfile.write.record.Tablet;
import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
@@ -58,42 +59,74 @@ public class SessionUtils {
private static void getValueBufferOfDataType(
TSDataType dataType, Tablet tablet, int i, ByteBuffer valueBuffer) {
+ BitMap curBitMap = tablet.bitMaps[i];
+ long[] curLongs = curBitMap.getLongArray();
+ for (int j = 0; j < curLongs.length; j++) {
+ valueBuffer.putLong(curLongs[j]);
+ }
+
switch (dataType) {
case INT32:
int[] intValues = (int[]) tablet.values[i];
for (int index = 0; index < tablet.rowSize; index++) {
- valueBuffer.putInt(intValues[index]);
+ if (curBitMap.get(index)) {
+ valueBuffer.putInt(intValues[index]);
+ } else {
+ valueBuffer.putInt(-1);
+ }
}
break;
case INT64:
long[] longValues = (long[]) tablet.values[i];
for (int index = 0; index < tablet.rowSize; index++) {
- valueBuffer.putLong(longValues[index]);
+ if (curBitMap.get(index)) {
+ valueBuffer.putLong(longValues[index]);
+ } else {
+ valueBuffer.putLong(-1);
+ }
}
break;
case FLOAT:
float[] floatValues = (float[]) tablet.values[i];
for (int index = 0; index < tablet.rowSize; index++) {
- valueBuffer.putFloat(floatValues[index]);
+ if (curBitMap.get(index)) {
+ valueBuffer.putFloat(floatValues[index]);
+ } else {
+ valueBuffer.putFloat(-1);
+ }
}
break;
case DOUBLE:
double[] doubleValues = (double[]) tablet.values[i];
for (int index = 0; index < tablet.rowSize; index++) {
- valueBuffer.putDouble(doubleValues[index]);
+ if (curBitMap.get(index)) {
+ valueBuffer.putDouble(doubleValues[index]);
+ } else {
+ valueBuffer.putDouble(-1);
+ }
}
break;
case BOOLEAN:
boolean[] boolValues = (boolean[]) tablet.values[i];
for (int index = 0; index < tablet.rowSize; index++) {
- valueBuffer.put(BytesUtils.boolToByte(boolValues[index]));
+ if (curBitMap.get(index)) {
+ valueBuffer.put(BytesUtils.boolToByte(boolValues[index]));
+ } else {
+ valueBuffer.put(BytesUtils.boolToByte(false));
+ }
}
break;
case TEXT:
Binary[] binaryValues = (Binary[]) tablet.values[i];
for (int index = 0; index < tablet.rowSize; index++) {
- valueBuffer.putInt(binaryValues[index].getLength());
- valueBuffer.put(binaryValues[index].getValues());
+ if (curBitMap.get(index)) {
+ valueBuffer.putInt(binaryValues[index].getLength());
+ valueBuffer.put(binaryValues[index].getValues());
+ } else {
+ Binary emptyStr = new Binary(".");
+ valueBuffer.putInt(emptyStr.getLength());
+ valueBuffer.put(emptyStr.getValues());
+ }
}
break;
default:
diff --git a/session/src/test/java/org/apache/iotdb/session/IoTDBSessionSimpleIT.java b/session/src/test/java/org/apache/iotdb/session/IoTDBSessionSimpleIT.java
index 5db9303..b05707d 100644
--- a/session/src/test/java/org/apache/iotdb/session/IoTDBSessionSimpleIT.java
+++ b/session/src/test/java/org/apache/iotdb/session/IoTDBSessionSimpleIT.java
@@ -122,9 +122,15 @@ public class IoTDBSessionSimpleIT {
for (long row = 0; row < 15; row++) {
int rowIndex = tablet.rowSize++;
tablet.addTimestamp(rowIndex, timestamp);
- tablet.addValue("s1", rowIndex, 1L);
- tablet.addValue("s2", rowIndex, 1D);
- tablet.addValue("s3", rowIndex, new Binary("1"));
+ if (row % 5 == 0) {
+ tablet.addValue("s1", rowIndex, null);
+ tablet.addValue("s2", rowIndex, null);
+ tablet.addValue("s3", rowIndex, null);
+ } else {
+ tablet.addValue("s1", rowIndex, 1L);
+ tablet.addValue("s2", rowIndex, 1D);
+ tablet.addValue("s3", rowIndex, new Binary("1"));
+ }
if (tablet.rowSize == tablet.getMaxRowNumber()) {
session.insertTablet(tablet, true);
tablet.reset();
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/record/BitMap.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/record/BitMap.java
new file mode 100644
index 0000000..4dcfe1a
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/record/BitMap.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.tsfile.write.record;
+
+import java.util.Arrays;
+
+public class BitMap {
+ private static final byte[] BIT_UTIL = new byte[] {1, 2, 4, 8, 16, 32, 64, -128};
+ private static final byte[] UNMARK_BIT_UTIL =
+ new byte[] {
+ (byte) 0XFE,
+ (byte) 0XFD,
+ (byte) 0XFA,
+ (byte) 0XF7,
+ (byte) 0XEF,
+ (byte) 0XDF,
+ (byte) 0XAF,
+ (byte) 0X7F
+ };
+
+ private byte[] bits;
+ private int size;
+ private int length;
+
+ public BitMap(int size) {
+ this.size = size;
+ if (size % Byte.SIZE == 0) {
+ this.length = size / Byte.SIZE;
+ } else {
+ this.length = size / Byte.SIZE + 1;
+ }
+ this.length = size / Byte.SIZE + 1;
+ bits = new byte[this.length];
+ Arrays.fill(bits, (byte) 0);
+ }
+
+ public byte[] getByteArray() {
+ return this.bits;
+ }
+
+ public long[] getLongArray() {
+ int newLength;
+ if (this.length % (Long.SIZE / Byte.SIZE) == 0) {
+ newLength = this.length / (Long.SIZE / Byte.SIZE);
+ } else {
+ newLength = this.length / (Long.SIZE / Byte.SIZE) + 1;
+ }
+
+ long[] retLong = new long[newLength];
+ for (int i = 0; i < newLength; i++) {
+ long curLong = 0;
+ for (int j = 0; j < Byte.SIZE; j++) {
+ curLong |= ((long) (this.bits[i + j]) << j);
+ }
+ retLong[i] = curLong;
+ }
+ return retLong;
+ }
+
+ /** returns the value of the bit with the specified index. */
+ public boolean get(int position) {
+ return (bits[position / Byte.SIZE] & BIT_UTIL[position % Byte.SIZE]) > 0;
+ }
+
+ /** mark as 1 at the given bit position */
+ public void mark(int position) {
+ bits[position / Byte.SIZE] |= BIT_UTIL[position % Byte.SIZE];
+ }
+
+ /** mark as 0 at all position or the given bit position */
+ public void unmark() {
+ for (int i = 0; i < this.length; i++) {
+ bits[i] = (byte) 0;
+ }
+ }
+
+ public void unmark(int position) {
+ bits[position / Byte.SIZE] &= UNMARK_BIT_UTIL[position % Byte.SIZE];
+ }
+
+ /** whether all bits are one */
+ public boolean isAllZero() {
+ int j;
+ for (j = 0; j < size / Byte.SIZE; j++) {
+ if (bits[j] != (byte) 0) {
+ return false;
+ }
+ }
+ for (j = 0; j < size % Byte.SIZE; j++) {
+ if ((bits[size / Byte.SIZE] & BIT_UTIL[j]) != BIT_UTIL[j]) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /** whether all bits are zero */
+ public boolean isAllOne() {
+ int j;
+ for (j = 0; j < size / Byte.SIZE; j++) {
+ if (bits[j] != (byte) 0XFF) {
+ return false;
+ }
+ }
+ for (j = 0; j < size % Byte.SIZE; j++) {
+ if ((bits[size / Byte.SIZE] & BIT_UTIL[j]) == 0) {
+ return false;
+ }
+ }
+ return true;
+ }
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/record/Tablet.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/record/Tablet.java
index b5bd095..3ae4b2d 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/record/Tablet.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/record/Tablet.java
@@ -58,6 +58,8 @@ public class Tablet {
public long[] timestamps;
/** each object is a primitive type array, which represents values of one measurement */
public Object[] values;
+ /** each bitset represents the existence of each value in the current column */
+ public BitMap[] bitMaps;
/** the number of rows to include in this tablet */
public int rowSize;
/** the maximum number of rows for this tablet */
@@ -124,6 +126,14 @@ public class Tablet {
private void addValueOfDataType(
TSDataType dataType, int rowIndex, int indexOfValue, Object value) {
+
+ // set bitset
+ if (value == null) {
+ bitMaps[indexOfValue].unmark(rowIndex);
+ return;
+ }
+ bitMaps[indexOfValue].mark(rowIndex);
+
switch (dataType) {
case TEXT:
{
@@ -191,14 +201,17 @@ public class Tablet {
valueColumnsSize++;
}
}
+
+ // value column and bitset column
values = new Object[valueColumnsSize];
+ bitMaps = new BitMap[valueColumnsSize];
int columnIndex = 0;
- // create value columns
for (IMeasurementSchema schema : schemas) {
TSDataType dataType = schema.getType();
if (dataType.equals(TSDataType.VECTOR)) {
columnIndex = buildVectorColumns((VectorMeasurementSchema) schema, columnIndex);
} else {
+ bitMaps[columnIndex] = new BitMap(maxRowNumber);
values[columnIndex] = createValueColumnOfDataType(dataType);
columnIndex++;
}
@@ -208,6 +221,7 @@ public class Tablet {
private int buildVectorColumns(VectorMeasurementSchema schema, int idx) {
for (int i = 0; i < schema.getValueMeasurementIdList().size(); i++) {
TSDataType dataType = schema.getValueTSDataTypeList().get(i);
+ bitMaps[idx] = new BitMap(maxRowNumber);
values[idx] = createValueColumnOfDataType(dataType);
idx++;
}
@@ -280,8 +294,14 @@ public class Tablet {
case TEXT:
valueOccupation += rowSize * 4;
Binary[] binaries = (Binary[]) values[i];
+ BitMap curBitMap = bitMaps[i];
for (int rowIndex = 0; rowIndex < rowSize; rowIndex++) {
- valueOccupation += binaries[rowIndex].getLength();
+ if (curBitMap.get(rowIndex)) {
+ valueOccupation += binaries[rowIndex].getLength();
+ } else {
+ Binary emptyStr = new Binary(".");
+ valueOccupation += emptyStr.getLength();
+ }
}
break;
default: