You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2023/09/06 11:55:11 UTC
[iotdb] 01/01: try opt
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch ty-opt
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 42d83cb9e8f9d9b3999e5a7e1c59724c890245ea
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Wed Sep 6 19:54:45 2023 +0800
try opt
---
.../tsfile/read/reader/page/AlignedPageReader.java | 169 +++++++++++++--------
.../tsfile/read/reader/page/ValuePageReader.java | 38 +++++
.../read/reader/series/PaginationController.java | 8 +
3 files changed, 150 insertions(+), 65 deletions(-)
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java
index 67be092ada5..e467d86bb22 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java
@@ -167,93 +167,132 @@ public class AlignedPageReader implements IPageReader, IAlignedPageReader {
@Override
public TsBlock getAllSatisfiedData() throws IOException {
- builder.reset();
if (!pageSatisfy()) {
return builder.build();
}
long[] timeBatch = timePageReader.getNextTimeBatch();
- // if all the sub sensors' value are null in current row, just discard it
- // if !filter.satisfy, discard this row
- boolean[] keepCurrentRow = new boolean[timeBatch.length];
- if (filter == null) {
- Arrays.fill(keepCurrentRow, true);
+ if (queryAllSensors && !isModified) {
+ // skip all the page
+ if (paginationController.hasCurOffset(timeBatch.length)) {
+ paginationController.consumeOffset(timeBatch.length);
+ } else {
+ // consume the remaining offset
+ if (paginationController.hasCurOffset()) {
+ paginationController.consumeOffset(paginationController.getCurOffset());
+ }
+ int readStartIndex =
+ paginationController.hasCurOffset() ? (int) paginationController.getCurOffset() : 0;
+ // not included
+ int readEndIndex =
+ (paginationController.hasCurLimit()
+ && (paginationController.getCurLimit() < timeBatch.length - readStartIndex + 1))
+ ? readStartIndex + (int) paginationController.getCurLimit()
+ : timeBatch.length;
+ if (paginationController.hasCurLimit()) {
+ paginationController.consumeLimit(readEndIndex - readStartIndex);
+ }
+ // construct time column
+ for (int i = readStartIndex; i < readEndIndex; i++) {
+ builder.getTimeColumnBuilder().writeLong(timeBatch[i]);
+ builder.declarePosition();
+ }
+ // construct value columns
+ for (int i = 0; i < valueCount; i++) {
+ ValuePageReader pageReader = valuePageReaderList.get(i);
+ if (pageReader != null) {
+ pageReader.writeColumnBuilderWithNextBatch(
+ readStartIndex, readEndIndex, builder.getColumnBuilder(i));
+ } else {
+ for (int j = readStartIndex; j < readEndIndex; j++) {
+ builder.getColumnBuilder(i).appendNull();
+ }
+ }
+ }
+ }
} else {
- for (int i = 0, n = timeBatch.length; i < n; i++) {
- keepCurrentRow[i] = filter.satisfy(timeBatch[i], null);
+ // if all the sub sensors' value are null in current row, just discard it
+ // if !filter.satisfy, discard this row
+ boolean[] keepCurrentRow = new boolean[timeBatch.length];
+ if (filter == null) {
+ Arrays.fill(keepCurrentRow, true);
+ } else {
+ for (int i = 0, n = timeBatch.length; i < n; i++) {
+ keepCurrentRow[i] = filter.satisfy(timeBatch[i], null);
+ }
}
- }
- boolean[][] isDeleted = null;
- if (valueCount != 0) {
- // using bitMap in valuePageReaders to indicate whether columns of current row are all null.
- byte[] bitmask = new byte[(timeBatch.length - 1) / 8 + 1];
- Arrays.fill(bitmask, (byte) 0x00);
- isDeleted = new boolean[valueCount][timeBatch.length];
- for (int columnIndex = 0; columnIndex < valueCount; columnIndex++) {
- ValuePageReader pageReader = valuePageReaderList.get(columnIndex);
- if (pageReader != null) {
- byte[] bitmap = pageReader.getBitmap();
- pageReader.fillIsDeleted(timeBatch, isDeleted[columnIndex]);
+ boolean[][] isDeleted = null;
+ if (valueCount != 0) {
+ // using bitMap in valuePageReaders to indicate whether columns of current row are all null.
+ byte[] bitmask = new byte[(timeBatch.length - 1) / 8 + 1];
+ Arrays.fill(bitmask, (byte) 0x00);
+ isDeleted = new boolean[valueCount][timeBatch.length];
+ for (int columnIndex = 0; columnIndex < valueCount; columnIndex++) {
+ ValuePageReader pageReader = valuePageReaderList.get(columnIndex);
+ if (pageReader != null) {
+ byte[] bitmap = pageReader.getBitmap();
+ pageReader.fillIsDeleted(timeBatch, isDeleted[columnIndex]);
- for (int i = 0, n = isDeleted[columnIndex].length; i < n; i++) {
- if (isDeleted[columnIndex][i]) {
- int shift = i % 8;
- bitmap[i / 8] = (byte) (bitmap[i / 8] & (~(MASK >>> shift)));
+ for (int i = 0, n = isDeleted[columnIndex].length; i < n; i++) {
+ if (isDeleted[columnIndex][i]) {
+ int shift = i % 8;
+ bitmap[i / 8] = (byte) (bitmap[i / 8] & (~(MASK >>> shift)));
+ }
+ }
+ for (int i = 0, n = bitmask.length; i < n; i++) {
+ bitmask[i] = (byte) (bitmap[i] | bitmask[i]);
}
- }
- for (int i = 0, n = bitmask.length; i < n; i++) {
- bitmask[i] = (byte) (bitmap[i] | bitmask[i]);
}
}
- }
- for (int i = 0, n = bitmask.length; i < n; i++) {
- if (bitmask[i] == (byte) 0xFF) {
- // 8 rows are not all null, do nothing
- } else if (bitmask[i] == (byte) 0x00) {
- for (int j = 0; j < 8 && (i * 8 + j < keepCurrentRow.length); j++) {
- keepCurrentRow[i * 8 + j] = false;
- }
- } else {
- for (int j = 0; j < 8 && (i * 8 + j < keepCurrentRow.length); j++) {
- if (((bitmask[i] & 0xFF) & (MASK >>> j)) == 0) {
+ for (int i = 0, n = bitmask.length; i < n; i++) {
+ if (bitmask[i] == (byte) 0xFF) {
+ // 8 rows are not all null, do nothing
+ } else if (bitmask[i] == (byte) 0x00) {
+ for (int j = 0; j < 8 && (i * 8 + j < keepCurrentRow.length); j++) {
keepCurrentRow[i * 8 + j] = false;
}
+ } else {
+ for (int j = 0; j < 8 && (i * 8 + j < keepCurrentRow.length); j++) {
+ if (((bitmask[i] & 0xFF) & (MASK >>> j)) == 0) {
+ keepCurrentRow[i * 8 + j] = false;
+ }
+ }
}
}
}
- }
- // construct time column
- int readEndIndex = timeBatch.length;
- for (int i = 0; i < timeBatch.length; i++) {
- if (keepCurrentRow[i]) {
- if (paginationController.hasCurOffset()) {
- paginationController.consumeOffset();
- keepCurrentRow[i] = false;
- } else if (paginationController.hasCurLimit()) {
- builder.getTimeColumnBuilder().writeLong(timeBatch[i]);
- builder.declarePosition();
- paginationController.consumeLimit();
- } else {
- readEndIndex = i;
- break;
+ // construct time column
+ int readEndIndex = timeBatch.length;
+ for (int i = 0; i < timeBatch.length; i++) {
+ if (keepCurrentRow[i]) {
+ if (paginationController.hasCurOffset()) {
+ paginationController.consumeOffset();
+ keepCurrentRow[i] = false;
+ } else if (paginationController.hasCurLimit()) {
+ builder.getTimeColumnBuilder().writeLong(timeBatch[i]);
+ builder.declarePosition();
+ paginationController.consumeLimit();
+ } else {
+ readEndIndex = i;
+ break;
+ }
}
}
- }
- // construct value columns
- for (int i = 0; i < valueCount; i++) {
- ValuePageReader pageReader = valuePageReaderList.get(i);
- if (pageReader != null) {
- pageReader.writeColumnBuilderWithNextBatch(
- readEndIndex, builder.getColumnBuilder(i), keepCurrentRow, isDeleted[i]);
- } else {
- for (int j = 0; j < readEndIndex; j++) {
- if (keepCurrentRow[j]) {
- builder.getColumnBuilder(i).appendNull();
+ // construct value columns
+ for (int i = 0; i < valueCount; i++) {
+ ValuePageReader pageReader = valuePageReaderList.get(i);
+ if (pageReader != null) {
+ pageReader.writeColumnBuilderWithNextBatch(
+ readEndIndex, builder.getColumnBuilder(i), keepCurrentRow, isDeleted[i]);
+ } else {
+ for (int j = 0; j < readEndIndex; j++) {
+ if (keepCurrentRow[j]) {
+ builder.getColumnBuilder(i).appendNull();
+ }
}
}
}
@@ -316,6 +355,6 @@ public class AlignedPageReader implements IPageReader, IAlignedPageReader {
@Override
public void initTsBlockBuilder(List<TSDataType> dataTypes) {
- builder = new TsBlockBuilder(dataTypes);
+ builder = new TsBlockBuilder((int) timePageReader.getStatistics().getCount(), dataTypes);
}
}
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/ValuePageReader.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/ValuePageReader.java
index 79e923c97f0..6018afd374e 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/ValuePageReader.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/ValuePageReader.java
@@ -329,6 +329,44 @@ public class ValuePageReader {
}
}
+ public void writeColumnBuilderWithNextBatch(
+ int readStartIndex, int readEndIndex, ColumnBuilder columnBuilder) {
+ if (valueBuffer == null) {
+ for (int i = readStartIndex; i < readEndIndex; i++) {
+ columnBuilder.appendNull();
+ }
+ return;
+ }
+ for (int i = readStartIndex; i < readEndIndex; i++) {
+ if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) {
+ columnBuilder.appendNull();
+ continue;
+ }
+ switch (dataType) {
+ case BOOLEAN:
+ columnBuilder.writeBoolean(valueDecoder.readBoolean(valueBuffer));
+ break;
+ case INT32:
+ columnBuilder.writeInt(valueDecoder.readInt(valueBuffer));
+ break;
+ case INT64:
+ columnBuilder.writeLong(valueDecoder.readLong(valueBuffer));
+ break;
+ case FLOAT:
+ columnBuilder.writeFloat(valueDecoder.readFloat(valueBuffer));
+ break;
+ case DOUBLE:
+ columnBuilder.writeDouble(valueDecoder.readDouble(valueBuffer));
+ break;
+ case TEXT:
+ columnBuilder.writeBinary(valueDecoder.readBinary(valueBuffer));
+ break;
+ default:
+ throw new UnSupportedDataTypeException(String.valueOf(dataType));
+ }
+ }
+ }
+
public Statistics getStatistics() {
return pageHeader.getStatistics();
}
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/PaginationController.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/PaginationController.java
index 108fa0d0a5b..aa0f7971e47 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/PaginationController.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/PaginationController.java
@@ -60,6 +60,14 @@ public class PaginationController {
curOffset--;
}
+ public long getCurOffset() {
+ return curOffset;
+ }
+
+ public long getCurLimit() {
+ return curLimit;
+ }
+
public void consumeLimit() {
if (hasLimit) {
curLimit--;