You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2023/09/06 11:55:10 UTC
[iotdb] branch ty-opt created (now 42d83cb9e8f)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a change to branch ty-opt
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at 42d83cb9e8f try opt
This branch includes the following new commits:
new 42d83cb9e8f try opt
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: try opt
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch ty-opt
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 42d83cb9e8f9d9b3999e5a7e1c59724c890245ea
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Wed Sep 6 19:54:45 2023 +0800
try opt
---
.../tsfile/read/reader/page/AlignedPageReader.java | 169 +++++++++++++--------
.../tsfile/read/reader/page/ValuePageReader.java | 38 +++++
.../read/reader/series/PaginationController.java | 8 +
3 files changed, 150 insertions(+), 65 deletions(-)
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java
index 67be092ada5..e467d86bb22 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java
@@ -167,93 +167,132 @@ public class AlignedPageReader implements IPageReader, IAlignedPageReader {
@Override
public TsBlock getAllSatisfiedData() throws IOException {
- builder.reset();
if (!pageSatisfy()) {
return builder.build();
}
long[] timeBatch = timePageReader.getNextTimeBatch();
- // if all the sub sensors' value are null in current row, just discard it
- // if !filter.satisfy, discard this row
- boolean[] keepCurrentRow = new boolean[timeBatch.length];
- if (filter == null) {
- Arrays.fill(keepCurrentRow, true);
+ if (queryAllSensors && !isModified) {
+ // skip all the page
+ if (paginationController.hasCurOffset(timeBatch.length)) {
+ paginationController.consumeOffset(timeBatch.length);
+ } else {
+ // consume the remaining offset
+ if (paginationController.hasCurOffset()) {
+ paginationController.consumeOffset(paginationController.getCurOffset());
+ }
+ int readStartIndex =
+ paginationController.hasCurOffset() ? (int) paginationController.getCurOffset() : 0;
+ // not included
+ int readEndIndex =
+ (paginationController.hasCurLimit()
+ && (paginationController.getCurLimit() < timeBatch.length - readStartIndex + 1))
+ ? readStartIndex + (int) paginationController.getCurLimit()
+ : timeBatch.length;
+ if (paginationController.hasCurLimit()) {
+ paginationController.consumeLimit(readEndIndex - readStartIndex);
+ }
+ // construct time column
+ for (int i = readStartIndex; i < readEndIndex; i++) {
+ builder.getTimeColumnBuilder().writeLong(timeBatch[i]);
+ builder.declarePosition();
+ }
+ // construct value columns
+ for (int i = 0; i < valueCount; i++) {
+ ValuePageReader pageReader = valuePageReaderList.get(i);
+ if (pageReader != null) {
+ pageReader.writeColumnBuilderWithNextBatch(
+ readStartIndex, readEndIndex, builder.getColumnBuilder(i));
+ } else {
+ for (int j = readStartIndex; j < readEndIndex; j++) {
+ builder.getColumnBuilder(i).appendNull();
+ }
+ }
+ }
+ }
} else {
- for (int i = 0, n = timeBatch.length; i < n; i++) {
- keepCurrentRow[i] = filter.satisfy(timeBatch[i], null);
+ // if all the sub sensors' value are null in current row, just discard it
+ // if !filter.satisfy, discard this row
+ boolean[] keepCurrentRow = new boolean[timeBatch.length];
+ if (filter == null) {
+ Arrays.fill(keepCurrentRow, true);
+ } else {
+ for (int i = 0, n = timeBatch.length; i < n; i++) {
+ keepCurrentRow[i] = filter.satisfy(timeBatch[i], null);
+ }
}
- }
- boolean[][] isDeleted = null;
- if (valueCount != 0) {
- // using bitMap in valuePageReaders to indicate whether columns of current row are all null.
- byte[] bitmask = new byte[(timeBatch.length - 1) / 8 + 1];
- Arrays.fill(bitmask, (byte) 0x00);
- isDeleted = new boolean[valueCount][timeBatch.length];
- for (int columnIndex = 0; columnIndex < valueCount; columnIndex++) {
- ValuePageReader pageReader = valuePageReaderList.get(columnIndex);
- if (pageReader != null) {
- byte[] bitmap = pageReader.getBitmap();
- pageReader.fillIsDeleted(timeBatch, isDeleted[columnIndex]);
+ boolean[][] isDeleted = null;
+ if (valueCount != 0) {
+ // using bitMap in valuePageReaders to indicate whether columns of current row are all null.
+ byte[] bitmask = new byte[(timeBatch.length - 1) / 8 + 1];
+ Arrays.fill(bitmask, (byte) 0x00);
+ isDeleted = new boolean[valueCount][timeBatch.length];
+ for (int columnIndex = 0; columnIndex < valueCount; columnIndex++) {
+ ValuePageReader pageReader = valuePageReaderList.get(columnIndex);
+ if (pageReader != null) {
+ byte[] bitmap = pageReader.getBitmap();
+ pageReader.fillIsDeleted(timeBatch, isDeleted[columnIndex]);
- for (int i = 0, n = isDeleted[columnIndex].length; i < n; i++) {
- if (isDeleted[columnIndex][i]) {
- int shift = i % 8;
- bitmap[i / 8] = (byte) (bitmap[i / 8] & (~(MASK >>> shift)));
+ for (int i = 0, n = isDeleted[columnIndex].length; i < n; i++) {
+ if (isDeleted[columnIndex][i]) {
+ int shift = i % 8;
+ bitmap[i / 8] = (byte) (bitmap[i / 8] & (~(MASK >>> shift)));
+ }
+ }
+ for (int i = 0, n = bitmask.length; i < n; i++) {
+ bitmask[i] = (byte) (bitmap[i] | bitmask[i]);
}
- }
- for (int i = 0, n = bitmask.length; i < n; i++) {
- bitmask[i] = (byte) (bitmap[i] | bitmask[i]);
}
}
- }
- for (int i = 0, n = bitmask.length; i < n; i++) {
- if (bitmask[i] == (byte) 0xFF) {
- // 8 rows are not all null, do nothing
- } else if (bitmask[i] == (byte) 0x00) {
- for (int j = 0; j < 8 && (i * 8 + j < keepCurrentRow.length); j++) {
- keepCurrentRow[i * 8 + j] = false;
- }
- } else {
- for (int j = 0; j < 8 && (i * 8 + j < keepCurrentRow.length); j++) {
- if (((bitmask[i] & 0xFF) & (MASK >>> j)) == 0) {
+ for (int i = 0, n = bitmask.length; i < n; i++) {
+ if (bitmask[i] == (byte) 0xFF) {
+ // 8 rows are not all null, do nothing
+ } else if (bitmask[i] == (byte) 0x00) {
+ for (int j = 0; j < 8 && (i * 8 + j < keepCurrentRow.length); j++) {
keepCurrentRow[i * 8 + j] = false;
}
+ } else {
+ for (int j = 0; j < 8 && (i * 8 + j < keepCurrentRow.length); j++) {
+ if (((bitmask[i] & 0xFF) & (MASK >>> j)) == 0) {
+ keepCurrentRow[i * 8 + j] = false;
+ }
+ }
}
}
}
- }
- // construct time column
- int readEndIndex = timeBatch.length;
- for (int i = 0; i < timeBatch.length; i++) {
- if (keepCurrentRow[i]) {
- if (paginationController.hasCurOffset()) {
- paginationController.consumeOffset();
- keepCurrentRow[i] = false;
- } else if (paginationController.hasCurLimit()) {
- builder.getTimeColumnBuilder().writeLong(timeBatch[i]);
- builder.declarePosition();
- paginationController.consumeLimit();
- } else {
- readEndIndex = i;
- break;
+ // construct time column
+ int readEndIndex = timeBatch.length;
+ for (int i = 0; i < timeBatch.length; i++) {
+ if (keepCurrentRow[i]) {
+ if (paginationController.hasCurOffset()) {
+ paginationController.consumeOffset();
+ keepCurrentRow[i] = false;
+ } else if (paginationController.hasCurLimit()) {
+ builder.getTimeColumnBuilder().writeLong(timeBatch[i]);
+ builder.declarePosition();
+ paginationController.consumeLimit();
+ } else {
+ readEndIndex = i;
+ break;
+ }
}
}
- }
- // construct value columns
- for (int i = 0; i < valueCount; i++) {
- ValuePageReader pageReader = valuePageReaderList.get(i);
- if (pageReader != null) {
- pageReader.writeColumnBuilderWithNextBatch(
- readEndIndex, builder.getColumnBuilder(i), keepCurrentRow, isDeleted[i]);
- } else {
- for (int j = 0; j < readEndIndex; j++) {
- if (keepCurrentRow[j]) {
- builder.getColumnBuilder(i).appendNull();
+ // construct value columns
+ for (int i = 0; i < valueCount; i++) {
+ ValuePageReader pageReader = valuePageReaderList.get(i);
+ if (pageReader != null) {
+ pageReader.writeColumnBuilderWithNextBatch(
+ readEndIndex, builder.getColumnBuilder(i), keepCurrentRow, isDeleted[i]);
+ } else {
+ for (int j = 0; j < readEndIndex; j++) {
+ if (keepCurrentRow[j]) {
+ builder.getColumnBuilder(i).appendNull();
+ }
}
}
}
@@ -316,6 +355,6 @@ public class AlignedPageReader implements IPageReader, IAlignedPageReader {
@Override
public void initTsBlockBuilder(List<TSDataType> dataTypes) {
- builder = new TsBlockBuilder(dataTypes);
+ builder = new TsBlockBuilder((int) timePageReader.getStatistics().getCount(), dataTypes);
}
}
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/ValuePageReader.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/ValuePageReader.java
index 79e923c97f0..6018afd374e 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/ValuePageReader.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/ValuePageReader.java
@@ -329,6 +329,44 @@ public class ValuePageReader {
}
}
+ public void writeColumnBuilderWithNextBatch(
+ int readStartIndex, int readEndIndex, ColumnBuilder columnBuilder) {
+ if (valueBuffer == null) {
+ for (int i = readStartIndex; i < readEndIndex; i++) {
+ columnBuilder.appendNull();
+ }
+ return;
+ }
+ for (int i = readStartIndex; i < readEndIndex; i++) {
+ if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) {
+ columnBuilder.appendNull();
+ continue;
+ }
+ switch (dataType) {
+ case BOOLEAN:
+ columnBuilder.writeBoolean(valueDecoder.readBoolean(valueBuffer));
+ break;
+ case INT32:
+ columnBuilder.writeInt(valueDecoder.readInt(valueBuffer));
+ break;
+ case INT64:
+ columnBuilder.writeLong(valueDecoder.readLong(valueBuffer));
+ break;
+ case FLOAT:
+ columnBuilder.writeFloat(valueDecoder.readFloat(valueBuffer));
+ break;
+ case DOUBLE:
+ columnBuilder.writeDouble(valueDecoder.readDouble(valueBuffer));
+ break;
+ case TEXT:
+ columnBuilder.writeBinary(valueDecoder.readBinary(valueBuffer));
+ break;
+ default:
+ throw new UnSupportedDataTypeException(String.valueOf(dataType));
+ }
+ }
+ }
+
public Statistics getStatistics() {
return pageHeader.getStatistics();
}
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/PaginationController.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/PaginationController.java
index 108fa0d0a5b..aa0f7971e47 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/PaginationController.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/PaginationController.java
@@ -60,6 +60,14 @@ public class PaginationController {
curOffset--;
}
+ public long getCurOffset() {
+ return curOffset;
+ }
+
+ public long getCurLimit() {
+ return curLimit;
+ }
+
public void consumeLimit() {
if (hasLimit) {
curLimit--;