You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2024/03/23 08:24:07 UTC
(iotdb) 03/03: Optimize SeriesScanUtil by memorizing the order time and satisfied info for each Seq and Unseq Resource
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch ty/OptSeriesScan
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 4a3f7ef0bd96b9afcab33e5b3c2ecb4b04f9cc27
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Sat Mar 23 16:22:03 2024 +0800
Optimize SeriesScanUtil by memorizing the order time and satisfied info for each Seq and Unseq Resource
---
.../execution/operator/source/SeriesScanUtil.java | 90 +++++++++-----------
.../buffer/TimeSeriesMetadataCache.java | 6 +-
.../dataregion/read/QueryDataSource.java | 95 ++++++++++++++++++++--
.../iotdb/tsfile/file/metadata/PlainDeviceID.java | 2 +-
4 files changed, 129 insertions(+), 64 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java
index d86b01dd6d0..f0f48e54542 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java
@@ -33,6 +33,7 @@ import org.apache.iotdb.db.storageengine.dataregion.read.reader.common.PriorityM
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException;
import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
+import org.apache.iotdb.tsfile.file.metadata.IDeviceID;
import org.apache.iotdb.tsfile.file.metadata.IMetadata;
import org.apache.iotdb.tsfile.file.metadata.ITimeSeriesMetadata;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -68,6 +69,8 @@ public class SeriesScanUtil {
// The path of the target series which will be scanned.
protected final PartialPath seriesPath;
+
+ private final IDeviceID deviceID;
protected boolean isAligned = false;
private final TSDataType dataType;
@@ -113,6 +116,7 @@ public class SeriesScanUtil {
SeriesScanOptions scanOptions,
FragmentInstanceContext context) {
this.seriesPath = seriesPath;
+ this.deviceID = seriesPath.getIDeviceID();
this.dataType = seriesPath.getSeriesType();
this.scanOptions = scanOptions;
@@ -1116,12 +1120,10 @@ public class SeriesScanUtil {
private void unpackAllOverlappedTsFilesToTimeSeriesMetadata(long endpointTime)
throws IOException {
- while (orderUtils.hasNextUnseqResource()
- && orderUtils.isOverlapped(endpointTime, orderUtils.getNextUnseqFileResource(false))) {
+ while (orderUtils.hasNextUnseqResource() && orderUtils.isCurUnSeqOverlappedWith(endpointTime)) {
unpackUnseqTsFileResource();
}
- while (orderUtils.hasNextSeqResource()
- && orderUtils.isOverlapped(endpointTime, orderUtils.getNextSeqFileResource(false))) {
+ while (orderUtils.hasNextSeqResource() && orderUtils.isCurSeqOverlappedWith(endpointTime)) {
unpackSeqTsFileResource();
}
}
@@ -1256,15 +1258,15 @@ public class SeriesScanUtil {
long getOrderTime(Statistics<? extends Object> statistics);
- long getOrderTime(TsFileResource fileResource);
-
long getOverlapCheckTime(Statistics<? extends Object> range);
boolean isOverlapped(Statistics<? extends Object> left, Statistics<? extends Object> right);
boolean isOverlapped(long time, Statistics<? extends Object> right);
- boolean isOverlapped(long time, TsFileResource right);
+ boolean isCurSeqOverlappedWith(long time);
+
+ boolean isCurUnSeqOverlappedWith(long time);
<T> Comparator<T> comparingLong(ToLongFunction<? super T> keyExtractor);
@@ -1300,12 +1302,6 @@ public class SeriesScanUtil {
return statistics.getEndTime();
}
- @SuppressWarnings("squid:S3740")
- @Override
- public long getOrderTime(TsFileResource fileResource) {
- return fileResource.getEndTime(seriesPath.getIDeviceID());
- }
-
@SuppressWarnings("squid:S3740")
@Override
public long getOverlapCheckTime(Statistics range) {
@@ -1325,8 +1321,13 @@ public class SeriesScanUtil {
}
@Override
- public boolean isOverlapped(long time, TsFileResource right) {
- return time <= right.getEndTime(seriesPath.getIDeviceID());
+ public boolean isCurSeqOverlappedWith(long time) {
+ return time <= dataSource.getCurrentSeqOrderTime(curSeqFileIndex);
+ }
+
+ @Override
+ public boolean isCurUnSeqOverlappedWith(long time) {
+ return time <= dataSource.getCurrentUnSeqOrderTime(curUnseqFileIndex);
}
@Override
@@ -1365,32 +1366,26 @@ public class SeriesScanUtil {
@Override
public boolean hasNextSeqResource() {
- while (dataSource.hasNextSeqResource(curSeqFileIndex, getAscending())) {
- TsFileResource tsFileResource = dataSource.getSeqResourceByIndex(curSeqFileIndex);
- if (tsFileResource != null
- && (dataSource.isSingleDevice()
- || tsFileResource.isSatisfied(
- seriesPath.getIDeviceID(), scanOptions.getGlobalTimeFilter(), true, false))) {
+ while (dataSource.hasNextSeqResource(curSeqFileIndex, false, deviceID)) {
+ if (dataSource.isSeqSatisfied(
+ deviceID, curSeqFileIndex, scanOptions.getGlobalTimeFilter(), false)) {
break;
}
curSeqFileIndex--;
}
- return dataSource.hasNextSeqResource(curSeqFileIndex, getAscending());
+ return dataSource.hasNextSeqResource(curSeqFileIndex, false, deviceID);
}
@Override
public boolean hasNextUnseqResource() {
- while (dataSource.hasNextUnseqResource(curUnseqFileIndex)) {
- TsFileResource tsFileResource = dataSource.getUnseqResourceByIndex(curUnseqFileIndex);
- if (tsFileResource != null
- && (dataSource.isSingleDevice()
- || tsFileResource.isSatisfied(
- seriesPath.getIDeviceID(), scanOptions.getGlobalTimeFilter(), false, false))) {
+ while (dataSource.hasNextUnseqResource(curUnseqFileIndex, false, deviceID)) {
+ if (dataSource.isUnSeqSatisfied(
+ deviceID, curUnseqFileIndex, scanOptions.getGlobalTimeFilter(), false)) {
break;
}
curUnseqFileIndex++;
}
- return dataSource.hasNextUnseqResource(curUnseqFileIndex);
+ return dataSource.hasNextUnseqResource(curUnseqFileIndex, false, deviceID);
}
@Override
@@ -1425,12 +1420,6 @@ public class SeriesScanUtil {
return statistics.getStartTime();
}
- @SuppressWarnings("squid:S3740")
- @Override
- public long getOrderTime(TsFileResource fileResource) {
- return fileResource.getStartTime(seriesPath.getIDeviceID());
- }
-
@SuppressWarnings("squid:S3740")
@Override
public long getOverlapCheckTime(Statistics range) {
@@ -1450,8 +1439,13 @@ public class SeriesScanUtil {
}
@Override
- public boolean isOverlapped(long time, TsFileResource right) {
- return time >= right.getStartTime(seriesPath.getIDeviceID());
+ public boolean isCurSeqOverlappedWith(long time) {
+ return time >= dataSource.getCurrentSeqOrderTime(curSeqFileIndex);
+ }
+
+ @Override
+ public boolean isCurUnSeqOverlappedWith(long time) {
+ return time >= dataSource.getCurrentUnSeqOrderTime(curUnseqFileIndex);
}
@Override
@@ -1490,32 +1484,26 @@ public class SeriesScanUtil {
@Override
public boolean hasNextSeqResource() {
- while (dataSource.hasNextSeqResource(curSeqFileIndex, getAscending())) {
- TsFileResource tsFileResource = dataSource.getSeqResourceByIndex(curSeqFileIndex);
- if (tsFileResource != null
- && (dataSource.isSingleDevice()
- || tsFileResource.isSatisfied(
- seriesPath.getIDeviceID(), scanOptions.getGlobalTimeFilter(), true, false))) {
+ while (dataSource.hasNextSeqResource(curSeqFileIndex, true, deviceID)) {
+ if (dataSource.isSeqSatisfied(
+ deviceID, curSeqFileIndex, scanOptions.getGlobalTimeFilter(), false)) {
break;
}
curSeqFileIndex++;
}
- return dataSource.hasNextSeqResource(curSeqFileIndex, getAscending());
+ return dataSource.hasNextSeqResource(curSeqFileIndex, true, deviceID);
}
@Override
public boolean hasNextUnseqResource() {
- while (dataSource.hasNextUnseqResource(curUnseqFileIndex)) {
- TsFileResource tsFileResource = dataSource.getUnseqResourceByIndex(curUnseqFileIndex);
- if (tsFileResource != null
- && (dataSource.isSingleDevice()
- || tsFileResource.isSatisfied(
- seriesPath.getIDeviceID(), scanOptions.getGlobalTimeFilter(), false, false))) {
+ while (dataSource.hasNextUnseqResource(curUnseqFileIndex, true, deviceID)) {
+ if (dataSource.isUnSeqSatisfied(
+ deviceID, curUnseqFileIndex, scanOptions.getGlobalTimeFilter(), false)) {
break;
}
curUnseqFileIndex++;
}
- return dataSource.hasNextUnseqResource(curUnseqFileIndex);
+ return dataSource.hasNextUnseqResource(curUnseqFileIndex, true, deviceID);
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java
index 97723754f7b..3557646ab82 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java
@@ -261,7 +261,7 @@ public class TimeSeriesMetadataCache {
private static final long INSTANCE_SIZE =
RamUsageEstimator.shallowSizeOfInstance(TimeSeriesMetadataCacheKey.class)
- + 2 * RamUsageEstimator.shallowSizeOfInstance(String.class);
+ + RamUsageEstimator.shallowSizeOfInstance(String.class);
private final int regionId;
private final long timePartitionId;
@@ -296,9 +296,7 @@ public class TimeSeriesMetadataCache {
}
public long getRetainedSizeInBytes() {
- return INSTANCE_SIZE
- + sizeOfCharArray(((PlainDeviceID) device).toStringID().length())
- + sizeOfCharArray(measurement.length());
+ return INSTANCE_SIZE + device.ramBytesUsed() + sizeOfCharArray(measurement.length());
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/QueryDataSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/QueryDataSource.java
index 87dabd465c0..89f92b96274 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/QueryDataSource.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/QueryDataSource.java
@@ -20,7 +20,9 @@
package org.apache.iotdb.db.storageengine.dataregion.read;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import org.apache.iotdb.tsfile.file.metadata.IDeviceID;
import org.apache.iotdb.tsfile.file.metadata.PlainDeviceID;
+import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import java.util.ArrayList;
import java.util.Comparator;
@@ -41,8 +43,22 @@ public class QueryDataSource {
*/
private final List<TsFileResource> seqResources;
+ private int curSeqIndex = -1;
+
+ // asc: startTime; desc: endTime
+ private long curSeqOrderTime = 0;
+
+ private Boolean curSeqSatisfied = null;
+
private final List<TsFileResource> unseqResources;
+ private int curUnSeqIndex = -1;
+
+ // asc: startTime; desc: endTime
+ private long curUnSeqOrderTime = 0;
+
+ private Boolean curUnSeqSatisfied = null;
+
private boolean isSingleDevice;
/* The traversal order of unseqResources (different for each device) */
@@ -74,6 +90,40 @@ public class QueryDataSource {
this.dataTTL = dataTTL;
}
+ public boolean hasNextSeqResource(int curIndex, boolean ascending, IDeviceID deviceID) {
+ boolean res = ascending ? curIndex < seqResources.size() : curIndex >= 0;
+ if (res && curIndex != this.curSeqIndex) {
+ this.curSeqIndex = curIndex;
+ this.curSeqOrderTime = seqResources.get(curIndex).getOrderTime(deviceID, ascending);
+ this.curSeqSatisfied = null;
+ }
+ return res;
+ }
+
+ public boolean isSeqSatisfied(
+ IDeviceID deviceID, int curIndex, Filter timeFilter, boolean debug) {
+ if (curIndex != this.curSeqIndex) {
+ throw new IllegalArgumentException(
+ String.format("curIndex %d is not equal to curSeqIndex %d", curIndex, this.curSeqIndex));
+ }
+ if (curSeqSatisfied == null) {
+ TsFileResource tsFileResource = seqResources.get(curSeqIndex);
+ curSeqSatisfied =
+ tsFileResource != null
+ && (isSingleDevice || tsFileResource.isSatisfied(deviceID, timeFilter, true, debug));
+ }
+
+ return curSeqSatisfied;
+ }
+
+ public long getCurrentSeqOrderTime(int curIndex) {
+ if (curIndex != this.curSeqIndex) {
+ throw new IllegalArgumentException(
+ String.format("curIndex %d is not equal to curSeqIndex %d", curIndex, this.curSeqIndex));
+ }
+ return this.curSeqOrderTime;
+ }
+
public TsFileResource getSeqResourceByIndex(int curIndex) {
if (curIndex < seqResources.size()) {
return seqResources.get(curIndex);
@@ -81,6 +131,43 @@ public class QueryDataSource {
return null;
}
+ public boolean hasNextUnseqResource(int curIndex, boolean ascending, IDeviceID deviceID) {
+ boolean res = curIndex < unseqResources.size();
+ if (res && curIndex != this.curUnSeqIndex) {
+ this.curUnSeqIndex = curIndex;
+ this.curUnSeqOrderTime =
+ seqResources.get(unSeqFileOrderIndex[curIndex]).getOrderTime(deviceID, ascending);
+ this.curUnSeqSatisfied = null;
+ }
+ return res;
+ }
+
+ public boolean isUnSeqSatisfied(
+ IDeviceID deviceID, int curIndex, Filter timeFilter, boolean debug) {
+ if (curIndex != this.curUnSeqIndex) {
+ throw new IllegalArgumentException(
+ String.format(
+ "curIndex %d is not equal to curUnSeqIndex %d", curIndex, this.curUnSeqIndex));
+ }
+ if (curUnSeqSatisfied == null) {
+ TsFileResource tsFileResource = seqResources.get(unSeqFileOrderIndex[curIndex]);
+ curUnSeqSatisfied =
+ tsFileResource != null
+ && (isSingleDevice || tsFileResource.isSatisfied(deviceID, timeFilter, false, debug));
+ }
+
+ return curUnSeqSatisfied;
+ }
+
+ public long getCurrentUnSeqOrderTime(int curIndex) {
+ if (curIndex != this.curUnSeqIndex) {
+ throw new IllegalArgumentException(
+ String.format(
+ "curIndex %d is not equal to curSeqIndex %d", curIndex, this.curUnSeqIndex));
+ }
+ return this.curUnSeqOrderTime;
+ }
+
public TsFileResource getUnseqResourceByIndex(int curIndex) {
int actualIndex = unSeqFileOrderIndex[curIndex];
if (actualIndex < unseqResources.size()) {
@@ -89,14 +176,6 @@ public class QueryDataSource {
return null;
}
- public boolean hasNextSeqResource(int curIndex, boolean ascending) {
- return ascending ? curIndex < seqResources.size() : curIndex >= 0;
- }
-
- public boolean hasNextUnseqResource(int curIndex) {
- return curIndex < unseqResources.size();
- }
-
public int getSeqResourcesSize() {
return seqResources.size();
}
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/PlainDeviceID.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/PlainDeviceID.java
index 1f0f862e5ba..1721dc1e1aa 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/PlainDeviceID.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/PlainDeviceID.java
@@ -35,7 +35,7 @@ public class PlainDeviceID implements IDeviceID {
private static final long INSTANCE_SIZE =
RamUsageEstimator.shallowSizeOfInstance(PlainDeviceID.class)
+ RamUsageEstimator.shallowSizeOfInstance(String.class);
- String deviceID;
+ private final String deviceID;
public PlainDeviceID(String deviceID) {
this.deviceID = deviceID;