You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hu...@apache.org on 2023/04/11 15:57:21 UTC
[iotdb] branch lmh/groupByTest updated: finish
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/groupByTest
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/lmh/groupByTest by this push:
new f2169a8c4f finish
f2169a8c4f is described below
commit f2169a8c4fd888d41ac2db6626b363d0b3307651
Author: liuminghui233 <54...@qq.com>
AuthorDate: Tue Apr 11 23:47:35 2023 +0800
finish
---
.../AbstractSeriesAggregationScanOperator.java | 192 ++++++++++++---------
.../iotdb/db/mpp/statistics/QueryStatistics.java | 24 ++-
2 files changed, 135 insertions(+), 81 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/AbstractSeriesAggregationScanOperator.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/AbstractSeriesAggregationScanOperator.java
index f1eb150626..e200869271 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/AbstractSeriesAggregationScanOperator.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/AbstractSeriesAggregationScanOperator.java
@@ -48,8 +48,11 @@ import static org.apache.iotdb.db.mpp.metric.QueryExecutionMetricSet.AGGREGATION
import static org.apache.iotdb.db.mpp.statistics.QueryStatistics.AGG_SCAN_OPERATOR;
import static org.apache.iotdb.db.mpp.statistics.QueryStatistics.BUILD_AGG_RES;
import static org.apache.iotdb.db.mpp.statistics.QueryStatistics.CAL_AGG_FROM_CHUNK;
+import static org.apache.iotdb.db.mpp.statistics.QueryStatistics.CAL_AGG_FROM_CHUNK_STAT;
import static org.apache.iotdb.db.mpp.statistics.QueryStatistics.CAL_AGG_FROM_FILE;
+import static org.apache.iotdb.db.mpp.statistics.QueryStatistics.CAL_AGG_FROM_FILE_STAT;
import static org.apache.iotdb.db.mpp.statistics.QueryStatistics.CAL_AGG_FROM_PAGE;
+import static org.apache.iotdb.db.mpp.statistics.QueryStatistics.CAL_AGG_FROM_PAGE_STAT;
import static org.apache.iotdb.db.mpp.statistics.QueryStatistics.CAL_NEXT_AGG_RES;
public abstract class AbstractSeriesAggregationScanOperator extends AbstractDataSourceOperator {
@@ -243,120 +246,149 @@ public abstract class AbstractSeriesAggregationScanOperator extends AbstractData
protected boolean readAndCalcFromFile() throws IOException {
long startTime = System.nanoTime();
+ long chunkCostTime = 0L;
try {
while (seriesScanUtil.hasNextFile()) {
- if (canUseCurrentFileStatistics()) {
- Statistics fileTimeStatistics = seriesScanUtil.currentFileTimeStatistics();
- if (fileTimeStatistics.getStartTime() > curTimeRange.getMax()) {
- if (ascending) {
- return true;
- } else {
- seriesScanUtil.skipCurrentFile();
- continue;
- }
- }
- // calc from fileMetaData
- if (curTimeRange.contains(
- fileTimeStatistics.getStartTime(), fileTimeStatistics.getEndTime())) {
- Statistics[] statisticsList = new Statistics[subSensorSize];
- for (int i = 0; i < subSensorSize; i++) {
- statisticsList[i] = seriesScanUtil.currentFileStatistics(i);
+ long start = System.nanoTime();
+ try {
+ if (canUseCurrentFileStatistics()) {
+ Statistics fileTimeStatistics = seriesScanUtil.currentFileTimeStatistics();
+ if (fileTimeStatistics.getStartTime() > curTimeRange.getMax()) {
+ if (ascending) {
+ return true;
+ } else {
+ seriesScanUtil.skipCurrentFile();
+ continue;
+ }
}
- calcFromStatistics(statisticsList);
- seriesScanUtil.skipCurrentFile();
- if (isAllAggregatorsHasFinalResult(aggregators) && !isGroupByQuery) {
- return true;
- } else {
- continue;
+ // calc from fileMetaData
+ if (curTimeRange.contains(
+ fileTimeStatistics.getStartTime(), fileTimeStatistics.getEndTime())) {
+ Statistics[] statisticsList = new Statistics[subSensorSize];
+ for (int i = 0; i < subSensorSize; i++) {
+ statisticsList[i] = seriesScanUtil.currentFileStatistics(i);
+ }
+ calcFromStatistics(statisticsList);
+ seriesScanUtil.skipCurrentFile();
+ if (isAllAggregatorsHasFinalResult(aggregators) && !isGroupByQuery) {
+ return true;
+ } else {
+ continue;
+ }
}
}
+ } finally {
+ operatorContext.addOperatorTime(CAL_AGG_FROM_FILE_STAT, System.nanoTime() - start);
}
// read chunk
-
- if (readAndCalcFromChunk()) {
- return true;
+ start = System.nanoTime();
+ try {
+ if (readAndCalcFromChunk()) {
+ return true;
+ }
+ } finally {
+ chunkCostTime += System.nanoTime() - start;
}
}
return false;
} finally {
- operatorContext.addOperatorTime(CAL_AGG_FROM_FILE, System.nanoTime() - startTime);
+ operatorContext.addOperatorTime(
+ CAL_AGG_FROM_FILE, System.nanoTime() - startTime - chunkCostTime);
}
}
protected boolean readAndCalcFromChunk() throws IOException {
long startTime = System.nanoTime();
+ long pageCostTime = 0L;
try {
while (seriesScanUtil.hasNextChunk()) {
- if (canUseCurrentChunkStatistics()) {
- Statistics chunkTimeStatistics = seriesScanUtil.currentChunkTimeStatistics();
- if (chunkTimeStatistics.getStartTime() > curTimeRange.getMax()) {
- if (ascending) {
- return true;
- } else {
- seriesScanUtil.skipCurrentChunk();
- continue;
+ long start = System.nanoTime();
+ try {
+ if (canUseCurrentChunkStatistics()) {
+ Statistics chunkTimeStatistics = seriesScanUtil.currentChunkTimeStatistics();
+ if (chunkTimeStatistics.getStartTime() > curTimeRange.getMax()) {
+ if (ascending) {
+ return true;
+ } else {
+ seriesScanUtil.skipCurrentChunk();
+ continue;
+ }
}
- }
- // calc from chunkMetaData
- if (curTimeRange.contains(
- chunkTimeStatistics.getStartTime(), chunkTimeStatistics.getEndTime())) {
// calc from chunkMetaData
- Statistics[] statisticsList = new Statistics[subSensorSize];
- for (int i = 0; i < subSensorSize; i++) {
- statisticsList[i] = seriesScanUtil.currentChunkStatistics(i);
- }
- calcFromStatistics(statisticsList);
- seriesScanUtil.skipCurrentChunk();
- if (isAllAggregatorsHasFinalResult(aggregators) && !isGroupByQuery) {
- return true;
- } else {
- continue;
+ if (curTimeRange.contains(
+ chunkTimeStatistics.getStartTime(), chunkTimeStatistics.getEndTime())) {
+ // calc from chunkMetaData
+ Statistics[] statisticsList = new Statistics[subSensorSize];
+ for (int i = 0; i < subSensorSize; i++) {
+ statisticsList[i] = seriesScanUtil.currentChunkStatistics(i);
+ }
+ calcFromStatistics(statisticsList);
+ seriesScanUtil.skipCurrentChunk();
+ if (isAllAggregatorsHasFinalResult(aggregators) && !isGroupByQuery) {
+ return true;
+ } else {
+ continue;
+ }
}
}
+ } finally {
+ operatorContext.addOperatorTime(CAL_AGG_FROM_CHUNK_STAT, System.nanoTime() - start);
}
// read page
- if (readAndCalcFromPage()) {
- return true;
+ start = System.nanoTime();
+ try {
+ if (readAndCalcFromPage()) {
+ return true;
+ }
+ } finally {
+ pageCostTime += System.nanoTime() - start;
}
}
return false;
} finally {
- operatorContext.addOperatorTime(CAL_AGG_FROM_CHUNK, System.nanoTime() - startTime);
+ operatorContext.addOperatorTime(
+ CAL_AGG_FROM_CHUNK, System.nanoTime() - startTime - pageCostTime);
}
}
protected boolean readAndCalcFromPage() throws IOException {
long startTime = System.nanoTime();
+ long rawDataCostTime = 0L;
try {
while (seriesScanUtil.hasNextPage()) {
- if (canUseCurrentPageStatistics()) {
- Statistics pageTimeStatistics = seriesScanUtil.currentPageTimeStatistics();
- // There is no more eligible points in current time range
- if (pageTimeStatistics.getStartTime() > curTimeRange.getMax()) {
- if (ascending) {
- return true;
- } else {
- seriesScanUtil.skipCurrentPage();
- continue;
+ long start = System.nanoTime();
+ try {
+ if (canUseCurrentPageStatistics()) {
+ Statistics pageTimeStatistics = seriesScanUtil.currentPageTimeStatistics();
+ // There is no more eligible points in current time range
+ if (pageTimeStatistics.getStartTime() > curTimeRange.getMax()) {
+ if (ascending) {
+ return true;
+ } else {
+ seriesScanUtil.skipCurrentPage();
+ continue;
+ }
}
- }
- // can use pageHeader
- if (curTimeRange.contains(
- pageTimeStatistics.getStartTime(), pageTimeStatistics.getEndTime())) {
- Statistics[] statisticsList = new Statistics[subSensorSize];
- for (int i = 0; i < subSensorSize; i++) {
- statisticsList[i] = seriesScanUtil.currentPageStatistics(i);
- }
- calcFromStatistics(statisticsList);
- seriesScanUtil.skipCurrentPage();
- if (isAllAggregatorsHasFinalResult(aggregators) && !isGroupByQuery) {
- return true;
- } else {
- continue;
+ // can use pageHeader
+ if (curTimeRange.contains(
+ pageTimeStatistics.getStartTime(), pageTimeStatistics.getEndTime())) {
+ Statistics[] statisticsList = new Statistics[subSensorSize];
+ for (int i = 0; i < subSensorSize; i++) {
+ statisticsList[i] = seriesScanUtil.currentPageStatistics(i);
+ }
+ calcFromStatistics(statisticsList);
+ seriesScanUtil.skipCurrentPage();
+ if (isAllAggregatorsHasFinalResult(aggregators) && !isGroupByQuery) {
+ return true;
+ } else {
+ continue;
+ }
}
}
+ } finally {
+ operatorContext.addOperatorTime(CAL_AGG_FROM_PAGE_STAT, System.nanoTime() - start);
}
// calc from page data
@@ -366,13 +398,19 @@ public abstract class AbstractSeriesAggregationScanOperator extends AbstractData
}
// calc from raw data
- if (calcFromRawData(tsBlock)) {
- return true;
+ start = System.nanoTime();
+ try {
+ if (calcFromRawData(tsBlock)) {
+ return true;
+ }
+ } finally {
+ rawDataCostTime += System.nanoTime() - start;
}
}
return false;
} finally {
- operatorContext.addOperatorTime(CAL_AGG_FROM_PAGE, System.nanoTime() - startTime);
+ operatorContext.addOperatorTime(
+ CAL_AGG_FROM_PAGE, System.nanoTime() - startTime - rawDataCostTime);
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java b/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java
index fcbc98cb0c..fbb49c0631 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java
@@ -101,6 +101,10 @@ public class QueryStatistics {
public static final String CAL_AGG_FROM_FILE = "CalcAggFromFile";
+ public static final String CAL_AGG_FROM_FILE_STAT = "CalcAggFromFileStat";
+ public static final String CAL_AGG_FROM_CHUNK_STAT = "CalcAggFromChunkStat";
+ public static final String CAL_AGG_FROM_PAGE_STAT = "CalcAggFromPageStat";
+
public static final String BUILD_AGG_RES = "BuildAggRes";
public static final String FILTER_AND_PROJECT_OPERATOR = "FilterAndProjectOperator";
@@ -282,14 +286,14 @@ public class QueryStatistics {
.append("| |___AggScanOperator ")
.append(operationStatistics.get(AGG_SCAN_OPERATOR))
.append(System.lineSeparator());
- builder
- .append("| | |___AggFromStat ")
- .append(operationStatistics.get(CAL_AGG_FROM_STAT))
- .append(System.lineSeparator());
builder
.append("| | |___CalcNextAggRes ")
.append(operationStatistics.get(CAL_NEXT_AGG_RES))
.append(System.lineSeparator());
+ builder
+ .append("| | |___AggFromStat ")
+ .append(operationStatistics.get(CAL_AGG_FROM_STAT))
+ .append(System.lineSeparator());
builder
.append("| | |___loadTSMeta ")
.append(operationStatistics.get(LOAD_TIME_SERIES_METADATA_ALIGNED))
@@ -298,6 +302,10 @@ public class QueryStatistics {
.append("| | |___AggFromFile ")
.append(operationStatistics.get(CAL_AGG_FROM_FILE))
.append(System.lineSeparator());
+ builder
+ .append("| | | |___AggFromFileStat ")
+ .append(operationStatistics.get(CAL_AGG_FROM_FILE_STAT))
+ .append(System.lineSeparator());
builder
.append("| | | |___loadChunkMeta ")
.append(operationStatistics.get(LOAD_CHUNK_METADATA_LIST))
@@ -306,6 +314,10 @@ public class QueryStatistics {
.append("| | | |___AggFromChunk ")
.append(operationStatistics.get(CAL_AGG_FROM_CHUNK))
.append(System.lineSeparator());
+ builder
+ .append("| | | |___AggFromChunkStat ")
+ .append(operationStatistics.get(CAL_AGG_FROM_CHUNK_STAT))
+ .append(System.lineSeparator());
builder
.append("| | | |___loadChunk ")
.append(operationStatistics.get(LOAD_PAGE_READER_LIST))
@@ -314,6 +326,10 @@ public class QueryStatistics {
.append("| | | |___AggFromPage ")
.append(operationStatistics.get(CAL_AGG_FROM_PAGE))
.append(System.lineSeparator());
+ builder
+ .append("| | | |___AggFromPageStat ")
+ .append(operationStatistics.get(CAL_AGG_FROM_PAGE_STAT))
+ .append(System.lineSeparator());
builder
.append("| | | |___loadPage ")
.append(operationStatistics.get(PAGE_READER))