You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hu...@apache.org on 2023/04/14 02:40:54 UTC
[iotdb] branch lmh/groupByTest updated (930d224607 -> e8cc0dcb1c)
This is an automated email from the ASF dual-hosted git repository.
hui pushed a change to branch lmh/groupByTest
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 930d224607 spotless apply
new 59bcf3a57d fix load chunk
new e8cc0dcb1c fix load chunk
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../apache/iotdb/db/engine/cache/ChunkCache.java | 5 ++-
.../execution/operator/source/SeriesScanUtil.java | 50 ++++++++++------------
.../iotdb/db/mpp/statistics/QueryStatistics.java | 50 ++++++++--------------
.../query/reader/chunk/DiskAlignedChunkLoader.java | 5 ++-
.../db/query/reader/chunk/DiskChunkLoader.java | 5 ++-
.../org/apache/iotdb/db/utils/FileLoaderUtils.java | 3 +-
6 files changed, 53 insertions(+), 65 deletions(-)
[iotdb] 02/02: fix load chunk
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/groupByTest
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit e8cc0dcb1ca29469392e42c17a8e1f3b3fe48d99
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Fri Apr 14 10:40:35 2023 +0800
fix load chunk
---
.../execution/operator/source/SeriesScanUtil.java | 50 ++++++++++------------
.../iotdb/db/mpp/statistics/QueryStatistics.java | 40 ++++-------------
.../org/apache/iotdb/db/utils/FileLoaderUtils.java | 3 +-
3 files changed, 31 insertions(+), 62 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/SeriesScanUtil.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/SeriesScanUtil.java
index 70a540070e..2bebeef734 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/SeriesScanUtil.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/SeriesScanUtil.java
@@ -927,37 +927,31 @@ public class SeriesScanUtil {
}
private void initFirstPageReader() throws IOException {
- long startTime = System.nanoTime();
- try {
- while (this.firstPageReader == null) {
- VersionPageReader firstPageReader = getFirstPageReaderFromCachedReaders();
-
- // unpack overlapped page using current page reader
- if (firstPageReader != null) {
- long overlapCheckTime = orderUtils.getOverlapCheckTime(firstPageReader.getStatistics());
- unpackAllOverlappedTsFilesToTimeSeriesMetadata(overlapCheckTime);
- unpackAllOverlappedTimeSeriesMetadataToCachedChunkMetadata(overlapCheckTime, false);
- unpackAllOverlappedChunkMetadataToPageReaders(overlapCheckTime, false);
-
- // this page after unpacking must be the first page
- if (firstPageReader.equals(getFirstPageReaderFromCachedReaders())) {
- this.firstPageReader = firstPageReader;
- if (!seqPageReaders.isEmpty() && firstPageReader.equals(seqPageReaders.get(0))) {
- seqPageReaders.remove(0);
- break;
- } else if (!unSeqPageReaders.isEmpty()
- && firstPageReader.equals(unSeqPageReaders.peek())) {
- unSeqPageReaders.poll();
- break;
- }
+ while (this.firstPageReader == null) {
+ VersionPageReader firstPageReader = getFirstPageReaderFromCachedReaders();
+
+ // unpack overlapped page using current page reader
+ if (firstPageReader != null) {
+ long overlapCheckTime = orderUtils.getOverlapCheckTime(firstPageReader.getStatistics());
+ unpackAllOverlappedTsFilesToTimeSeriesMetadata(overlapCheckTime);
+ unpackAllOverlappedTimeSeriesMetadataToCachedChunkMetadata(overlapCheckTime, false);
+ unpackAllOverlappedChunkMetadataToPageReaders(overlapCheckTime, false);
+
+ // this page after unpacking must be the first page
+ if (firstPageReader.equals(getFirstPageReaderFromCachedReaders())) {
+ this.firstPageReader = firstPageReader;
+ if (!seqPageReaders.isEmpty() && firstPageReader.equals(seqPageReaders.get(0))) {
+ seqPageReaders.remove(0);
+ break;
+ } else if (!unSeqPageReaders.isEmpty()
+ && firstPageReader.equals(unSeqPageReaders.peek())) {
+ unSeqPageReaders.poll();
+ break;
}
- } else {
- return;
}
+ } else {
+ return;
}
- } finally {
- QueryStatistics.getInstance()
- .addCost(QueryStatistics.INIT_FIRST_PAGE, System.nanoTime() - startTime);
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java b/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java
index 783af3bb82..70492f79d0 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java
@@ -59,10 +59,6 @@ public class QueryStatistics {
public static final String CHECK_MEMORY = "CheckMem";
- public static final String ALLOC_EX_MEMORY = "AllocExchangeMem";
-
- public static final String QUERY_EXECUTION = "QueryExecution";
-
public static final String QUERY_RESOURCE_INIT = "QueryResourceInit";
public static final String INIT_SOURCE_OP = "InitSourceOp";
@@ -74,17 +70,8 @@ public class QueryStatistics {
public static final String LOCAL_SOURCE_HANDLE_SER_TSBLOCK = "LocalSourceHandleSerializeTsBlock";
- public static final String REMOTE_SOURCE_HANDLE_GET_TSBLOCK = "RemoteSourceHandleGetTsBlock";
-
- public static final String REMOTE_SOURCE_HANDLE_DESER_TSBLOCK =
- "RemoteSourceHandleDeserializeTsBlock";
-
public static final String WAIT_FOR_RESULT = "WaitForResult";
- public static final String SERIES_SCAN_OPERATOR = "SeriesScanOperator";
-
- public static final String ALIGNED_SERIES_SCAN_OPERATOR = "AlignedSeriesScanOperator";
-
public static final String AGG_SCAN_OPERATOR = "AbstractSeriesAggregationScanOperator";
public static final String CAL_NEXT_AGG_RES = "CalcNextAggRes";
@@ -107,11 +94,6 @@ public class QueryStatistics {
public static final String BUILD_AGG_RES = "BuildAggRes";
- public static final String FILTER_AND_PROJECT_OPERATOR = "FilterAndProjectOperator";
-
- public static final String SINGLE_INPUT_AGG_OPERATOR = "SingleInputAggregationOperator";
-
- public static final String PAGE_READER = "IPageReader";
public static final String PARSER = "Parser";
public static final String CREATE_QUERY_EXEC = "CreateQueryExec";
@@ -125,8 +107,6 @@ public class QueryStatistics {
public static final String DISTRIBUTION_PLANNER = "DistributionPlanner";
public static final String DISPATCHER = "Dispatcher";
- public static final String WAIT_FOR_DISPATCH = "WaitForDispatch";
-
public static final String DISPATCH_READ = "DispatchRead";
public static final String DRIVER_CLOSE = "CloseDriver";
@@ -153,27 +133,23 @@ public class QueryStatistics {
public static final String SERVER_RPC_RT = "ServerRpcRT";
- public static final String LOAD_TIME_SERIES_METADATA_ALIGNED = "loadTimeSeriesMetadata-aligned";
public static final String LOAD_TIME_SERIES_METADATA = "loadTimeSeriesMetadata";
public static final String LOAD_CHUNK_METADATA_LIST = "loadChunkMetadataList";
public static final String LOAD_PAGE_READER_LIST = "loadPageReaderList";
- public static final String TIME_SERIES_METADATA_CACHE_MISS = "TimeSeriesMetadataCacheMiss";
- public static final String CHUNK_CACHE_MISS = "ChunkCacheMiss";
- public static final String LOAD_CHUNK = "ChunkCacheMiss";
+ public static final String LOAD_CHUNK = "loadChunk";
public static final String INIT_PAGE_READERS = "initAllPageReaders";
+ public static final String PAGE_READER = "IPageReader";
public static final String HAS_NEXT_FILE = "hasNextFile";
- public static final String HAS_NEXT_CHUNK = "hasNextChunk";
- public static final String HAS_NEXT_PAGE = "hasNextPage";
- public static final String HAS_NEXT_OVERLAPPED_PAGE = "hasNextOverlappedPage";
-
public static final String FILTER_FIRST_TIMESERIES_METADATA = "filterFirstTimeSeriesMetadata";
- public static final String FILTER_FIRST_CHUNK_METADATA = "filterFirstChunkMetadata";
-
public static final String FIND_END_TIME = "findEndTime";
public static final String PICK_FIRST_TIMESERIES_METADATA = "pickFirstTimeSeriesMetadata";
- public static final String INIT_FIRST_PAGE = "initFirstPage";
+ public static final String HAS_NEXT_CHUNK = "hasNextChunk";
+ public static final String FILTER_FIRST_CHUNK_METADATA = "filterFirstChunkMetadata";
+
+ public static final String HAS_NEXT_PAGE = "hasNextPage";
+ public static final String HAS_NEXT_OVERLAPPED_PAGE = "hasNextOverlappedPage";
public static final String MERGE_READER_ADD_READER = "mergeReader#addReader";
public static final String MERGE_READER_NEXT = "mergeReader#nextTimeValuePair";
public static final String MERGE_READER_UPDATE_HEAP = "mergeReader#updateHeap";
@@ -310,7 +286,7 @@ public class QueryStatistics {
builder.append("| | |[FileLoaderInterface]").append(System.lineSeparator());
builder
.append("| | |___loadTSMetadata ")
- .append(operationStatistics.get(LOAD_TIME_SERIES_METADATA_ALIGNED))
+ .append(operationStatistics.get(LOAD_TIME_SERIES_METADATA))
.append(System.lineSeparator());
builder
.append("| | |___loadChunkMetaList ")
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
index 5387aafbf0..6cd3a6d50c 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
@@ -294,8 +294,7 @@ public class FileLoaderUtils {
? LOAD_TIMESERIES_METADATA_ALIGNED_MEM
: LOAD_TIMESERIES_METADATA_ALIGNED_DISK,
costTime);
- QueryStatistics.getInstance()
- .addCost(QueryStatistics.LOAD_TIME_SERIES_METADATA_ALIGNED, costTime);
+ QueryStatistics.getInstance().addCost(QueryStatistics.LOAD_TIME_SERIES_METADATA, costTime);
}
}
[iotdb] 01/02: fix load chunk
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/groupByTest
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 59bcf3a57d5f6bf24cada9c549760a1e2380d30d
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Fri Apr 14 10:32:48 2023 +0800
fix load chunk
---
.../java/org/apache/iotdb/db/engine/cache/ChunkCache.java | 5 ++++-
.../org/apache/iotdb/db/mpp/statistics/QueryStatistics.java | 12 +++++++++++-
.../iotdb/db/query/reader/chunk/DiskAlignedChunkLoader.java | 5 ++++-
.../apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java | 5 ++++-
4 files changed, 23 insertions(+), 4 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
index 5564df780c..0258522c09 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.mpp.metric.ChunkCacheMetrics;
import org.apache.iotdb.db.mpp.metric.QueryMetricsManager;
+import org.apache.iotdb.db.mpp.statistics.QueryStatistics;
import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
@@ -136,7 +137,9 @@ public class ChunkCache {
chunkMetaData.getDeleteIntervalList(),
chunkMetaData.getStatistics());
} finally {
- QUERY_METRICS.recordSeriesScanCost(READ_CHUNK_ALL, System.nanoTime() - startTime);
+ long costTime = System.nanoTime() - startTime;
+ QUERY_METRICS.recordSeriesScanCost(READ_CHUNK_ALL, costTime);
+ QueryStatistics.getInstance().addCost(QueryStatistics.LOAD_CHUNK, costTime);
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java b/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java
index ef615393d4..783af3bb82 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java
@@ -38,7 +38,7 @@ import java.util.concurrent.atomic.AtomicLong;
@ThreadSafe
public class QueryStatistics {
- private static final long QUERY_STATISTICS_PRINT_INTERVAL_IN_MS = 1_000_000;
+ private static final long QUERY_STATISTICS_PRINT_INTERVAL_IN_MS = 100_000;
private static final Logger QUERY_STATISTICS_LOGGER =
LoggerFactory.getLogger(IoTDBConstant.QUERY_STATISTICS_LOGGER_NAME);
@@ -159,6 +159,8 @@ public class QueryStatistics {
public static final String LOAD_PAGE_READER_LIST = "loadPageReaderList";
public static final String TIME_SERIES_METADATA_CACHE_MISS = "TimeSeriesMetadataCacheMiss";
public static final String CHUNK_CACHE_MISS = "ChunkCacheMiss";
+ public static final String LOAD_CHUNK = "ChunkCacheMiss";
+ public static final String INIT_PAGE_READERS = "initAllPageReaders";
public static final String HAS_NEXT_FILE = "hasNextFile";
public static final String HAS_NEXT_CHUNK = "hasNextChunk";
@@ -318,6 +320,14 @@ public class QueryStatistics {
.append("| | |___loadPageReaderList ")
.append(operationStatistics.get(LOAD_PAGE_READER_LIST))
.append(System.lineSeparator());
+ builder
+ .append("| | | |___loadChunk ")
+ .append(operationStatistics.get(LOAD_CHUNK))
+ .append(System.lineSeparator());
+ builder
+ .append("| | | |___initPageReaders ")
+ .append(operationStatistics.get(INIT_PAGE_READERS))
+ .append(System.lineSeparator());
builder
.append("| | |___pageReader ")
.append(operationStatistics.get(PAGE_READER))
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskAlignedChunkLoader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskAlignedChunkLoader.java
index b433641af0..efb0ca8dd2 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskAlignedChunkLoader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskAlignedChunkLoader.java
@@ -20,6 +20,7 @@ package org.apache.iotdb.db.query.reader.chunk;
import org.apache.iotdb.db.engine.cache.ChunkCache;
import org.apache.iotdb.db.mpp.metric.QueryMetricsManager;
+import org.apache.iotdb.db.mpp.statistics.QueryStatistics;
import org.apache.iotdb.tsfile.file.metadata.AlignedChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
@@ -72,7 +73,9 @@ public class DiskAlignedChunkLoader implements IChunkLoader {
long t2 = System.nanoTime();
IChunkReader chunkReader = new AlignedChunkReader(timeChunk, valueChunkList, timeFilter);
- QUERY_METRICS.recordSeriesScanCost(INIT_CHUNK_READER_ALIGNED_DISK, System.nanoTime() - t2);
+ long costTime = System.nanoTime() - t2;
+ QUERY_METRICS.recordSeriesScanCost(INIT_CHUNK_READER_ALIGNED_DISK, costTime);
+ QueryStatistics.getInstance().addCost(QueryStatistics.INIT_PAGE_READERS, costTime);
return chunkReader;
} finally {
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java
index 85ee83bee4..b038737719 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.db.query.reader.chunk;
import org.apache.iotdb.db.engine.cache.ChunkCache;
import org.apache.iotdb.db.mpp.metric.QueryMetricsManager;
+import org.apache.iotdb.db.mpp.statistics.QueryStatistics;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
import org.apache.iotdb.tsfile.read.common.Chunk;
@@ -65,7 +66,9 @@ public class DiskChunkLoader implements IChunkLoader {
long t2 = System.nanoTime();
IChunkReader chunkReader = new ChunkReader(chunk, timeFilter);
- QUERY_METRICS.recordSeriesScanCost(INIT_CHUNK_READER_NONALIGNED_DISK, System.nanoTime() - t2);
+ long costTime = System.nanoTime() - t2;
+ QUERY_METRICS.recordSeriesScanCost(INIT_CHUNK_READER_NONALIGNED_DISK, costTime);
+ QueryStatistics.getInstance().addCost(QueryStatistics.INIT_PAGE_READERS, costTime);
return chunkReader;
} finally {