You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by le...@apache.org on 2023/07/09 17:17:37 UTC
[iotdb] 11/46: [important] four-layer metrics design done, from client, server execute&fetch, dataset&executor, basic ops ABCD, improved D search array 4 functions
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit b24dbd33b034bf71f505476f9b9f97e481ef318f
Author: Lei Rui <10...@qq.com>
AuthorDate: Tue Jun 13 22:08:35 2023 +0800
[important] four-layer metrics design done, from client, server execute&fetch, dataset&executor, basic ops ABCD, improved D search array 4 functions
---
.../dataset/groupby/LocalGroupByExecutor4CPV.java | 18 ++
.../iotdb/tsfile/read/common/ChunkSuit4CPV.java | 40 ++-
.../iotdb/tsfile/read/common/IOMonitor2.java | 282 +++++++++++++++++----
.../iotdb/tsfile/read/reader/page/PageReader.java | 3 +
4 files changed, 289 insertions(+), 54 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java
index b4c59643688..92026420e17 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java
@@ -38,6 +38,8 @@ import org.apache.iotdb.tsfile.file.metadata.statistics.IntegerStatistics;
import org.apache.iotdb.tsfile.file.metadata.statistics.LongStatistics;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.common.ChunkSuit4CPV;
+import org.apache.iotdb.tsfile.read.common.IOMonitor2;
+import org.apache.iotdb.tsfile.read.common.IOMonitor2.Operation;
import org.apache.iotdb.tsfile.read.common.TimeRange;
import org.apache.iotdb.tsfile.read.filter.GroupByFilter;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
@@ -93,6 +95,7 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
TsFileFilter fileFilter,
boolean ascending)
throws StorageEngineException, QueryProcessException {
+ long start = System.nanoTime();
this.tsDataType = dataType;
// this.mergeReader = new PriorityMergeReader();
@@ -225,6 +228,8 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
} catch (IOException e) {
throw new QueryProcessException(e.getMessage());
}
+
+ IOMonitor2.addMeasure(Operation.M4_LSM_INIT_LOAD_ALL_CHUNKMETADATAS, System.nanoTime() - start);
}
@Override
@@ -309,16 +314,29 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
result.reset();
}
+ long start = System.nanoTime();
getCurrentChunkListFromFutureChunkList(curStartTime, curEndTime, startTime, endTime, interval);
+ IOMonitor2.addMeasure(Operation.M4_LSM_MERGE_M4_TIME_SPAN, System.nanoTime() - start);
if (currentChunkList.size() == 0) {
return results;
}
+ start = System.nanoTime();
calculateFirstPoint(currentChunkList, startTime, endTime, interval, curStartTime);
+ IOMonitor2.addMeasure(Operation.M4_LSM_FP, System.nanoTime() - start);
+
+ start = System.nanoTime();
calculateLastPoint(currentChunkList, startTime, endTime, interval, curStartTime);
+ IOMonitor2.addMeasure(Operation.M4_LSM_LP, System.nanoTime() - start);
+
+ start = System.nanoTime();
calculateBottomPoint(currentChunkList, startTime, endTime, interval, curStartTime);
+ IOMonitor2.addMeasure(Operation.M4_LSM_BP, System.nanoTime() - start);
+
+ start = System.nanoTime();
calculateTopPoint(currentChunkList, startTime, endTime, interval, curStartTime);
+ IOMonitor2.addMeasure(Operation.M4_LSM_TP, System.nanoTime() - start);
return results;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
index 0834348a268..67520439d22 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
@@ -29,6 +29,7 @@ import org.apache.iotdb.tsfile.file.metadata.statistics.LongStatistics;
import org.apache.iotdb.tsfile.file.metadata.statistics.MinMaxInfo;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.file.metadata.statistics.StepRegress;
+import org.apache.iotdb.tsfile.read.common.IOMonitor2.Operation;
import org.apache.iotdb.tsfile.read.reader.page.PageReader;
import java.io.IOException;
@@ -245,10 +246,12 @@ public class ChunkSuit4CPV {
* @return the position of the point, starting from 0
*/
public int updateFPwithTheClosetPointEqualOrAfter(long targetTimestamp) throws IOException {
+ long start = System.nanoTime();
+ int estimatedPos;
if (TSFileDescriptor.getInstance().getConfig().isUseChunkIndex()) {
StepRegress stepRegress = chunkMetadata.getStatistics().getStepRegress();
// infer position starts from 1, so minus 1 here
- int estimatedPos = (int) Math.round(stepRegress.infer(targetTimestamp)) - 1;
+ estimatedPos = (int) Math.round(stepRegress.infer(targetTimestamp)) - 1;
// search from estimatePos in the timeBuffer to find the closet timestamp equal to or larger
// than the given timestamp
@@ -256,15 +259,18 @@ public class ChunkSuit4CPV {
while (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
estimatedPos++;
IOMonitor.incPointsTravered();
+ IOMonitor2.DCP_D_traversedPointNum++;
}
} else if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
while (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
estimatedPos--;
IOMonitor.incPointsTravered();
+ IOMonitor2.DCP_D_traversedPointNum++;
}
if (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
estimatedPos++;
IOMonitor.incPointsTravered();
+ IOMonitor2.DCP_D_traversedPointNum++;
} // else equal
} // else equal
this.startPos = estimatedPos; // note this
@@ -298,16 +304,16 @@ public class ChunkSuit4CPV {
default:
throw new IOException("Unsupported data type!");
}
- return estimatedPos;
} else {
// search from estimatePos in the timeBuffer to find the closet timestamp equal to or larger
// than the given timestamp
- int estimatedPos = -1;
+ estimatedPos = -1;
pageReader.timeBuffer.position(0);
pageReader.valueBuffer.position(pageReader.timeBufferLength);
while (pageReader.timeBuffer.remaining() > 0) {
estimatedPos++;
IOMonitor.incPointsTravered();
+ IOMonitor2.DCP_D_traversedPointNum++;
long t = pageReader.timeBuffer.getLong();
if (t >= targetTimestamp) {
break;
@@ -346,6 +352,8 @@ public class ChunkSuit4CPV {
}
return estimatedPos;
}
+ IOMonitor2.addMeasure(Operation.SEARCH_ARRAY_b_genFP, System.nanoTime() - start);
+ return estimatedPos;
}
/**
@@ -356,10 +364,12 @@ public class ChunkSuit4CPV {
* @return the position of the point, starting from 0
*/
public int updateLPwithTheClosetPointEqualOrBefore(long targetTimestamp) throws IOException {
+ long start = System.nanoTime();
+ int estimatedPos;
if (TSFileDescriptor.getInstance().getConfig().isUseChunkIndex()) {
StepRegress stepRegress = chunkMetadata.getStatistics().getStepRegress();
// infer position starts from 1, so minus 1 here
- int estimatedPos = (int) Math.round(stepRegress.infer(targetTimestamp)) - 1;
+ estimatedPos = (int) Math.round(stepRegress.infer(targetTimestamp)) - 1;
// search from estimatePos in the timeBuffer to find the closet timestamp equal to or smaller
// than the given timestamp
@@ -367,15 +377,18 @@ public class ChunkSuit4CPV {
while (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
estimatedPos--;
IOMonitor.incPointsTravered();
+ IOMonitor2.DCP_D_traversedPointNum++;
}
} else if (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
while (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
estimatedPos++;
IOMonitor.incPointsTravered();
+ IOMonitor2.DCP_D_traversedPointNum++;
}
if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
estimatedPos--;
IOMonitor.incPointsTravered();
+ IOMonitor2.DCP_D_traversedPointNum++;
} // else equal
} // else equal
this.endPos = estimatedPos; // note this
@@ -413,12 +426,13 @@ public class ChunkSuit4CPV {
} else {
// to find the closet timestamp equal to or smaller
// than the given timestamp
- int estimatedPos = -1;
+ estimatedPos = -1;
pageReader.timeBuffer.position(0);
pageReader.valueBuffer.position(pageReader.timeBufferLength);
while (pageReader.timeBuffer.remaining() > 0) {
estimatedPos++;
IOMonitor.incPointsTravered();
+ IOMonitor2.DCP_D_traversedPointNum++;
long t = pageReader.timeBuffer.getLong();
if (t >= targetTimestamp) {
break;
@@ -426,6 +440,7 @@ public class ChunkSuit4CPV {
}
if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
IOMonitor.incPointsTravered();
+ IOMonitor2.DCP_D_traversedPointNum++;
estimatedPos--;
} // else equals no need to minus 1
@@ -460,8 +475,9 @@ public class ChunkSuit4CPV {
default:
throw new IOException("Unsupported data type!");
}
- return estimatedPos;
}
+ IOMonitor2.addMeasure(Operation.SEARCH_ARRAY_b_genLP, System.nanoTime() - start);
+ return estimatedPos;
}
/**
@@ -471,6 +487,8 @@ public class ChunkSuit4CPV {
* @return true if exists; false not exist
*/
public boolean checkIfExist(long targetTimestamp) throws IOException {
+ long start = System.nanoTime();
+ boolean exist;
if (TSFileDescriptor.getInstance().getConfig().isUseChunkIndex()) {
StepRegress stepRegress = chunkMetadata.getStatistics().getStepRegress();
// infer position starts from 1, so minus 1 here
@@ -483,22 +501,25 @@ public class ChunkSuit4CPV {
while (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
estimatedPos--;
IOMonitor.incPointsTravered();
+ IOMonitor2.DCP_D_traversedPointNum++;
}
} else if (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
while (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
estimatedPos++;
IOMonitor.incPointsTravered();
+ IOMonitor2.DCP_D_traversedPointNum++;
}
if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
estimatedPos--;
IOMonitor.incPointsTravered();
+ IOMonitor2.DCP_D_traversedPointNum++;
} // else equal
} // else equal
// since we have constrained that targetTimestamp must be within the chunk time range
// [startTime, endTime],
// estimatedPos will not be out of range.
- return pageReader.timeBuffer.getLong(estimatedPos * 8) == targetTimestamp;
+ exist = pageReader.timeBuffer.getLong(estimatedPos * 8) == targetTimestamp;
} else {
// search from estimatePos in the timeBuffer to find the closet timestamp equal to or smaller
// than the given timestamp
@@ -510,6 +531,7 @@ public class ChunkSuit4CPV {
if (!flag) {
estimatedPos++;
IOMonitor.incPointsTravered();
+ IOMonitor2.DCP_D_traversedPointNum++;
}
long t = pageReader.timeBuffer.getLong();
if (t >= targetTimestamp) {
@@ -520,8 +542,10 @@ public class ChunkSuit4CPV {
// since we have constrained that targetTimestamp must be within the chunk time range
// [startTime, endTime],
// estimatedPos will not be out of range.
- return pageReader.timeBuffer.getLong(estimatedPos * 8) == targetTimestamp;
+ exist = pageReader.timeBuffer.getLong(estimatedPos * 8) == targetTimestamp;
}
+ IOMonitor2.addMeasure(Operation.SEARCH_ARRAY_a_verifBPTP, System.nanoTime() - start);
+ return exist;
}
public void updateFP(MinMaxInfo point) {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
index d9eeaf30b6c..9c17f371e34 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
@@ -19,10 +19,8 @@
package org.apache.iotdb.tsfile.read.common;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
public class IOMonitor2 {
+
public enum DataSetType { // dataSet, executor, reader, file
NONE,
RawQueryDataSetWithoutValueFilter,
@@ -33,12 +31,21 @@ public class IOMonitor2 {
public enum Operation {
DCP_Server_Query_Execute("DCP_Server_Query_Execute"),
DCP_Server_Query_Fetch("DCP_Server_Query_Fetch"),
- DCP_SeriesScanOperator_hasNext("DCP_SeriesScanOperator_hasNext"),
DCP_A_GET_CHUNK_METADATAS("DCP_A_GET_CHUNK_METADATAS"),
DCP_B_READ_MEM_CHUNK("DCP_B_READ_MEM_CHUNK"),
DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA(
"DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA"),
- DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS("DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS");
+ DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS("DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS"),
+ SEARCH_ARRAY_a_verifBPTP("SEARCH_ARRAY_a_verifBPTP"),
+ SEARCH_ARRAY_b_genFP("SEARCH_ARRAY_b_genFP"),
+ SEARCH_ARRAY_b_genLP("SEARCH_ARRAY_b_genLP"),
+ SEARCH_ARRAY_c_genBPTP("SEARCH_ARRAY_c_genBPTP"),
+ M4_LSM_INIT_LOAD_ALL_CHUNKMETADATAS("M4_LSM_INIT_LOAD_ALL_CHUNKMETADATAS"),
+ M4_LSM_MERGE_M4_TIME_SPAN("M4_LSM_MERGE_M4_TIME_SPAN"),
+ M4_LSM_FP("M4_LSM_FP"),
+ M4_LSM_LP("M4_LSM_LP"),
+ M4_LSM_BP("M4_LSM_BP"),
+ M4_LSM_TP("M4_LSM_TP");
public String getName() {
return name;
@@ -51,23 +58,14 @@ public class IOMonitor2 {
}
}
- /**
- * [Raw data query metrics] - server端execute&fetch的次数和耗时 -
- * SeriesScanOperator.hasNext的次数和耗时（问题：root.hasNext执行次数等于decode
- * page次数，但是SeriesScanOperator.hasNext次数大于那个次数？破案了：是root.isFinished->SeriesScanOperator.isFinished里面调用了hasNext，和root.hasNext里调用SeriesScanOperator.hasNext的次数加起来就对了。）
- * - A：从磁盘解读chunk元数据的次数和耗时 - B：从磁盘加载chunk数据的次数和耗时 - C：解压缩所有page构造pageReaderList的次数和耗时 -
- * D：解码page并遍历点的次数和耗时 （遍历点数不用统计，因为一定是全量点）
- */
- public static int DCP_Server_Query_Execute_count = 0;
+ public static DataSetType dataSetType = DataSetType.NONE;
+ public static int DCP_Server_Query_Execute_count = 0;
public static long DCP_Server_Query_Execute_ns = 0;
public static int DCP_Server_Query_Fetch_count = 0;
public static long DCP_Server_Query_Fetch_ns = 0;
- public static int DCP_SeriesScanOperator_hasNext_count = 0;
- public static long DCP_SeriesScanOperator_hasNext_ns = 0;
-
public static int DCP_A_GET_CHUNK_METADATAS_count = 0;
public static long DCP_A_GET_CHUNK_METADATAS_ns = 0;
@@ -77,25 +75,49 @@ public class IOMonitor2 {
public static int DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_count = 0;
public static long DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_ns = 0;
+ public static long DCP_D_traversedPointNum = 0;
public static int DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_count = 0;
public static long DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_ns = 0;
- public static long DCP_D_traversedPointNum = 0;
+ public static int SEARCH_ARRAY_a_verifBPTP_count = 0;
+ public static long SEARCH_ARRAY_a_verifBPTP_ns = 0;
- public static DataSetType dataSetType = DataSetType.NONE;
+ public static int SEARCH_ARRAY_b_genFP_count = 0;
+ public static long SEARCH_ARRAY_b_genFP_ns = 0;
+
+ public static int SEARCH_ARRAY_b_genLP_count = 0;
+ public static long SEARCH_ARRAY_b_genLP_ns = 0;
+
+ public static int SEARCH_ARRAY_c_genBPTP_count = 0;
+ public static long SEARCH_ARRAY_c_genBPTP_ns = 0;
+
+ public static int M4_LSM_init_loadAllChunkMetadatas_count = 0;
+ public static long M4_LSM_init_loadAllChunkMetadatas_ns = 0;
- private static final Logger DEBUG_LOGGER = LoggerFactory.getLogger("IOMonitor2");
+ public static int M4_LSM_merge_M4_time_span_count = 0;
+ public static long M4_LSM_merge_M4_time_span_ns = 0;
+
+ public static int M4_LSM_FP_count = 0;
+ public static long M4_LSM_FP_ns = 0;
+
+ public static int M4_LSM_LP_count = 0;
+ public static long M4_LSM_LP_ns = 0;
+
+ public static int M4_LSM_BP_count = 0;
+ public static long M4_LSM_BP_ns = 0;
+
+ public static int M4_LSM_TP_count = 0;
+ public static long M4_LSM_TP_ns = 0;
private static void reset() {
+ dataSetType = DataSetType.NONE;
+
DCP_Server_Query_Execute_count = 0;
DCP_Server_Query_Execute_ns = 0;
DCP_Server_Query_Fetch_count = 0;
DCP_Server_Query_Fetch_ns = 0;
- DCP_SeriesScanOperator_hasNext_count = 0;
- DCP_SeriesScanOperator_hasNext_ns = 0;
-
DCP_A_GET_CHUNK_METADATAS_count = 0;
DCP_A_GET_CHUNK_METADATAS_ns = 0;
@@ -105,17 +127,39 @@ public class IOMonitor2 {
DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_count = 0;
DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_ns = 0;
+ DCP_D_traversedPointNum = 0;
DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_count = 0;
DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_ns = 0;
- DCP_D_traversedPointNum = 0;
+ SEARCH_ARRAY_a_verifBPTP_count = 0;
+ SEARCH_ARRAY_a_verifBPTP_ns = 0;
+ SEARCH_ARRAY_b_genFP_count = 0;
+ SEARCH_ARRAY_b_genFP_ns = 0;
+ SEARCH_ARRAY_b_genLP_count = 0;
+ SEARCH_ARRAY_b_genLP_ns = 0;
+ SEARCH_ARRAY_c_genBPTP_count = 0;
+ SEARCH_ARRAY_c_genBPTP_ns = 0;
- dataSetType = DataSetType.NONE;
+ M4_LSM_init_loadAllChunkMetadatas_count = 0;
+ M4_LSM_init_loadAllChunkMetadatas_ns = 0;
+
+ M4_LSM_merge_M4_time_span_count = 0;
+ M4_LSM_merge_M4_time_span_ns = 0;
+
+ M4_LSM_FP_count = 0;
+ M4_LSM_FP_ns = 0;
+
+ M4_LSM_LP_count = 0;
+ M4_LSM_LP_ns = 0;
+
+ M4_LSM_BP_count = 0;
+ M4_LSM_BP_ns = 0;
+
+ M4_LSM_TP_count = 0;
+ M4_LSM_TP_ns = 0;
}
public static void addMeasure(Operation operation, long elapsedTimeInNanosecond) {
- // TODO tmp for debug
- // DEBUG_LOGGER.info(operation.getName() + ": " + elapsedTimeInNanosecond + " ns");
switch (operation) {
case DCP_Server_Query_Execute:
DCP_Server_Query_Execute_count++;
@@ -125,10 +169,6 @@ public class IOMonitor2 {
DCP_Server_Query_Fetch_count++;
DCP_Server_Query_Fetch_ns += elapsedTimeInNanosecond;
break;
- case DCP_SeriesScanOperator_hasNext:
- DCP_SeriesScanOperator_hasNext_count++;
- DCP_SeriesScanOperator_hasNext_ns += elapsedTimeInNanosecond;
- break;
case DCP_A_GET_CHUNK_METADATAS:
DCP_A_GET_CHUNK_METADATAS_count++;
DCP_A_GET_CHUNK_METADATAS_ns += elapsedTimeInNanosecond;
@@ -145,6 +185,46 @@ public class IOMonitor2 {
DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_count++;
DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_ns += elapsedTimeInNanosecond;
break;
+ case SEARCH_ARRAY_a_verifBPTP:
+ SEARCH_ARRAY_a_verifBPTP_count++;
+ SEARCH_ARRAY_a_verifBPTP_ns += elapsedTimeInNanosecond;
+ break;
+ case SEARCH_ARRAY_b_genFP:
+ SEARCH_ARRAY_b_genFP_count++;
+ SEARCH_ARRAY_b_genFP_ns += elapsedTimeInNanosecond;
+ break;
+ case SEARCH_ARRAY_b_genLP:
+ SEARCH_ARRAY_b_genLP_count++;
+ SEARCH_ARRAY_b_genLP_ns += elapsedTimeInNanosecond;
+ break;
+ case SEARCH_ARRAY_c_genBPTP:
+ SEARCH_ARRAY_c_genBPTP_count++;
+ SEARCH_ARRAY_c_genBPTP_ns += elapsedTimeInNanosecond;
+ break;
+ case M4_LSM_INIT_LOAD_ALL_CHUNKMETADATAS:
+ M4_LSM_init_loadAllChunkMetadatas_count++;
+ M4_LSM_init_loadAllChunkMetadatas_ns += elapsedTimeInNanosecond;
+ break;
+ case M4_LSM_MERGE_M4_TIME_SPAN:
+ M4_LSM_merge_M4_time_span_count++;
+ M4_LSM_merge_M4_time_span_ns += elapsedTimeInNanosecond;
+ break;
+ case M4_LSM_FP:
+ M4_LSM_FP_count++;
+ M4_LSM_FP_ns += elapsedTimeInNanosecond;
+ break;
+ case M4_LSM_LP:
+ M4_LSM_LP_count++;
+ M4_LSM_LP_ns += elapsedTimeInNanosecond;
+ break;
+ case M4_LSM_BP:
+ M4_LSM_BP_count++;
+ M4_LSM_BP_ns += elapsedTimeInNanosecond;
+ break;
+ case M4_LSM_TP:
+ M4_LSM_TP_count++;
+ M4_LSM_TP_ns += elapsedTimeInNanosecond;
+ break;
default:
System.out.println("not supported operation type"); // this will not happen
break;
@@ -161,10 +241,6 @@ public class IOMonitor2 {
DCP_Server_Query_Fetch_count += count;
DCP_Server_Query_Fetch_ns += elapsedTimeInNanosecond;
break;
- case DCP_SeriesScanOperator_hasNext:
- DCP_SeriesScanOperator_hasNext_count += count;
- DCP_SeriesScanOperator_hasNext_ns += elapsedTimeInNanosecond;
- break;
case DCP_A_GET_CHUNK_METADATAS:
DCP_A_GET_CHUNK_METADATAS_count += count;
DCP_A_GET_CHUNK_METADATAS_ns += elapsedTimeInNanosecond;
@@ -181,6 +257,46 @@ public class IOMonitor2 {
DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_count += count;
DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_ns += elapsedTimeInNanosecond;
break;
+ case SEARCH_ARRAY_a_verifBPTP:
+ SEARCH_ARRAY_a_verifBPTP_count += count;
+ SEARCH_ARRAY_a_verifBPTP_ns += elapsedTimeInNanosecond;
+ break;
+ case SEARCH_ARRAY_b_genFP:
+ SEARCH_ARRAY_b_genFP_count += count;
+ SEARCH_ARRAY_b_genFP_ns += elapsedTimeInNanosecond;
+ break;
+ case SEARCH_ARRAY_b_genLP:
+ SEARCH_ARRAY_b_genLP_count += count;
+ SEARCH_ARRAY_b_genLP_ns += elapsedTimeInNanosecond;
+ break;
+ case SEARCH_ARRAY_c_genBPTP:
+ SEARCH_ARRAY_c_genBPTP_count += count;
+ SEARCH_ARRAY_c_genBPTP_ns += elapsedTimeInNanosecond;
+ break;
+ case M4_LSM_INIT_LOAD_ALL_CHUNKMETADATAS:
+ M4_LSM_init_loadAllChunkMetadatas_count += count;
+ M4_LSM_init_loadAllChunkMetadatas_ns += elapsedTimeInNanosecond;
+ break;
+ case M4_LSM_MERGE_M4_TIME_SPAN:
+ M4_LSM_merge_M4_time_span_count += count;
+ M4_LSM_merge_M4_time_span_ns += elapsedTimeInNanosecond;
+ break;
+ case M4_LSM_FP:
+ M4_LSM_FP_count += count;
+ M4_LSM_FP_ns += elapsedTimeInNanosecond;
+ break;
+ case M4_LSM_LP:
+ M4_LSM_LP_count += count;
+ M4_LSM_LP_ns += elapsedTimeInNanosecond;
+ break;
+ case M4_LSM_BP:
+ M4_LSM_BP_count += count;
+ M4_LSM_BP_ns += elapsedTimeInNanosecond;
+ break;
+ case M4_LSM_TP:
+ M4_LSM_TP_count += count;
+ M4_LSM_TP_ns += elapsedTimeInNanosecond;
+ break;
default:
System.out.println("not supported operation type"); // this will not happen
break;
@@ -202,22 +318,59 @@ public class IOMonitor2 {
.append(DCP_Server_Query_Fetch_ns)
.append("\n");
stringBuilder
- .append("SeriesScanOperator_hasNext_ns")
+ .append("M4_LSM_init_loadAllChunkMetadatas_ns")
+ .append(",")
+ .append(M4_LSM_init_loadAllChunkMetadatas_ns)
+ .append("\n");
+ stringBuilder
+ .append("M4_LSM_merge_M4_time_span_ns")
+ .append(",")
+ .append(M4_LSM_merge_M4_time_span_ns)
+ .append("\n");
+ stringBuilder.append("M4_LSM_FP_ns").append(",").append(M4_LSM_FP_ns).append("\n");
+ stringBuilder.append("M4_LSM_LP_ns").append(",").append(M4_LSM_LP_ns).append("\n");
+ stringBuilder.append("M4_LSM_BP_ns").append(",").append(M4_LSM_BP_ns).append("\n");
+ stringBuilder.append("M4_LSM_TP_ns").append(",").append(M4_LSM_TP_ns).append("\n");
+ stringBuilder
+ .append("DCP_A_GET_CHUNK_METADATAS_ns")
+ .append(",")
+ .append(DCP_A_GET_CHUNK_METADATAS_ns)
+ .append("\n");
+ stringBuilder
+ .append("DCP_B_READ_MEM_CHUNK_ns")
.append(",")
- .append(DCP_SeriesScanOperator_hasNext_ns)
+ .append(DCP_B_READ_MEM_CHUNK_ns)
.append("\n");
- stringBuilder.append("A_ns").append(",").append(DCP_A_GET_CHUNK_METADATAS_ns).append("\n");
- stringBuilder.append("B_ns").append(",").append(DCP_B_READ_MEM_CHUNK_ns).append("\n");
stringBuilder
- .append("C_ns")
+ .append("DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_ns")
.append(",")
.append(DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_ns)
.append("\n");
stringBuilder
- .append("D_ns")
+ .append("DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_ns")
.append(",")
.append(DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_ns)
.append("\n");
+ stringBuilder
+ .append("SEARCH_ARRAY_a_verifBPTP_ns")
+ .append(",")
+ .append(SEARCH_ARRAY_a_verifBPTP_ns)
+ .append("\n");
+ stringBuilder
+ .append("SEARCH_ARRAY_b_genFP_ns")
+ .append(",")
+ .append(SEARCH_ARRAY_b_genFP_ns)
+ .append("\n");
+ stringBuilder
+ .append("SEARCH_ARRAY_b_genLP_ns")
+ .append(",")
+ .append(SEARCH_ARRAY_b_genLP_ns)
+ .append("\n");
+ stringBuilder
+ .append("SEARCH_ARRAY_c_genBPTP_ns")
+ .append(",")
+ .append(SEARCH_ARRAY_c_genBPTP_ns)
+ .append("\n");
stringBuilder
.append("Server_Query_Execute_cnt")
@@ -230,22 +383,59 @@ public class IOMonitor2 {
.append(DCP_Server_Query_Fetch_count)
.append("\n");
stringBuilder
- .append("SeriesScanOperator_hasNext_cnt")
+ .append("M4_LSM_init_loadAllChunkMetadatas_cnt")
+ .append(",")
+ .append(M4_LSM_init_loadAllChunkMetadatas_count)
+ .append("\n");
+ stringBuilder
+ .append("M4_LSM_merge_M4_time_span_cnt")
.append(",")
- .append(DCP_SeriesScanOperator_hasNext_count)
+ .append(M4_LSM_merge_M4_time_span_count)
.append("\n");
- stringBuilder.append("A_cnt").append(",").append(DCP_A_GET_CHUNK_METADATAS_count).append("\n");
- stringBuilder.append("B_cnt").append(",").append(DCP_B_READ_MEM_CHUNK_count).append("\n");
+ stringBuilder.append("M4_LSM_FP_cnt").append(",").append(M4_LSM_FP_count).append("\n");
+ stringBuilder.append("M4_LSM_LP_cnt").append(",").append(M4_LSM_LP_count).append("\n");
+ stringBuilder.append("M4_LSM_BP_cnt").append(",").append(M4_LSM_BP_count).append("\n");
+ stringBuilder.append("M4_LSM_TP_cnt").append(",").append(M4_LSM_TP_count).append("\n");
stringBuilder
- .append("C_cnt")
+ .append("DCP_A_GET_CHUNK_METADATAS_cnt")
+ .append(",")
+ .append(DCP_A_GET_CHUNK_METADATAS_count)
+ .append("\n");
+ stringBuilder
+ .append("DCP_B_READ_MEM_CHUNK_cnt")
+ .append(",")
+ .append(DCP_B_READ_MEM_CHUNK_count)
+ .append("\n");
+ stringBuilder
+ .append("DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_cnt")
.append(",")
.append(DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_count)
.append("\n");
stringBuilder
- .append("D_cnt")
+ .append("DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_cnt")
.append(",")
.append(DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_count)
.append("\n");
+ stringBuilder
+ .append("SEARCH_ARRAY_a_verifBPTP_cnt")
+ .append(",")
+ .append(SEARCH_ARRAY_a_verifBPTP_count)
+ .append("\n");
+ stringBuilder
+ .append("SEARCH_ARRAY_b_genFP_cnt")
+ .append(",")
+ .append(SEARCH_ARRAY_b_genFP_count)
+ .append("\n");
+ stringBuilder
+ .append("SEARCH_ARRAY_b_genLP_cnt")
+ .append(",")
+ .append(SEARCH_ARRAY_b_genLP_count)
+ .append("\n");
+ stringBuilder
+ .append("SEARCH_ARRAY_c_genBPTP_cnt")
+ .append(",")
+ .append(SEARCH_ARRAY_c_genBPTP_count)
+ .append("\n");
stringBuilder
.append("DCP_D_traversedPointNum")
.append(",")
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
index 420a32735ac..cc667ab3844 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
@@ -168,6 +168,7 @@ public class PageReader implements IPageReader {
}
public void updateBPTP(ChunkSuit4CPV chunkSuit4CPV) {
+ long start = System.nanoTime();
deleteCursor = 0; // TODO DEBUG
Statistics statistics = null;
switch (dataType) {
@@ -187,6 +188,7 @@ public class PageReader implements IPageReader {
int count = 0; // update here, not in statistics
for (int pos = chunkSuit4CPV.startPos; pos <= chunkSuit4CPV.endPos; pos++) {
IOMonitor.incPointsTravered();
+ IOMonitor2.DCP_D_traversedPointNum++;
long timestamp = timeBuffer.getLong(pos * 8);
switch (dataType) {
case INT64:
@@ -229,6 +231,7 @@ public class PageReader implements IPageReader {
} else {
chunkSuit4CPV.statistics.setCount(0); // otherwise count won't be zero
}
+ IOMonitor2.addMeasure(Operation.SEARCH_ARRAY_c_genBPTP, System.nanoTime() - start);
}
/** @return the returned BatchData may be empty, but never be null */