You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by le...@apache.org on 2023/02/03 15:50:42 UTC
[iotdb] 22/28: for pointsTraversed
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit a52a15d960b37bd369a3616e679e9e9bf87ad62f
Author: Lei Rui <10...@qq.com>
AuthorDate: Wed Feb 1 16:20:00 2023 +0800
for pointsTraversed
---
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 2 +-
.../apache/iotdb/db/engine/cache/ChunkCache.java | 2 +-
.../dataset/groupby/LocalGroupByExecutor4CPV.java | 6 ++++++
.../org/apache/iotdb/db/service/TSServiceImpl.java | 2 +-
.../org/apache/iotdb/db/utils/FileLoaderUtils.java | 2 +-
.../iotdb/tsfile/read/common/ChunkSuit4CPV.java | 9 +++++++++
.../iotdb/tsfile/read/common}/IOMonitor.java | 23 +++++++++++++++++++++-
.../iotdb/tsfile/read/reader/page/PageReader.java | 3 +++
8 files changed, 44 insertions(+), 5 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index f45461a069..cee0f8e24b 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -153,7 +153,7 @@ public class IoTDBConfig {
private int flushWalThreshold = 10000;
/** this variable set timestamp precision as millisecond, microsecond or nanosecond */
- private String timestampPrecision = "ms";
+ private String timestampPrecision = "ns";
/**
* The cycle when write ahead log is periodically forced to be written to disk(in milliseconds) If
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
index 02a1f5ab20..0e96e2a4cc 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
@@ -19,7 +19,6 @@
package org.apache.iotdb.db.engine.cache;
-import org.apache.iotdb.db.IOMonitor;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.query.control.FileReaderManager;
@@ -27,6 +26,7 @@ import org.apache.iotdb.db.utils.TestOnly;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Chunk;
+import org.apache.iotdb.tsfile.read.common.IOMonitor;
import org.apache.iotdb.tsfile.utils.RamUsageEstimator;
import com.github.benmanes.caffeine.cache.Caffeine;
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java
index d961bbf02a..568d9a9cac 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java
@@ -291,6 +291,7 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
.setDeleteIntervalList(chunkSuit4CPV.getChunkMetadata().getDeleteIntervalList());
}
// TODO chunk data read operation (c): get all data points
+ // (c) get all data points
chunkSuit4CPV.getPageReader().updateBPTP(chunkSuit4CPV);
// TODO check if empty
if (chunkSuit4CPV.statistics.getCount() == 0) {
@@ -376,6 +377,7 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
// WHICH WILL INTRODUCE BUGS!
chunkSuit4CPV.setPageReader(pageReader);
}
+ // (a) check existence of data point at a timestamp:
isUpdate = chunkSuit4CPV.checkIfExist(candidateTimestamp);
if (isUpdate) { // 提前结束对overlaps块的scan，因为已经找到一个update点证明candidate失效
break;
@@ -492,6 +494,7 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
.setDeleteIntervalList(chunkSuit4CPV.getChunkMetadata().getDeleteIntervalList());
}
// TODO chunk data read operation (c): get all data points
+ // (c) get all data points
chunkSuit4CPV.getPageReader().updateBPTP(chunkSuit4CPV);
// TODO check if empty
if (chunkSuit4CPV.statistics.getCount() == 0) {
@@ -582,6 +585,7 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
// WHICH WILL INTRODUCE BUGS!
chunkSuit4CPV.setPageReader(pageReader);
}
+ // (a) check existence of data point at a timestamp
isUpdate = chunkSuit4CPV.checkIfExist(candidateTimestamp);
if (isUpdate) { // 提前结束对overlaps块的scan，因为已经找到一个update点证明candidate失效
break;
@@ -670,6 +674,7 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
susp_candidate.setPageReader(pageReader);
}
// TODO update FP equal to or after statistics.getEndTime
+ // (b) get the closest data point after or before a timestamp
susp_candidate.updateFPwithTheClosetPointEqualOrAfter(
susp_candidate.getStatistics().getStartTime()); // TODO DEBUG
susp_candidate.setLazyLoad(false); // DO NOT FORGET THIS!!!
@@ -778,6 +783,7 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
susp_candidate.setPageReader(pageReader);
}
// TODO update FP equal to or after statistics.getEndTime
+ // (b) get the closest data point after or before a timestamp
susp_candidate.updateLPwithTheClosetPointEqualOrBefore(
susp_candidate.getStatistics().getEndTime()); // TODO DEBUG
susp_candidate.setLazyLoad(false); // TODO DO NOT FORGET THIS!!!
diff --git a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
index 4b6db1bbaf..00c01bb37d 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
@@ -18,7 +18,6 @@
*/
package org.apache.iotdb.db.service;
-import org.apache.iotdb.db.IOMonitor;
import org.apache.iotdb.db.auth.AuthException;
import org.apache.iotdb.db.auth.AuthorityChecker;
import org.apache.iotdb.db.auth.authorizer.BasicAuthorizer;
@@ -128,6 +127,7 @@ import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.read.common.IOMonitor;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet;
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
index 7d2b9dc5cd..fd7b6f4cb1 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
@@ -18,7 +18,6 @@
*/
package org.apache.iotdb.db.utils;
-import org.apache.iotdb.db.IOMonitor;
import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
@@ -34,6 +33,7 @@ import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.file.metadata.TsFileMetadata;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Chunk;
+import org.apache.iotdb.tsfile.read.common.IOMonitor;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.controller.IChunkLoader;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
index 55116a7619..d61d14e193 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
@@ -254,13 +254,16 @@ public class ChunkSuit4CPV {
if (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
while (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
estimatedPos++;
+ IOMonitor.incPointsTravered();
}
} else if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
while (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
estimatedPos--;
+ IOMonitor.incPointsTravered();
}
if (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
estimatedPos++;
+ IOMonitor.incPointsTravered();
} // else equal
} // else equal
this.startPos = estimatedPos; // note this
@@ -315,13 +318,16 @@ public class ChunkSuit4CPV {
if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
while (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
estimatedPos--;
+ IOMonitor.incPointsTravered();
}
} else if (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
while (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
estimatedPos++;
+ IOMonitor.incPointsTravered();
}
if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
estimatedPos--;
+ IOMonitor.incPointsTravered();
} // else equal
} // else equal
this.endPos = estimatedPos; // note this
@@ -375,13 +381,16 @@ public class ChunkSuit4CPV {
if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
while (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
estimatedPos--;
+ IOMonitor.incPointsTravered();
}
} else if (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
while (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
estimatedPos++;
+ IOMonitor.incPointsTravered();
}
if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
estimatedPos--;
+ IOMonitor.incPointsTravered();
} // else equal
} // else equal
diff --git a/server/src/main/java/org/apache/iotdb/db/IOMonitor.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor.java
similarity index 89%
rename from server/src/main/java/org/apache/iotdb/db/IOMonitor.java
rename to tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor.java
index 698aa10cb3..3bfed74089 100644
--- a/server/src/main/java/org/apache/iotdb/db/IOMonitor.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.db;
+package org.apache.iotdb.tsfile.read.common;
import java.util.ArrayList;
import java.util.List;
@@ -34,6 +34,8 @@ public class IOMonitor {
public static long totalTime;
public static String sql;
+ public static long pointsTraversed;
+
public static List<Long> metaIOTimes = new ArrayList<>();
public static List<Long> dataIOTimes = new ArrayList<>();
public static List<Long> readMemChunkTimes = new ArrayList<>();
@@ -44,6 +46,8 @@ public class IOMonitor {
public static List<String> sqls = new ArrayList<>();
public static List<Long> totalTimes = new ArrayList<>();
+ public static List<Long> pointsTraversedList = new ArrayList<>();
+
public static boolean isSet = false;
public static void incReadMemChunkTime(long v) {
@@ -58,11 +62,20 @@ public class IOMonitor {
readMemChunkNum = 0;
}
+ public static void resetPointsTraversed() {
+ pointsTraversedList.add(pointsTraversed);
+ pointsTraversed = 0;
+ }
+
public static void incMeta(long v) {
metaIOTime += v;
metaIONum++;
}
+ public static void incPointsTravered() {
+ pointsTraversed++;
+ }
+
private static void resetMeta() {
metaIOTimes.add(metaIOTime);
metaIONums.add(metaIONum);
@@ -109,6 +122,7 @@ public class IOMonitor {
resetReadMemChunkTime();
sqls.add(sql);
sql = null;
+ resetPointsTraversed();
}
// private static double getAvg(List<Long> vals) {
@@ -148,6 +162,9 @@ public class IOMonitor {
dataIOTimes.clear();
sqls.clear();
totalTimes.clear();
+
+ pointsTraversedList.clear();
+ pointsTraversed = 0;
}
public static void finish() {
@@ -177,6 +194,8 @@ public class IOMonitor {
+ readMemChunkNums.get(i)
+ "\t total: \t"
+ totalTimes.get(i)
+ + "\t pointsTraversed: \t"
+ + pointsTraversedList.get(i)
+ "\n";
}
ret +=
@@ -194,6 +213,8 @@ public class IOMonitor {
+ getSumInteger(readMemChunkNums)
+ "\t avg total time: \t"
+ getSumLong(totalTimes)
+ + "\t points traversed: \t"
+ + getSumLong(pointsTraversedList)
+ "\t isSet: \t"
+ isSet;
return ret;
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
index cd67765f45..f3e300a7ea 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
@@ -30,6 +30,7 @@ import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.common.BatchData;
import org.apache.iotdb.tsfile.read.common.BatchDataFactory;
import org.apache.iotdb.tsfile.read.common.ChunkSuit4CPV;
+import org.apache.iotdb.tsfile.read.common.IOMonitor;
import org.apache.iotdb.tsfile.read.common.TimeRange;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.read.filter.operator.AndFilter;
@@ -136,6 +137,7 @@ public class PageReader implements IPageReader {
// TODO update FP,LP with the help of stepRegress index. BP/TP not update here.
int FP_pos = -1;
int LP_pos = -1;
+ // (b) get the closest data point after or before a timestamp
if (leftEndIncluded > chunkSuit4CPV.statistics.getStartTime()) {
FP_pos = chunkSuit4CPV.updateFPwithTheClosetPointEqualOrAfter(leftEndIncluded);
}
@@ -182,6 +184,7 @@ public class PageReader implements IPageReader {
// [startPos,endPos] definitely for curStartTime interval, thanks to split4CPV
int count = 0; // update here, not in statistics
for (int pos = chunkSuit4CPV.startPos; pos <= chunkSuit4CPV.endPos; pos++) {
+ IOMonitor.incPointsTravered();
long timestamp = timeBuffer.getLong(pos * 8);
switch (dataType) {
case INT64: