You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by le...@apache.org on 2023/07/09 17:17:26 UTC
[iotdb] branch research/M4-visualization updated (79e4e9266d0 -> 359389a85e8)
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a change to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 79e4e9266d0 update readme
new 5cc69fee9a0 fix sketch script
new 45a63a8139e improve sketch tool
new 052fd37301e add IOMonitor2 and add metrics for raw data query
new 1ded315f412 add license
new b634e42dd1d modify python-iotdb-connector add execute_finish
new bf06145984b add log info for plan type
new dbeeaccfecb add log info for plan type
new bf15b534d85 add m4_chunk_metadata log debug for visualization
new aeaafe1b7e4 add M4_QUERY_PARAM for log debug for visualization
new 55512608866 finish debug
new b24dbd33b03 [important] four-layer metrics design done, from client, server execute&fetch, dataset&executor, basic ops ABCD, improved D search array 4 functions
new 76c0ff69420 fix
new 7ed7a87606f modify iomonitor2 print
new 694409ed1f1 [important]add metric mapping from level3 to level4
new 4bb3a52976a add print
new b9d14edd76b deprecate old IOMonitor
new 4173b42565b modify print
new cdc4fc55c33 fix session fetchsize bug
new 5d1b38cf49a fix python-iotdb-connector with MIN_MAX self-defined datatype
new f3aefc3a74e detail dataSetType info
new 48bfd3d54d1 M4 udf add test
new 132ab5fe952 add udf time window TW
new 4dbc88e3f27 add config use_Statistics
new 8fd0e7e2081 update dataSetType
new 32254812dd3 exportCsv notimeout and fix
new 3d77172a060 fix
new dca90f4782e fix index out of bounds
new eb4a6ac5cf0 add tests in SDTEncoderTest
new 984bfebaf7d add test
new 19cc917b5b8 add test
new 3aa5d8d7cc0 add tests, confirm the best e
new be822bc3a67 add
new da7fa9210a0 save first and last points
new c7dfd4887e6 ni
new 49cd2898b93 runnable
new df9c9417648 rename
new 89c9cb7cb11 add metric
new 65099ff4384 fix metric
new b2956368b60 add log
new fcd4555e506 add log
new 960357c01ed debu
new d400a425893 fix
new 8d2f71561a0 fix
new c5fa371b3e2 fix
new 3992f77d0b1 add
new 359389a85e8 update readme
The 46 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
README.md | 2 +-
client-py/iotdb/Session.py | 8 +
client-py/iotdb/utils/Field.py | 24 +-
client-py/iotdb/utils/IoTDBConstants.py | 7 +
client-py/iotdb/utils/IoTDBRpcDataSet.py | 6 +-
client-py/iotdb/utils/SessionDataSet.py | 13 +-
.../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 7 +
.../resources/conf/iotdb-engine.properties | 6 +-
server/src/assembly/resources/conf/logback.xml | 20 +
.../tools/tsfileToolSet/print-tsfile-sketch.sh | 4 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 22 +-
.../apache/iotdb/db/engine/cache/ChunkCache.java | 9 +-
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 7 +-
.../groupby/GroupByWithoutValueFilterDataSet.java | 30 +-
.../dataset/groupby/LocalGroupByExecutor.java | 1 +
.../dataset/groupby/LocalGroupByExecutor4CPV.java | 172 ++++-
.../query/reader/series/SeriesAggregateReader.java | 10 +
.../db/query/udf/builtin/BuiltinFunction.java | 1 +
.../apache/iotdb/db/query/udf/builtin/UDTFM4.java | 315 +++++++++
.../iotdb/db/query/udf/builtin/UDTFM4MAC.java | 202 +++---
.../iotdb/db/rescon/PrimitiveArrayManager.java | 4 +
.../org/apache/iotdb/db/service/TSServiceImpl.java | 52 +-
.../apache/iotdb/db/tools/TsFileSketchTool.java | 89 ++-
.../org/apache/iotdb/db/utils/FileLoaderUtils.java | 19 +-
.../apache/iotdb/db/integration/m4/MyTest1.java | 8 +-
.../apache/iotdb/db/integration/m4/MyTest2.java | 6 +-
.../apache/iotdb/db/integration/m4/MyTest3.java | 64 +-
.../apache/iotdb/db/integration/m4/MyTest4.java | 6 +-
.../apache/iotdb/db/integration/m4/MyTest5.java | 281 ++++++++
.../m4/{MyTest1.java => MyTmpTest.java} | 65 +-
.../apache/iotdb/session/SessionConnection.java | 6 +-
.../org/apache/iotdb/session/SessionDataSet.java | 57 ++
.../iotdb/tsfile/common/conf/TSFileConfig.java | 30 +-
.../iotdb/tsfile/common/conf/TSFileDescriptor.java | 10 +-
.../encoding/decoder/DeltaBinaryDecoder.java | 2 +-
.../iotdb/tsfile/file/header/PageHeader.java | 3 +-
.../iotdb/tsfile/file/metadata/ChunkMetadata.java | 2 +-
.../tsfile/file/metadata/TimeseriesMetadata.java | 6 +-
.../file/metadata/statistics/Statistics.java | 256 ++++++-
.../file/metadata/statistics/StepRegress.java | 14 +-
.../file/metadata/statistics/ValueIndex.java | 121 ++++
.../iotdb/tsfile/read/common/ChunkSuit4CPV.java | 90 ++-
.../apache/iotdb/tsfile/read/common/IOMonitor.java | 445 ++++++------
.../iotdb/tsfile/read/common/IOMonitor2.java | 773 +++++++++++++++++++++
.../{ExceptionBatchData.java => ValuePoint.java} | 20 +-
.../iotdb/tsfile/read/filter/GroupByFilter.java | 4 +
.../tsfile/read/reader/chunk/ChunkReader.java | 5 +
.../iotdb/tsfile/read/reader/page/PageReader.java | 438 +++++++++++-
.../iotdb/tsfile/encoding/SDTEncoderTest.java | 373 ++++++++++
49 files changed, 3559 insertions(+), 556 deletions(-)
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/builtin/UDTFM4.java
create mode 100644 server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest5.java
copy server/src/test/java/org/apache/iotdb/db/integration/m4/{MyTest1.java => MyTmpTest.java} (95%)
create mode 100644 tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/ValueIndex.java
create mode 100644 tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
copy tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/{ExceptionBatchData.java => ValuePoint.java} (68%)
[iotdb] 26/46: fix
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 3d77172a060b553b868fb6d4cd0b9f0f1faa36d7
Author: Lei Rui <10...@qq.com>
AuthorDate: Tue Jun 20 21:46:52 2023 +0800
fix
---
cli/src/main/java/org/apache/iotdb/tool/ExportCsv.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/cli/src/main/java/org/apache/iotdb/tool/ExportCsv.java b/cli/src/main/java/org/apache/iotdb/tool/ExportCsv.java
index 23cebcd6147..d645447db71 100644
--- a/cli/src/main/java/org/apache/iotdb/tool/ExportCsv.java
+++ b/cli/src/main/java/org/apache/iotdb/tool/ExportCsv.java
@@ -288,7 +288,7 @@ public class ExportCsv extends AbstractCsvTool {
private static void dumpResult(String sql, int index) {
final String path = targetDirectory + targetFile + index + ".csv";
try {
- SessionDataSet sessionDataSet = session.executeQueryStatement(sql, 0);
+ SessionDataSet sessionDataSet = session.executeQueryStatement(sql, 10000);
writeCsvFile(sessionDataSet, path);
sessionDataSet.closeOperationHandle();
System.out.println("Export completely!");
[iotdb] 14/46: [important]add metric mapping from level3 to level4
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 694409ed1f1ce2db0128b920931689209ae3a0fe
Author: Lei Rui <10...@qq.com>
AuthorDate: Wed Jun 14 17:49:44 2023 +0800
[important]add metric mapping from level3 to level4
---
.../dataset/groupby/LocalGroupByExecutor4CPV.java | 6 +
.../iotdb/tsfile/read/common/IOMonitor2.java | 440 ++++++++++++++-------
2 files changed, 299 insertions(+), 147 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java
index 92026420e17..55f7629159a 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java
@@ -240,6 +240,8 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
private void getCurrentChunkListFromFutureChunkList(
long curStartTime, long curEndTime, long startTime, long endTime, long interval)
throws IOException {
+ IOMonitor2.M4_LSM_status = Operation.M4_LSM_MERGE_M4_TIME_SPAN;
+
// empty currentChunkList
currentChunkList = new ArrayList<>();
@@ -348,6 +350,7 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
long interval,
long curStartTime)
throws IOException {
+ IOMonitor2.M4_LSM_status = Operation.M4_LSM_BP;
// check size>0 because after updateBPTP because empty ChunkSuit4CPV will be removed from
// currentChunkList
while (currentChunkList.size() > 0) { // loop 1
@@ -542,6 +545,7 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
long interval,
long curStartTime)
throws IOException {
+ IOMonitor2.M4_LSM_status = Operation.M4_LSM_TP;
// check size>0 because after updateBPTP empty ChunkSuit4CPV will be removed from
// currentChunkList
while (currentChunkList.size() > 0) { // loop 1
@@ -742,6 +746,7 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
long interval,
long curStartTime)
throws IOException {
+ IOMonitor2.M4_LSM_status = Operation.M4_LSM_FP;
while (currentChunkList.size() > 0) { // loop 1
// sorted by startTime and version, find FP candidate
currentChunkList.sort(
@@ -852,6 +857,7 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
long interval,
long curStartTime)
throws IOException {
+ IOMonitor2.M4_LSM_status = Operation.M4_LSM_LP;
while (currentChunkList.size() > 0) { // loop 1
// sorted by endTime and version, find LP candidate
currentChunkList.sort(
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
index 496bf9ab678..90bccfaf4ca 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
@@ -58,66 +58,128 @@ public class IOMonitor2 {
}
}
- public static DataSetType dataSetType = DataSetType.NONE;
+ public static int DCP_Server_Query_Execute_count = 0; // level 2
+ public static long DCP_Server_Query_Execute_ns = 0; // level 2
+
+ public static int DCP_Server_Query_Fetch_count = 0; // level 2
+ public static long DCP_Server_Query_Fetch_ns = 0; // level 2
+
+ public static DataSetType dataSetType = DataSetType.NONE; // level 3
+
+ public static int M4_LSM_init_loadAllChunkMetadatas_count = 0; // level 3
+ public static long M4_LSM_init_loadAllChunkMetadatas_ns = 0; // level 3
+
+ public static int M4_LSM_merge_M4_time_span_count = 0; // level 3
+ public static long M4_LSM_merge_M4_time_span_ns = 0; // level 3
+
+ public static int M4_LSM_FP_count = 0; // level 3
+ public static long M4_LSM_FP_ns = 0; // level 3
+
+ public static int M4_LSM_LP_count = 0; // level 3
+ public static long M4_LSM_LP_ns = 0; // level 3
+
+ public static int M4_LSM_BP_count = 0; // level 3
+ public static long M4_LSM_BP_ns = 0; // level 3
+
+ public static int M4_LSM_TP_count = 0; // level 3
+ public static long M4_LSM_TP_ns = 0; // level 3
+
+ public static int DCP_A_GET_CHUNK_METADATAS_count = 0; // level 4
+ public static long DCP_A_GET_CHUNK_METADATAS_ns = 0; // level 4
+
+ public static int DCP_B_READ_MEM_CHUNK_count = 0; // level 4
+ public static long DCP_B_READ_MEM_CHUNK_ns = 0; // level 4
+
+ public static int DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_count = 0; // level 4
+ public static long DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_ns = 0; // level 4
+
+ public static long DCP_D_traversedPointNum = 0; // level 4
+ public static int DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_count = 0; // level 4
+ public static long DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_ns = 0; // level 4
+
+ public static int SEARCH_ARRAY_a_verifBPTP_count = 0; // level 4
+ public static long SEARCH_ARRAY_a_verifBPTP_ns = 0; // level 4
+
+ public static int SEARCH_ARRAY_b_genFP_count = 0; // level 4
+ public static long SEARCH_ARRAY_b_genFP_ns = 0; // level 4
+
+ public static int SEARCH_ARRAY_b_genLP_count = 0; // level 4
+ public static long SEARCH_ARRAY_b_genLP_ns = 0; // level 4
+
+ public static int SEARCH_ARRAY_c_genBPTP_count = 0; // level 4
+ public static long SEARCH_ARRAY_c_genBPTP_ns = 0; // level 4
+
+ public static Operation M4_LSM_status =
+ null; // for counting the number of calling BCD by each step of M4-LSM
+ public static int M4_LSM_merge_M4_time_span_B_READ_MEM_CHUNK_cnt = 0; // map from level 3 to 4
+ public static int M4_LSM_merge_M4_time_span_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_cnt =
+ 0; // map from level 3 to 4
+ public static int M4_LSM_merge_M4_time_span_SEARCH_ARRAY_a_verifBPTP_cnt =
+ 0; // map from level 3 to 4
+ public static int M4_LSM_merge_M4_time_span_SEARCH_ARRAY_b_genFP_cnt = 0; // map from level 3 to 4
+ public static int M4_LSM_merge_M4_time_span_SEARCH_ARRAY_b_genLP_cnt = 0; // map from level 3 to 4
+ public static int M4_LSM_merge_M4_time_span_SEARCH_ARRAY_c_genBPTP_cnt =
+ 0; // map from level 3 to 4
+ public static int M4_LSM_FP_B_READ_MEM_CHUNK_cnt = 0; // map from level 3 to 4
+ public static int M4_LSM_FP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_cnt =
+ 0; // map from level 3 to 4
+ public static int M4_LSM_FP_SEARCH_ARRAY_a_verifBPTP_cnt = 0; // map from level 3 to 4
+ public static int M4_LSM_FP_SEARCH_ARRAY_b_genFP_cnt = 0; // map from level 3 to 4
+ public static int M4_LSM_FP_SEARCH_ARRAY_b_genLP_cnt = 0; // map from level 3 to 4
+ public static int M4_LSM_FP_SEARCH_ARRAY_c_genBPTP_cnt = 0; // map from level 3 to 4
+ public static int M4_LSM_LP_B_READ_MEM_CHUNK_cnt = 0; // map from level 3 to 4
+ public static int M4_LSM_LP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_cnt =
+ 0; // map from level 3 to 4
+ public static int M4_LSM_LP_SEARCH_ARRAY_a_verifBPTP_cnt = 0; // map from level 3 to 4
+ public static int M4_LSM_LP_SEARCH_ARRAY_b_genFP_cnt = 0; // map from level 3 to 4
+ public static int M4_LSM_LP_SEARCH_ARRAY_b_genLP_cnt = 0; // map from level 3 to 4
+ public static int M4_LSM_LP_SEARCH_ARRAY_c_genBPTP_cnt = 0; // map from level 3 to 4
+ public static int M4_LSM_BP_B_READ_MEM_CHUNK_cnt = 0; // map from level 3 to 4
+ public static int M4_LSM_BP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_cnt =
+ 0; // map from level 3 to 4
+ public static int M4_LSM_BP_SEARCH_ARRAY_a_verifBPTP_cnt = 0; // map from level 3 to 4
+ public static int M4_LSM_BP_SEARCH_ARRAY_b_genFP_cnt = 0; // map from level 3 to 4
+ public static int M4_LSM_BP_SEARCH_ARRAY_b_genLP_cnt = 0; // map from level 3 to 4
+ public static int M4_LSM_BP_SEARCH_ARRAY_c_genBPTP_cnt = 0; // map from level 3 to 4
+ public static int M4_LSM_TP_B_READ_MEM_CHUNK_cnt = 0; // map from level 3 to 4
+ public static int M4_LSM_TP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_cnt =
+ 0; // map from level 3 to 4
+ public static int M4_LSM_TP_SEARCH_ARRAY_a_verifBPTP_cnt = 0; // map from level 3 to 4
+ public static int M4_LSM_TP_SEARCH_ARRAY_b_genFP_cnt = 0; // map from level 3 to 4
+ public static int M4_LSM_TP_SEARCH_ARRAY_b_genLP_cnt = 0; // map from level 3 to 4
+ public static int M4_LSM_TP_SEARCH_ARRAY_c_genBPTP_cnt = 0; // map from level 3 to 4
- public static int DCP_Server_Query_Execute_count = 0;
- public static long DCP_Server_Query_Execute_ns = 0;
-
- public static int DCP_Server_Query_Fetch_count = 0;
- public static long DCP_Server_Query_Fetch_ns = 0;
-
- public static int DCP_A_GET_CHUNK_METADATAS_count = 0;
- public static long DCP_A_GET_CHUNK_METADATAS_ns = 0;
-
- public static int DCP_B_READ_MEM_CHUNK_count = 0;
- public static long DCP_B_READ_MEM_CHUNK_ns = 0;
-
- public static int DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_count = 0;
- public static long DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_ns = 0;
-
- public static long DCP_D_traversedPointNum = 0;
- public static int DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_count = 0;
- public static long DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_ns = 0;
-
- public static int SEARCH_ARRAY_a_verifBPTP_count = 0;
- public static long SEARCH_ARRAY_a_verifBPTP_ns = 0;
-
- public static int SEARCH_ARRAY_b_genFP_count = 0;
- public static long SEARCH_ARRAY_b_genFP_ns = 0;
-
- public static int SEARCH_ARRAY_b_genLP_count = 0;
- public static long SEARCH_ARRAY_b_genLP_ns = 0;
-
- public static int SEARCH_ARRAY_c_genBPTP_count = 0;
- public static long SEARCH_ARRAY_c_genBPTP_ns = 0;
-
- public static int M4_LSM_init_loadAllChunkMetadatas_count = 0;
- public static long M4_LSM_init_loadAllChunkMetadatas_ns = 0;
+ private static void reset() {
+ // level 1 is client elapsed time, not measured by the server side
+ /** level 2: server execute & fetch */
+ DCP_Server_Query_Execute_count = 0;
+ DCP_Server_Query_Execute_ns = 0;
- public static int M4_LSM_merge_M4_time_span_count = 0;
- public static long M4_LSM_merge_M4_time_span_ns = 0;
+ DCP_Server_Query_Fetch_count = 0;
+ DCP_Server_Query_Fetch_ns = 0;
- public static int M4_LSM_FP_count = 0;
- public static long M4_LSM_FP_ns = 0;
+ /** level 3 */
+ dataSetType = DataSetType.NONE;
- public static int M4_LSM_LP_count = 0;
- public static long M4_LSM_LP_ns = 0;
+ M4_LSM_init_loadAllChunkMetadatas_count = 0;
+ M4_LSM_init_loadAllChunkMetadatas_ns = 0;
- public static int M4_LSM_BP_count = 0;
- public static long M4_LSM_BP_ns = 0;
+ M4_LSM_merge_M4_time_span_count = 0;
+ M4_LSM_merge_M4_time_span_ns = 0;
- public static int M4_LSM_TP_count = 0;
- public static long M4_LSM_TP_ns = 0;
+ M4_LSM_FP_count = 0;
+ M4_LSM_FP_ns = 0;
- private static void reset() {
- dataSetType = DataSetType.NONE;
+ M4_LSM_LP_count = 0;
+ M4_LSM_LP_ns = 0;
- DCP_Server_Query_Execute_count = 0;
- DCP_Server_Query_Execute_ns = 0;
+ M4_LSM_BP_count = 0;
+ M4_LSM_BP_ns = 0;
- DCP_Server_Query_Fetch_count = 0;
- DCP_Server_Query_Fetch_ns = 0;
+ M4_LSM_TP_count = 0;
+ M4_LSM_TP_ns = 0;
+ /** level 4 */
DCP_A_GET_CHUNK_METADATAS_count = 0;
DCP_A_GET_CHUNK_METADATAS_ns = 0;
@@ -128,6 +190,7 @@ public class IOMonitor2 {
DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_ns = 0;
DCP_D_traversedPointNum = 0;
+
DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_count = 0;
DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_ns = 0;
@@ -140,23 +203,43 @@ public class IOMonitor2 {
SEARCH_ARRAY_c_genBPTP_count = 0;
SEARCH_ARRAY_c_genBPTP_ns = 0;
- M4_LSM_init_loadAllChunkMetadatas_count = 0;
- M4_LSM_init_loadAllChunkMetadatas_ns = 0;
-
- M4_LSM_merge_M4_time_span_count = 0;
- M4_LSM_merge_M4_time_span_ns = 0;
-
- M4_LSM_FP_count = 0;
- M4_LSM_FP_ns = 0;
-
- M4_LSM_LP_count = 0;
- M4_LSM_LP_ns = 0;
-
- M4_LSM_BP_count = 0;
- M4_LSM_BP_ns = 0;
-
- M4_LSM_TP_count = 0;
- M4_LSM_TP_ns = 0;
+ /** map from level 3 to level 4 */
+ M4_LSM_status = null;
+ M4_LSM_merge_M4_time_span_B_READ_MEM_CHUNK_cnt =
+ 0; // for counting the number of calling BCD by each step of M4-LSM
+ M4_LSM_merge_M4_time_span_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_cnt = 0;
+ M4_LSM_merge_M4_time_span_SEARCH_ARRAY_a_verifBPTP_cnt = 0;
+ M4_LSM_merge_M4_time_span_SEARCH_ARRAY_b_genFP_cnt = 0;
+ M4_LSM_merge_M4_time_span_SEARCH_ARRAY_b_genLP_cnt = 0;
+ M4_LSM_merge_M4_time_span_SEARCH_ARRAY_c_genBPTP_cnt = 0;
+ M4_LSM_FP_B_READ_MEM_CHUNK_cnt =
+ 0; // for counting the number of calling BCD by each step of M4-LSM
+ M4_LSM_FP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_cnt = 0;
+ M4_LSM_FP_SEARCH_ARRAY_a_verifBPTP_cnt = 0;
+ M4_LSM_FP_SEARCH_ARRAY_b_genFP_cnt = 0;
+ M4_LSM_FP_SEARCH_ARRAY_b_genLP_cnt = 0;
+ M4_LSM_FP_SEARCH_ARRAY_c_genBPTP_cnt = 0;
+ M4_LSM_LP_B_READ_MEM_CHUNK_cnt =
+ 0; // for counting the number of calling BCD by each step of M4-LSM
+ M4_LSM_LP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_cnt = 0;
+ M4_LSM_LP_SEARCH_ARRAY_a_verifBPTP_cnt = 0;
+ M4_LSM_LP_SEARCH_ARRAY_b_genFP_cnt = 0;
+ M4_LSM_LP_SEARCH_ARRAY_b_genLP_cnt = 0;
+ M4_LSM_LP_SEARCH_ARRAY_c_genBPTP_cnt = 0;
+ M4_LSM_BP_B_READ_MEM_CHUNK_cnt =
+ 0; // for counting the number of calling BCD by each step of M4-LSM
+ M4_LSM_BP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_cnt = 0;
+ M4_LSM_BP_SEARCH_ARRAY_a_verifBPTP_cnt = 0;
+ M4_LSM_BP_SEARCH_ARRAY_b_genFP_cnt = 0;
+ M4_LSM_BP_SEARCH_ARRAY_b_genLP_cnt = 0;
+ M4_LSM_BP_SEARCH_ARRAY_c_genBPTP_cnt = 0;
+ M4_LSM_TP_B_READ_MEM_CHUNK_cnt =
+ 0; // for counting the number of calling BCD by each step of M4-LSM
+ M4_LSM_TP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_cnt = 0;
+ M4_LSM_TP_SEARCH_ARRAY_a_verifBPTP_cnt = 0;
+ M4_LSM_TP_SEARCH_ARRAY_b_genFP_cnt = 0;
+ M4_LSM_TP_SEARCH_ARRAY_b_genLP_cnt = 0;
+ M4_LSM_TP_SEARCH_ARRAY_c_genBPTP_cnt = 0;
}
public static void addMeasure(Operation operation, long elapsedTimeInNanosecond) {
@@ -169,38 +252,6 @@ public class IOMonitor2 {
DCP_Server_Query_Fetch_count++;
DCP_Server_Query_Fetch_ns += elapsedTimeInNanosecond;
break;
- case DCP_A_GET_CHUNK_METADATAS:
- DCP_A_GET_CHUNK_METADATAS_count++;
- DCP_A_GET_CHUNK_METADATAS_ns += elapsedTimeInNanosecond;
- break;
- case DCP_B_READ_MEM_CHUNK:
- DCP_B_READ_MEM_CHUNK_count++;
- DCP_B_READ_MEM_CHUNK_ns += elapsedTimeInNanosecond;
- break;
- case DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA:
- DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_count++;
- DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_ns += elapsedTimeInNanosecond;
- break;
- case DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS:
- DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_count++;
- DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_ns += elapsedTimeInNanosecond;
- break;
- case SEARCH_ARRAY_a_verifBPTP:
- SEARCH_ARRAY_a_verifBPTP_count++;
- SEARCH_ARRAY_a_verifBPTP_ns += elapsedTimeInNanosecond;
- break;
- case SEARCH_ARRAY_b_genFP:
- SEARCH_ARRAY_b_genFP_count++;
- SEARCH_ARRAY_b_genFP_ns += elapsedTimeInNanosecond;
- break;
- case SEARCH_ARRAY_b_genLP:
- SEARCH_ARRAY_b_genLP_count++;
- SEARCH_ARRAY_b_genLP_ns += elapsedTimeInNanosecond;
- break;
- case SEARCH_ARRAY_c_genBPTP:
- SEARCH_ARRAY_c_genBPTP_count++;
- SEARCH_ARRAY_c_genBPTP_ns += elapsedTimeInNanosecond;
- break;
case M4_LSM_INIT_LOAD_ALL_CHUNKMETADATAS:
M4_LSM_init_loadAllChunkMetadatas_count++;
M4_LSM_init_loadAllChunkMetadatas_ns += elapsedTimeInNanosecond;
@@ -225,77 +276,172 @@ public class IOMonitor2 {
M4_LSM_TP_count++;
M4_LSM_TP_ns += elapsedTimeInNanosecond;
break;
- default:
- System.out.println("not supported operation type"); // this will not happen
- break;
- }
- }
-
- public static void addMeasure(Operation operation, long elapsedTimeInNanosecond, int count) {
- switch (operation) {
- case DCP_Server_Query_Execute:
- DCP_Server_Query_Execute_count += count;
- DCP_Server_Query_Execute_ns += elapsedTimeInNanosecond;
- break;
- case DCP_Server_Query_Fetch:
- DCP_Server_Query_Fetch_count += count;
- DCP_Server_Query_Fetch_ns += elapsedTimeInNanosecond;
- break;
case DCP_A_GET_CHUNK_METADATAS:
- DCP_A_GET_CHUNK_METADATAS_count += count;
+ DCP_A_GET_CHUNK_METADATAS_count++;
DCP_A_GET_CHUNK_METADATAS_ns += elapsedTimeInNanosecond;
break;
case DCP_B_READ_MEM_CHUNK:
- DCP_B_READ_MEM_CHUNK_count += count;
+ DCP_B_READ_MEM_CHUNK_count++;
DCP_B_READ_MEM_CHUNK_ns += elapsedTimeInNanosecond;
+ if (M4_LSM_status != null) {
+ switch (M4_LSM_status) {
+ case M4_LSM_MERGE_M4_TIME_SPAN:
+ M4_LSM_merge_M4_time_span_B_READ_MEM_CHUNK_cnt++;
+ break;
+ case M4_LSM_FP:
+ M4_LSM_FP_B_READ_MEM_CHUNK_cnt++;
+ break;
+ case M4_LSM_LP:
+ M4_LSM_LP_B_READ_MEM_CHUNK_cnt++;
+ break;
+ case M4_LSM_BP:
+ M4_LSM_BP_B_READ_MEM_CHUNK_cnt++;
+ break;
+ case M4_LSM_TP:
+ M4_LSM_TP_B_READ_MEM_CHUNK_cnt++;
+ break;
+ default:
+ System.out.println("unsupported M4_LSM_status!");
+ break;
+ }
+ }
break;
case DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA:
- DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_count += count;
+ DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_count++;
DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_ns += elapsedTimeInNanosecond;
+ if (M4_LSM_status != null) {
+ // this actually can be omitted because the number is equal to B as chunk = page in this
+ // case
+ switch (M4_LSM_status) {
+ case M4_LSM_MERGE_M4_TIME_SPAN:
+ M4_LSM_merge_M4_time_span_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_cnt++;
+ break;
+ case M4_LSM_FP:
+ M4_LSM_FP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_cnt++;
+ break;
+ case M4_LSM_LP:
+ M4_LSM_LP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_cnt++;
+ break;
+ case M4_LSM_BP:
+ M4_LSM_BP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_cnt++;
+ break;
+ case M4_LSM_TP:
+ M4_LSM_TP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_cnt++;
+ break;
+ default:
+ System.out.println("unsupported M4_LSM_status!");
+ break;
+ }
+ }
break;
case DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS:
- DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_count += count;
+ DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_count++;
DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_ns += elapsedTimeInNanosecond;
+ // M4-LSM will not use this op
break;
case SEARCH_ARRAY_a_verifBPTP:
- SEARCH_ARRAY_a_verifBPTP_count += count;
+ SEARCH_ARRAY_a_verifBPTP_count++;
SEARCH_ARRAY_a_verifBPTP_ns += elapsedTimeInNanosecond;
+ if (M4_LSM_status != null) {
+ switch (M4_LSM_status) {
+ case M4_LSM_MERGE_M4_TIME_SPAN:
+ M4_LSM_merge_M4_time_span_SEARCH_ARRAY_a_verifBPTP_cnt++;
+ break;
+ case M4_LSM_FP:
+ M4_LSM_FP_SEARCH_ARRAY_a_verifBPTP_cnt++;
+ break;
+ case M4_LSM_LP:
+ M4_LSM_LP_SEARCH_ARRAY_a_verifBPTP_cnt++;
+ break;
+ case M4_LSM_BP:
+ M4_LSM_BP_SEARCH_ARRAY_a_verifBPTP_cnt++;
+ break;
+ case M4_LSM_TP:
+ M4_LSM_TP_SEARCH_ARRAY_a_verifBPTP_cnt++;
+ break;
+ default:
+ System.out.println("unsupported M4_LSM_status!");
+ break;
+ }
+ }
break;
case SEARCH_ARRAY_b_genFP:
- SEARCH_ARRAY_b_genFP_count += count;
+ SEARCH_ARRAY_b_genFP_count++;
SEARCH_ARRAY_b_genFP_ns += elapsedTimeInNanosecond;
+ if (M4_LSM_status != null) {
+ switch (M4_LSM_status) {
+ case M4_LSM_MERGE_M4_TIME_SPAN:
+ M4_LSM_merge_M4_time_span_SEARCH_ARRAY_b_genFP_cnt++;
+ break;
+ case M4_LSM_FP:
+ M4_LSM_FP_SEARCH_ARRAY_b_genFP_cnt++;
+ break;
+ case M4_LSM_LP:
+ M4_LSM_LP_SEARCH_ARRAY_b_genFP_cnt++;
+ break;
+ case M4_LSM_BP:
+ M4_LSM_BP_SEARCH_ARRAY_b_genFP_cnt++;
+ break;
+ case M4_LSM_TP:
+ M4_LSM_TP_SEARCH_ARRAY_b_genFP_cnt++;
+ break;
+ default:
+ System.out.println("unsupported M4_LSM_status!");
+ break;
+ }
+ }
break;
case SEARCH_ARRAY_b_genLP:
- SEARCH_ARRAY_b_genLP_count += count;
+ SEARCH_ARRAY_b_genLP_count++;
SEARCH_ARRAY_b_genLP_ns += elapsedTimeInNanosecond;
+ if (M4_LSM_status != null) {
+ switch (M4_LSM_status) {
+ case M4_LSM_MERGE_M4_TIME_SPAN:
+ M4_LSM_merge_M4_time_span_SEARCH_ARRAY_b_genLP_cnt++;
+ break;
+ case M4_LSM_FP:
+ M4_LSM_FP_SEARCH_ARRAY_b_genLP_cnt++;
+ break;
+ case M4_LSM_LP:
+ M4_LSM_LP_SEARCH_ARRAY_b_genLP_cnt++;
+ break;
+ case M4_LSM_BP:
+ M4_LSM_BP_SEARCH_ARRAY_b_genLP_cnt++;
+ break;
+ case M4_LSM_TP:
+ M4_LSM_TP_SEARCH_ARRAY_b_genLP_cnt++;
+ break;
+ default:
+ System.out.println("unsupported M4_LSM_status!");
+ break;
+ }
+ }
break;
case SEARCH_ARRAY_c_genBPTP:
- SEARCH_ARRAY_c_genBPTP_count += count;
+ SEARCH_ARRAY_c_genBPTP_count++;
SEARCH_ARRAY_c_genBPTP_ns += elapsedTimeInNanosecond;
- break;
- case M4_LSM_INIT_LOAD_ALL_CHUNKMETADATAS:
- M4_LSM_init_loadAllChunkMetadatas_count += count;
- M4_LSM_init_loadAllChunkMetadatas_ns += elapsedTimeInNanosecond;
- break;
- case M4_LSM_MERGE_M4_TIME_SPAN:
- M4_LSM_merge_M4_time_span_count += count;
- M4_LSM_merge_M4_time_span_ns += elapsedTimeInNanosecond;
- break;
- case M4_LSM_FP:
- M4_LSM_FP_count += count;
- M4_LSM_FP_ns += elapsedTimeInNanosecond;
- break;
- case M4_LSM_LP:
- M4_LSM_LP_count += count;
- M4_LSM_LP_ns += elapsedTimeInNanosecond;
- break;
- case M4_LSM_BP:
- M4_LSM_BP_count += count;
- M4_LSM_BP_ns += elapsedTimeInNanosecond;
- break;
- case M4_LSM_TP:
- M4_LSM_TP_count += count;
- M4_LSM_TP_ns += elapsedTimeInNanosecond;
+ if (M4_LSM_status != null) {
+ switch (M4_LSM_status) {
+ case M4_LSM_MERGE_M4_TIME_SPAN:
+ M4_LSM_merge_M4_time_span_SEARCH_ARRAY_c_genBPTP_cnt++;
+ break;
+ case M4_LSM_FP:
+ M4_LSM_FP_SEARCH_ARRAY_c_genBPTP_cnt++;
+ break;
+ case M4_LSM_LP:
+ M4_LSM_LP_SEARCH_ARRAY_c_genBPTP_cnt++;
+ break;
+ case M4_LSM_BP:
+ M4_LSM_BP_SEARCH_ARRAY_c_genBPTP_cnt++;
+ break;
+ case M4_LSM_TP:
+ M4_LSM_TP_SEARCH_ARRAY_c_genBPTP_cnt++;
+ break;
+ default:
+ System.out.println("unsupported M4_LSM_status!");
+ break;
+ }
+ }
break;
default:
System.out.println("not supported operation type"); // this will not happen
[iotdb] 44/46: fix
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit c5fa371b3e2b11aa27af3df06421feec1b540fac
Author: Lei Rui <10...@qq.com>
AuthorDate: Fri Jul 7 15:29:49 2023 +0800
fix
---
.../file/metadata/statistics/StepRegress.java | 14 +++++++--
.../iotdb/tsfile/read/common/ChunkSuit4CPV.java | 36 +++++++++-------------
2 files changed, 26 insertions(+), 24 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/StepRegress.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/StepRegress.java
index 3e14571d3d0..9898d145e6f 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/StepRegress.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/StepRegress.java
@@ -61,7 +61,7 @@ public class StepRegress {
private double mean = 0; // mean of intervals
private double stdDev = 0; // standard deviation of intervals
- private long count = 0;
+ public long count = 0;
private double sumX2 = 0.0;
private double sumX1 = 0.0;
@@ -409,12 +409,22 @@ public class StepRegress {
}
}
+ public int infer(double t, int count) throws IOException {
+ int pos = (int) Math.round(infer_internal(t)); // starting from 1
+ // As double loses precision, although the last point is passed at the phase of learning,
+ // the inferred pos by rounding may still exceed max position count.
+ if (pos > count) {
+ pos--;
+ }
+ return pos;
+ }
+
/**
* @param t input timestamp
* @return output the value of the step regression function f(t), which is the estimated position
* in the chunk. Pay attention that f(t) starts from (startTime,1), ends at (endTime,count).
*/
- public double infer(double t) throws IOException {
+ public double infer_internal(double t) throws IOException {
if (segmentKeys.size() == 1) { // TODO DEBUG
return 1;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
index b5791f0ed4e..b640632e113 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
@@ -32,13 +32,11 @@ import org.apache.iotdb.tsfile.file.metadata.statistics.StepRegress;
import org.apache.iotdb.tsfile.read.common.IOMonitor2.Operation;
import org.apache.iotdb.tsfile.read.reader.page.PageReader;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.IOException;
public class ChunkSuit4CPV {
- private static final Logger M4_CHUNK_METADATA = LoggerFactory.getLogger("M4_CHUNK_METADATA");
+
+ // private static final Logger M4_CHUNK_METADATA = LoggerFactory.getLogger("M4_CHUNK_METADATA");
private ChunkMetadata chunkMetadata; // fixed info, including version, dataType, stepRegress
public int modelPointsCursor =
@@ -258,7 +256,10 @@ public class ChunkSuit4CPV {
if (TSFileDescriptor.getInstance().getConfig().isUseTimeIndex()) {
StepRegress stepRegress = chunkMetadata.getStatistics().getStepRegress();
// infer position starts from 1, so minus 1 here
- estimatedPos = (int) Math.round(stepRegress.infer(targetTimestamp)) - 1;
+ // note get count from global chunkMetadata.getStatistics().getCount(), not local
+ // statistics.getCount
+ estimatedPos =
+ stepRegress.infer(targetTimestamp, chunkMetadata.getStatistics().getCount()) - 1;
// search from estimatePos in the timeBuffer to find the closet timestamp equal to or larger
// than the given timestamp
@@ -374,24 +375,13 @@ public class ChunkSuit4CPV {
if (TSFileDescriptor.getInstance().getConfig().isUseTimeIndex()) {
StepRegress stepRegress = chunkMetadata.getStatistics().getStepRegress();
// infer position starts from 1, so minus 1 here
- estimatedPos = (int) Math.round(stepRegress.infer(targetTimestamp)) - 1;
+ // note get count from global chunkMetadata.getStatistics().getCount(), not local
+ // statistics.getCount
+ estimatedPos =
+ stepRegress.infer(targetTimestamp, chunkMetadata.getStatistics().getCount()) - 1;
// search from estimatePos in the timeBuffer to find the closet timestamp equal to or smaller
// than the given timestamp
- // TODO debug
- try {
- long tmp = pageReader.timeBuffer.getLong(estimatedPos * 8);
- } catch (Exception e) {
- M4_CHUNK_METADATA.info("targetTimestamp=" + targetTimestamp);
- M4_CHUNK_METADATA.info("estimatedPos=" + estimatedPos);
- M4_CHUNK_METADATA.info("count=" + chunkMetadata.getStatistics().getCount());
- M4_CHUNK_METADATA.info("stepregress segmentKeys=" + stepRegress.getSegmentKeys());
- M4_CHUNK_METADATA.info(
- "stepregress segmentIntercepts=" + stepRegress.getSegmentIntercepts());
- M4_CHUNK_METADATA.info("stepregress slope=" + stepRegress.getSlope());
- throw e;
- }
-
if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
while (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
estimatedPos--;
@@ -510,8 +500,10 @@ public class ChunkSuit4CPV {
if (TSFileDescriptor.getInstance().getConfig().isUseTimeIndex()) {
StepRegress stepRegress = chunkMetadata.getStatistics().getStepRegress();
// infer position starts from 1, so minus 1 here
- // TODO debug buffer.get(index)
- int estimatedPos = (int) Math.round(stepRegress.infer(targetTimestamp)) - 1;
+ // note get count from global chunkMetadata.getStatistics().getCount(), not local
+ // statistics.getCount
+ int estimatedPos =
+ stepRegress.infer(targetTimestamp, chunkMetadata.getStatistics().getCount()) - 1;
// search from estimatePos in the timeBuffer to find the closet timestamp equal to or smaller
// than the given timestamp
[iotdb] 04/46: add license
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 1ded315f412a51bf6554e51f35d23119537cf5bd
Author: Lei Rui <10...@qq.com>
AuthorDate: Mon Jun 12 14:29:51 2023 +0800
add license
---
.../apache/iotdb/tsfile/read/common/IOMonitor2.java | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
index 1682eaa6494..5af7b162f4c 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.iotdb.tsfile.read.common;
import org.slf4j.Logger;
[iotdb] 39/46: add log
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit b2956368b6093d87fd4b7820f53f95e359df50fb
Author: Lei Rui <10...@qq.com>
AuthorDate: Fri Jul 7 10:58:56 2023 +0800
add log
---
.../org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java
index f823fc2e94f..804a17d53d4 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java
@@ -155,7 +155,7 @@ public abstract class Statistics<T> {
byteLen += valueIndex.valueOut.size();
byteLen += ReadWriteIOUtils.write(valueIndex.errorBound, outputStream);
-
+ LOG.info("value_index_serialize_byteLen,{}", byteLen);
return byteLen;
}
@@ -173,6 +173,7 @@ public abstract class Statistics<T> {
for (int i = 1; i < segmentKeys.size() - 1; i++) { // t2,t3,...,tm-1
byteLen += ReadWriteIOUtils.write(segmentKeys.get(i), outputStream);
}
+ LOG.info("time_index_serialize_byteLen,{}", byteLen);
return byteLen;
}
[iotdb] 36/46: rename
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit df9c9417648ce69762fe61f2310e6c16fe6a0bec
Author: Lei Rui <10...@qq.com>
AuthorDate: Wed Jul 5 18:40:56 2023 +0800
rename
---
.../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 7 +
.../resources/conf/iotdb-engine.properties | 4 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 14 +-
.../groupby/GroupByWithoutValueFilterDataSet.java | 13 +-
.../dataset/groupby/LocalGroupByExecutor4CPV.java | 7 +-
.../apache/iotdb/db/integration/m4/MyTest1.java | 8 +-
.../apache/iotdb/db/integration/m4/MyTest2.java | 6 +-
.../apache/iotdb/db/integration/m4/MyTest3.java | 6 +-
.../apache/iotdb/db/integration/m4/MyTest4.java | 6 +-
.../apache/iotdb/db/integration/m4/MyTest5.java | 6 +-
.../apache/iotdb/db/integration/m4/MyTmpTest.java | 6 +-
.../iotdb/tsfile/common/conf/TSFileConfig.java | 20 +-
.../iotdb/tsfile/common/conf/TSFileDescriptor.java | 7 +-
.../iotdb/tsfile/read/common/ChunkSuit4CPV.java | 6 +-
.../iotdb/tsfile/read/common/IOMonitor2.java | 5 +-
.../iotdb/tsfile/read/reader/page/PageReader.java | 484 +++++++++++++++------
16 files changed, 422 insertions(+), 183 deletions(-)
diff --git a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBStatement.java b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBStatement.java
index f8e9c6b8065..19c8d0f72f3 100644
--- a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBStatement.java
+++ b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBStatement.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.service.rpc.thrift.TSCancelOperationReq;
import org.apache.iotdb.service.rpc.thrift.TSCloseOperationReq;
import org.apache.iotdb.service.rpc.thrift.TSExecuteBatchStatementReq;
+import org.apache.iotdb.service.rpc.thrift.TSExecuteFinishResp;
import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementReq;
import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementResp;
import org.apache.iotdb.service.rpc.thrift.TSIService;
@@ -383,6 +384,12 @@ public class IoTDBStatement implements Statement {
}
}
+ public String executeFinish() throws TException, StatementExecutionException {
+ TSExecuteFinishResp ret = client.executeFinish();
+ RpcUtils.verifySuccess(ret.getStatus());
+ return ret.executionInfo;
+ }
+
private ResultSet executeQuerySQL(String sql, long timeoutInMS) throws TException, SQLException {
isCancelled = false;
TSExecuteStatementReq execReq = new TSExecuteStatementReq(sessionId, sql, stmtId);
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index 28ca06a1e62..48e720a2995 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -608,10 +608,12 @@ enable_unseq_compaction=false
use_Statistics=true
-use_ChunkIndex=true
+use_TimeIndex=true
use_Mad=false
+use_ValueIndex=true
+
# group_size_in_byte=134217728
# The memory size for each series writer to pack page, default value is 64KB
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 04da72c848a..f40296e82b1 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -906,12 +906,12 @@ public class IoTDBDescriptor {
TSFileDescriptor.getInstance().getConfig().isUseStatistics()))));
TSFileDescriptor.getInstance()
.getConfig()
- .setUseChunkIndex(
+ .setUseTimeIndex(
Boolean.parseBoolean(
properties.getProperty(
- "use_ChunkIndex",
+ "use_TimeIndex",
Boolean.toString(
- TSFileDescriptor.getInstance().getConfig().isUseChunkIndex()))));
+ TSFileDescriptor.getInstance().getConfig().isUseTimeIndex()))));
TSFileDescriptor.getInstance()
.getConfig()
.setUseMad(
@@ -919,6 +919,14 @@ public class IoTDBDescriptor {
properties.getProperty(
"use_Mad",
Boolean.toString(TSFileDescriptor.getInstance().getConfig().isUseMad()))));
+ TSFileDescriptor.getInstance()
+ .getConfig()
+ .setUseValueIndex(
+ Boolean.parseBoolean(
+ properties.getProperty(
+ "use_ValueIndex",
+ Boolean.toString(
+ TSFileDescriptor.getInstance().getConfig().isUseValueIndex()))));
TSFileDescriptor.getInstance()
.getConfig()
.setGroupSizeInByte(
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
index 0b8d58f145e..bd02b795e77 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
@@ -201,12 +201,21 @@ public class GroupByWithoutValueFilterDataSet extends GroupByEngineDataSet {
boolean ascending)
throws StorageEngineException, QueryProcessException {
if (CONFIG.isEnableCPV()) {
- if (TSFileDescriptor.getInstance().getConfig().isUseChunkIndex()) {
+ if (TSFileDescriptor.getInstance().getConfig().isUseTimeIndex()
+ && TSFileDescriptor.getInstance().getConfig().isUseValueIndex()) {
IOMonitor2.dataSetType =
DataSetType.GroupByWithoutValueFilterDataSet_LocalGroupByExecutor4CPV_UseIndex;
+ } else if (!TSFileDescriptor.getInstance().getConfig().isUseTimeIndex()
+ && TSFileDescriptor.getInstance().getConfig().isUseValueIndex()) {
+ IOMonitor2.dataSetType =
+ DataSetType.GroupByWithoutValueFilterDataSet_LocalGroupByExecutor4CPV_NoTimeIndex;
+ } else if (TSFileDescriptor.getInstance().getConfig().isUseTimeIndex()
+ && !TSFileDescriptor.getInstance().getConfig().isUseValueIndex()) {
+ IOMonitor2.dataSetType =
+ DataSetType.GroupByWithoutValueFilterDataSet_LocalGroupByExecutor4CPV_NoValueIndex;
} else {
IOMonitor2.dataSetType =
- DataSetType.GroupByWithoutValueFilterDataSet_LocalGroupByExecutor4CPV_NotUseIndex;
+ DataSetType.GroupByWithoutValueFilterDataSet_LocalGroupByExecutor4CPV_NoTimeValueIndex;
}
return new LocalGroupByExecutor4CPV(
path, allSensors, dataType, context, timeFilter, fileFilter, ascending);
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java
index 5223ec6bde2..294129523e3 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java
@@ -411,8 +411,9 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
.getPageReader()
.setDeleteIntervalList(chunkSuit4CPV.getChunkMetadata().getDeleteIntervalList());
}
- // chunk data read operation (c): get all data points
- chunkSuit4CPV.getPageReader().updateBPTP(chunkSuit4CPV);
+ // chunk data read operation (c)
+ // chunkSuit4CPV.getPageReader().updateBPTP(chunkSuit4CPV);
+ chunkSuit4CPV.getPageReader().updateBP_withValueIndex(chunkSuit4CPV);
// check if empty
if (chunkSuit4CPV.statistics.getCount() == 0) {
currentChunkList.remove(chunkSuit4CPV);
@@ -608,7 +609,7 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
.getPageReader()
.setDeleteIntervalList(chunkSuit4CPV.getChunkMetadata().getDeleteIntervalList());
}
- // chunk data read operation (c): get all data points
+ // chunk data read operation (c)
// chunkSuit4CPV.getPageReader().updateBPTP(chunkSuit4CPV);
chunkSuit4CPV.getPageReader().updateTP_withValueIndex(chunkSuit4CPV); // TODO
// check if empty
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest1.java b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest1.java
index c2ab7f38682..db948cd3da0 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest1.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest1.java
@@ -24,6 +24,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.compaction.CompactionStrategy;
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.jdbc.Config;
+import org.apache.iotdb.jdbc.IoTDBStatement;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.junit.After;
@@ -71,8 +72,8 @@ public class MyTest1 {
originalEnableCPV = config.isEnableCPV();
config.setEnableCPV(true); // CPV
- originalUseChunkIndex = TSFileDescriptor.getInstance().getConfig().isUseChunkIndex();
- TSFileDescriptor.getInstance().getConfig().setUseChunkIndex(false);
+ originalUseChunkIndex = TSFileDescriptor.getInstance().getConfig().isUseTimeIndex();
+ TSFileDescriptor.getInstance().getConfig().setUseTimeIndex(false);
originalUseMad = TSFileDescriptor.getInstance().getConfig().isUseMad();
TSFileDescriptor.getInstance().getConfig().setUseMad(true);
@@ -87,7 +88,7 @@ public class MyTest1 {
EnvironmentUtils.cleanEnv();
config.setCompactionStrategy(originalCompactionStrategy);
config.setEnableCPV(originalEnableCPV);
- TSFileDescriptor.getInstance().getConfig().setUseChunkIndex(originalUseChunkIndex);
+ TSFileDescriptor.getInstance().getConfig().setUseTimeIndex(originalUseChunkIndex);
TSFileDescriptor.getInstance().getConfig().setUseMad(originalUseMad);
}
@@ -132,6 +133,7 @@ public class MyTest1 {
Assert.assertEquals(res[i++], ans);
}
}
+ System.out.println(((IoTDBStatement) statement).executeFinish());
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest2.java b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest2.java
index c6fc8dd3e1a..134fddbb2f6 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest2.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest2.java
@@ -73,8 +73,8 @@ public class MyTest2 {
originalSeqTsFileSize = config.getSeqTsFileSize();
originalUnSeqTsFileSize = config.getUnSeqTsFileSize();
- originalUseChunkIndex = TSFileDescriptor.getInstance().getConfig().isUseChunkIndex();
- TSFileDescriptor.getInstance().getConfig().setUseChunkIndex(false);
+ originalUseChunkIndex = TSFileDescriptor.getInstance().getConfig().isUseTimeIndex();
+ TSFileDescriptor.getInstance().getConfig().setUseTimeIndex(false);
config.setCompactionStrategy(CompactionStrategy.NO_COMPACTION);
@@ -96,7 +96,7 @@ public class MyTest2 {
config.setEnableCPV(originalEnableCPV);
config.setSeqTsFileSize(originalSeqTsFileSize);
config.setUnSeqTsFileSize(originalUnSeqTsFileSize);
- TSFileDescriptor.getInstance().getConfig().setUseChunkIndex(originalUseChunkIndex);
+ TSFileDescriptor.getInstance().getConfig().setUseTimeIndex(originalUseChunkIndex);
}
@Test
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest3.java b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest3.java
index 74923aed64a..fb5c514094e 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest3.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest3.java
@@ -69,8 +69,8 @@ public class MyTest3 {
// config.setEnableCPV(false); // MOC
config.setEnableCPV(true); // CPV
- originalUseChunkIndex = TSFileDescriptor.getInstance().getConfig().isUseChunkIndex();
- TSFileDescriptor.getInstance().getConfig().setUseChunkIndex(false);
+ originalUseChunkIndex = TSFileDescriptor.getInstance().getConfig().isUseTimeIndex();
+ TSFileDescriptor.getInstance().getConfig().setUseTimeIndex(false);
EnvironmentUtils.envSetUp();
Class.forName(Config.JDBC_DRIVER_NAME);
@@ -82,7 +82,7 @@ public class MyTest3 {
EnvironmentUtils.cleanEnv();
config.setCompactionStrategy(originalCompactionStrategy);
config.setEnableCPV(originalEnableCPV);
- TSFileDescriptor.getInstance().getConfig().setUseChunkIndex(originalUseChunkIndex);
+ TSFileDescriptor.getInstance().getConfig().setUseTimeIndex(originalUseChunkIndex);
}
@Test
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest4.java b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest4.java
index 7b4bd91c414..db38c7e4f35 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest4.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest4.java
@@ -69,8 +69,8 @@ public class MyTest4 {
originalEnableCPV = config.isEnableCPV();
config.setEnableCPV(true); // CPV
- originalUseChunkIndex = TSFileDescriptor.getInstance().getConfig().isUseChunkIndex();
- TSFileDescriptor.getInstance().getConfig().setUseChunkIndex(false);
+ originalUseChunkIndex = TSFileDescriptor.getInstance().getConfig().isUseTimeIndex();
+ TSFileDescriptor.getInstance().getConfig().setUseTimeIndex(false);
EnvironmentUtils.envSetUp();
Class.forName(Config.JDBC_DRIVER_NAME);
@@ -82,7 +82,7 @@ public class MyTest4 {
EnvironmentUtils.cleanEnv();
config.setCompactionStrategy(originalCompactionStrategy);
config.setEnableCPV(originalEnableCPV);
- TSFileDescriptor.getInstance().getConfig().setUseChunkIndex(originalUseChunkIndex);
+ TSFileDescriptor.getInstance().getConfig().setUseTimeIndex(originalUseChunkIndex);
}
@Test
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest5.java b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest5.java
index 253dc40dd16..2f406f1153c 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest5.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest5.java
@@ -75,8 +75,8 @@ public class MyTest5 {
// config.setEnableCPV(false); // MOC
config.setEnableCPV(true); // CPV
- originalUseChunkIndex = TSFileDescriptor.getInstance().getConfig().isUseChunkIndex();
- TSFileDescriptor.getInstance().getConfig().setUseChunkIndex(false);
+ originalUseChunkIndex = TSFileDescriptor.getInstance().getConfig().isUseTimeIndex();
+ TSFileDescriptor.getInstance().getConfig().setUseTimeIndex(false);
EnvironmentUtils.envSetUp();
Class.forName(Config.JDBC_DRIVER_NAME);
@@ -88,7 +88,7 @@ public class MyTest5 {
EnvironmentUtils.cleanEnv();
config.setCompactionStrategy(originalCompactionStrategy);
config.setEnableCPV(originalEnableCPV);
- TSFileDescriptor.getInstance().getConfig().setUseChunkIndex(originalUseChunkIndex);
+ TSFileDescriptor.getInstance().getConfig().setUseTimeIndex(originalUseChunkIndex);
}
@Test
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTmpTest.java b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTmpTest.java
index 1c8a47cd729..98419762bdd 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTmpTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTmpTest.java
@@ -71,8 +71,8 @@ public class MyTmpTest {
originalEnableCPV = config.isEnableCPV();
config.setEnableCPV(true); // CPV
- originalUseChunkIndex = TSFileDescriptor.getInstance().getConfig().isUseChunkIndex();
- TSFileDescriptor.getInstance().getConfig().setUseChunkIndex(true);
+ originalUseChunkIndex = TSFileDescriptor.getInstance().getConfig().isUseTimeIndex();
+ TSFileDescriptor.getInstance().getConfig().setUseTimeIndex(true);
originalUseMad = TSFileDescriptor.getInstance().getConfig().isUseMad();
TSFileDescriptor.getInstance().getConfig().setUseMad(true);
@@ -87,7 +87,7 @@ public class MyTmpTest {
EnvironmentUtils.cleanEnv();
config.setCompactionStrategy(originalCompactionStrategy);
config.setEnableCPV(originalEnableCPV);
- TSFileDescriptor.getInstance().getConfig().setUseChunkIndex(originalUseChunkIndex);
+ TSFileDescriptor.getInstance().getConfig().setUseTimeIndex(originalUseChunkIndex);
TSFileDescriptor.getInstance().getConfig().setUseMad(originalUseMad);
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
index 65cc33c8c6e..26e518b865d 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
@@ -29,7 +29,9 @@ public class TSFileConfig implements Serializable {
private boolean useStatistics = true;
- private boolean useChunkIndex = true;
+ private boolean useTimeIndex = true;
+
+ private boolean useValueIndex = true;
private boolean useMad = false;
@@ -161,12 +163,12 @@ public class TSFileConfig implements Serializable {
this.useStatistics = useStatistics;
}
- public boolean isUseChunkIndex() {
- return useChunkIndex;
+ public boolean isUseTimeIndex() {
+ return useTimeIndex;
}
- public void setUseChunkIndex(boolean useChunkIndex) {
- this.useChunkIndex = useChunkIndex;
+ public void setUseTimeIndex(boolean useTimeIndex) {
+ this.useTimeIndex = useTimeIndex;
}
public boolean isUseMad() {
@@ -177,6 +179,14 @@ public class TSFileConfig implements Serializable {
this.useMad = useMad;
}
+ public boolean isUseValueIndex() {
+ return useValueIndex;
+ }
+
+ public void setUseValueIndex(boolean useValueIndex) {
+ this.useValueIndex = useValueIndex;
+ }
+
public int getGroupSizeInByte() {
return groupSizeInByte;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java
index c287ad85d57..4394fd4ce67 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java
@@ -108,12 +108,15 @@ public class TSFileDescriptor {
conf.setUseStatistics(
Boolean.parseBoolean(
properties.getProperty("use_Statistics", Boolean.toString(conf.isUseStatistics()))));
- conf.setUseChunkIndex(
+ conf.setUseTimeIndex(
Boolean.parseBoolean(
- properties.getProperty("use_ChunkIndex", Boolean.toString(conf.isUseChunkIndex()))));
+ properties.getProperty("use_TimeIndex", Boolean.toString(conf.isUseTimeIndex()))));
conf.setUseMad(
Boolean.parseBoolean(
properties.getProperty("use_Mad", Boolean.toString(conf.isUseMad()))));
+ conf.setUseValueIndex(
+ Boolean.parseBoolean(
+ properties.getProperty("use_ValueIndex", Boolean.toString(conf.isUseValueIndex()))));
conf.setGroupSizeInByte(
Integer.parseInt(
properties.getProperty(
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
index 4c1e205781e..18d61065299 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
@@ -251,7 +251,7 @@ public class ChunkSuit4CPV {
public int updateFPwithTheClosetPointEqualOrAfter(long targetTimestamp) throws IOException {
long start = System.nanoTime();
int estimatedPos;
- if (TSFileDescriptor.getInstance().getConfig().isUseChunkIndex()) {
+ if (TSFileDescriptor.getInstance().getConfig().isUseTimeIndex()) {
StepRegress stepRegress = chunkMetadata.getStatistics().getStepRegress();
// infer position starts from 1, so minus 1 here
estimatedPos = (int) Math.round(stepRegress.infer(targetTimestamp)) - 1;
@@ -368,7 +368,7 @@ public class ChunkSuit4CPV {
public int updateLPwithTheClosetPointEqualOrBefore(long targetTimestamp) throws IOException {
long start = System.nanoTime();
int estimatedPos;
- if (TSFileDescriptor.getInstance().getConfig().isUseChunkIndex()) {
+ if (TSFileDescriptor.getInstance().getConfig().isUseTimeIndex()) {
StepRegress stepRegress = chunkMetadata.getStatistics().getStepRegress();
// infer position starts from 1, so minus 1 here
estimatedPos = (int) Math.round(stepRegress.infer(targetTimestamp)) - 1;
@@ -490,7 +490,7 @@ public class ChunkSuit4CPV {
public boolean checkIfExist(long targetTimestamp) throws IOException {
long start = System.nanoTime();
boolean exist;
- if (TSFileDescriptor.getInstance().getConfig().isUseChunkIndex()) {
+ if (TSFileDescriptor.getInstance().getConfig().isUseTimeIndex()) {
StepRegress stepRegress = chunkMetadata.getStatistics().getStepRegress();
// infer position starts from 1, so minus 1 here
// TODO debug buffer.get(index)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
index 62e79e40fac..44f5084c5da 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
@@ -27,8 +27,9 @@ public class IOMonitor2 {
UDTFAlignByTimeDataSet_M4_POINT, // mac
UDTFAlignByTimeDataSet_M4_TIMEWINDOW, // mac_tw
GroupByWithoutValueFilterDataSet_LocalGroupByExecutor4CPV_UseIndex, // cpv
- GroupByWithoutValueFilterDataSet_LocalGroupByExecutor4CPV_NotUseIndex, // cpv_noIndex
-
+ GroupByWithoutValueFilterDataSet_LocalGroupByExecutor4CPV_NoTimeIndex, // cpv_noTimeIndex
+ GroupByWithoutValueFilterDataSet_LocalGroupByExecutor4CPV_NoValueIndex, // cpv_noValueIndex
+ GroupByWithoutValueFilterDataSet_LocalGroupByExecutor4CPV_NoTimeValueIndex, // cpv_noTimeValueIndex
GroupByWithoutValueFilterDataSet_LocalGroupByExecutor_UseStatistics, // moc
GroupByWithoutValueFilterDataSet_LocalGroupByExecutor_NotUseStatistics // mac_groupBy
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
index db378202f56..27dac52c0b4 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
@@ -18,6 +18,7 @@
*/
package org.apache.iotdb.tsfile.read.reader.page;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.encoding.decoder.Decoder;
import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException;
import org.apache.iotdb.tsfile.file.header.PageHeader;
@@ -167,167 +168,362 @@ public class PageReader implements IPageReader {
}
public void updateTP_withValueIndex(ChunkSuit4CPV chunkSuit4CPV) {
- // NOTE: get valueIndex from chunkSuit4CPV.getChunkMetadata().getStatistics(), not
- // chunkSuit4CPV.getStatistics()!
- ValueIndex valueIndex = chunkSuit4CPV.getChunkMetadata().getStatistics().valueIndex;
-
- // step 1: find threshold
- boolean isFound = false;
- double foundValue = 0;
- // iterate SDT points from value big to small to find the first point not deleted
- for (int n = valueIndex.sortedModelPoints.size() - 1; n >= 0; n--) { // NOTE from big to small
- ValuePoint valuePoint = valueIndex.sortedModelPoints.get(n);
- int idx = valuePoint.index; // index starting from 1
- int pos = idx - 1; // pos starting from 0
- long time = timeBuffer.getLong(pos * 8);
- // check if deleted
- if ((pos >= chunkSuit4CPV.startPos)
- && (pos <= chunkSuit4CPV.endPos)
- && !isDeleted_NoHistoryDeleteCursor(time)) {
- // startPos&endPos conveys the virtual deletes of the current M4 time span
- isFound = true;
- foundValue = valuePoint.value;
- break;
+ long start = System.nanoTime();
+ if (TSFileDescriptor.getInstance().getConfig().isUseValueIndex()) {
+ // NOTE: get valueIndex from chunkSuit4CPV.getChunkMetadata().getStatistics(), not
+ // chunkSuit4CPV.getStatistics()!
+ ValueIndex valueIndex = chunkSuit4CPV.getChunkMetadata().getStatistics().valueIndex;
+
+ // step 1: find threshold
+ boolean isFound = false;
+ double foundValue = 0;
+ // iterate SDT points from value big to small to find the first point not deleted
+ for (int n = valueIndex.sortedModelPoints.size() - 1; n >= 0; n--) { // NOTE from big to small
+ IOMonitor2.DCP_D_traversedPointNum++; // TODO
+ ValuePoint valuePoint = valueIndex.sortedModelPoints.get(n);
+ int idx = valuePoint.index; // index starting from 1
+ int pos = idx - 1; // pos starting from 0
+ long time = timeBuffer.getLong(pos * 8);
+ // check if deleted
+ if ((pos >= chunkSuit4CPV.startPos)
+ && (pos <= chunkSuit4CPV.endPos)
+ && !isDeleted_NoHistoryDeleteCursor(time)) {
+ // startPos&endPos conveys the virtual deletes of the current M4 time span
+ isFound = true;
+ foundValue = valuePoint.value;
+ break;
+ }
}
+ if (!isFound) { // unfortunately all sdt points are deleted
+ // this includes the case that all points including model points are deleted,
+ // which is handled by updateBPTP
+ updateBPTP(chunkSuit4CPV); // then fall back to baseline method
+ return;
+ }
+ double threshold_LB = foundValue - valueIndex.errorBound; // near max LB
+
+ // step 2: calculate pruned intervals for TP: UB<threshold=near max LB
+ // increment global chunkSuit4CPV.modelPointsCursor
+ int idx2;
+ // note that the first and last points of a chunk are stored in model points
+ // there must exist idx2-1 >= startPos, otherwise this chunk won't be processed for the
+ // current
+ // time span
+ while ((idx2 = valueIndex.modelPointIdx_list.get(chunkSuit4CPV.modelPointsCursor)) - 1
+ < chunkSuit4CPV.startPos) {
+ // -1 because idx starting from 1 while pos starting from 0
+ chunkSuit4CPV.modelPointsCursor++;
+ // pointing to the right end of the first model segment that passes the left endpoint of the
+ // current time span
+ }
+ // increment local cursor starting from chunkSuit4CPV.modelPointsCursor for iterating model
+ // segments for the current time span
+ // do not increment modelPointsCursor because the model segments for this time span may be
+ // iterated multiple times
+ int localCursor =
+ chunkSuit4CPV.modelPointsCursor; // pointing to the right end of the model segment
+ List<Integer> prune_intervals_start = new ArrayList<>();
+ List<Integer> prune_intervals_end = new ArrayList<>();
+ int interval_start = -1;
+ int interval_end = -1;
+ int idx1;
+ // there must exist idx1-1 <= endPos, otherwise this chunk won't be processed for the current
+ // time span
+ while (localCursor < valueIndex.modelPointIdx_list.size()
+ && (idx1 = valueIndex.modelPointIdx_list.get(localCursor - 1)) - 1
+ <= chunkSuit4CPV.endPos) {
+ IOMonitor2.DCP_D_traversedPointNum++; // TODO
+ idx2 = valueIndex.modelPointIdx_list.get(localCursor);
+ double v1_UB = valueIndex.modelPointVal_list.get(localCursor - 1) + valueIndex.errorBound;
+ double v2_UB = valueIndex.modelPointVal_list.get(localCursor) + valueIndex.errorBound;
+ if (v1_UB < threshold_LB && v2_UB < threshold_LB) {
+ if (interval_start < 0) {
+ interval_start = idx1;
+ }
+ interval_end = idx2; // continuous
+ } else if (v1_UB < threshold_LB && v2_UB >= threshold_LB) {
+ if (interval_start < 0) {
+ interval_start = idx1;
+ }
+ prune_intervals_start.add(interval_start);
+ prune_intervals_end.add(
+ (int) Math.floor((threshold_LB - v1_UB) * (idx2 - idx1) / (v2_UB - v1_UB) + idx1));
+ interval_start = -1; // discontinuous
+ } else if (v1_UB >= threshold_LB && v2_UB < threshold_LB) {
+ interval_start =
+ (int) Math.ceil((threshold_LB - v1_UB) * (idx2 - idx1) / (v2_UB - v1_UB) + idx1);
+ interval_end = idx2; // continuous
+ }
+ localCursor++;
+ }
+ if (interval_start > 0) {
+ prune_intervals_start.add(interval_start);
+ prune_intervals_end.add(interval_end);
+ }
+
+ // step 3: calculate unpruned intervals
+ // deal with time span deletes -> update search_startPos and search_endPos
+ // note idx starting from 1, pos starting from 0
+ int search_startPos = chunkSuit4CPV.startPos;
+ int search_endPos = chunkSuit4CPV.endPos;
+ if (prune_intervals_start.size() > 0) {
+ // deal with time span left virtual delete -> update search_startPos
+ int prune_idx1 = prune_intervals_start.get(0);
+ if (prune_idx1 - 1 <= chunkSuit4CPV.startPos) {
+ // +1 for included, -1 for starting from 0
+ search_startPos = Math.max(search_startPos, prune_intervals_end.get(0) + 1 - 1);
+ prune_intervals_start.remove(0);
+ prune_intervals_end.remove(0);
+ }
+ }
+ if (prune_intervals_start.size() > 0) {
+ // deal with time span right virtual delete -> update search_endPos
+ int prune_idx2 = prune_intervals_end.get(prune_intervals_end.size() - 1);
+ if (prune_idx2 - 1 >= search_endPos) {
+ // -1 for included, -1 for starting from 0
+ search_endPos =
+ Math.min(
+ search_endPos,
+ prune_intervals_start.get(prune_intervals_start.size() - 1) - 1 - 1);
+ prune_intervals_start.remove(prune_intervals_start.size() - 1);
+ prune_intervals_end.remove(prune_intervals_end.size() - 1);
+ }
+ }
+ // add search_endPos+1 to the end of prune_intervals_start
+ // turning into search_intervals_end (excluded endpoints)
+ // note that idx&prune_interval&search_interval starting from 1, pos starting from 0
+ prune_intervals_start.add(search_endPos + 1 + 1); // +1 for starting from 1, +1 for excluded
+ // add search_startPos-1 to the start of prune_intervals_end
+ // turning into search_intervals_start (excluded endpoints)
+ prune_intervals_end.add(
+ 0, search_startPos + 1 - 1); // +1 for starting from 1, +1 for excluded
+
+ // step 4: search unpruned intervals
+ // also deal with normal delete intervals
+ if (dataType == TSDataType.DOUBLE) {
+ double candidateTPvalue = -1;
+ long candidateTPtime = -1;
+ // note that idx&prune_interval&search_interval starting from 1, pos starting from 0
+ for (int i = 0; i < prune_intervals_start.size(); i++) {
+ int search_interval_start = prune_intervals_end.get(i) + 1; // included
+ int search_interval_end = prune_intervals_start.get(i) - 1; // included
+ for (int j = search_interval_start;
+ j <= search_interval_end;
+ j++) { // idx starting from 1
+ IOMonitor2.DCP_D_traversedPointNum++; // TODO
+ double v = valueBuffer.getDouble(timeBufferLength + (j - 1) * 8); // pos starting from 0
+ long t = timeBuffer.getLong((j - 1) * 8); // pos starting from 0
+ if (v > candidateTPvalue && !isDeleted_NoHistoryDeleteCursor(t)) {
+ candidateTPvalue = v;
+ candidateTPtime = t;
+ }
+ }
+ }
+ chunkSuit4CPV.statistics.setMaxInfo(new MinMaxInfo(candidateTPvalue, candidateTPtime));
+ } else if (dataType == TSDataType.INT64) {
+ long candidateTPvalue = -1; // NOTE for TP
+ long candidateTPtime = -1;
+ for (int i = 0; i < prune_intervals_start.size(); i++) {
+ int search_interval_start = prune_intervals_end.get(i) + 1; // included
+ int search_interval_end = prune_intervals_start.get(i) - 1; // included
+ for (int j = search_interval_start; j <= search_interval_end; j++) { // starting from 1
+ IOMonitor2.DCP_D_traversedPointNum++; // TODO
+ long v = valueBuffer.getLong(timeBufferLength + (j - 1) * 8);
+ long t = timeBuffer.getLong((j - 1) * 8);
+ if (v > candidateTPvalue && !isDeleted_NoHistoryDeleteCursor(t)) {
+ candidateTPvalue = v;
+ candidateTPtime = t;
+ }
+ }
+ }
+ chunkSuit4CPV.statistics.setMaxInfo(new MinMaxInfo(candidateTPvalue, candidateTPtime));
+ } else {
+ throw new UnSupportedDataTypeException(String.valueOf(dataType));
+ }
+ } else {
+ updateBPTP(chunkSuit4CPV);
}
- if (!isFound) { // unfortunately all sdt points are deleted
- updateBPTP(chunkSuit4CPV); // then fall back to baseline method
- return;
- }
- double threshold_LB = foundValue - valueIndex.errorBound; // near max LB
-
- // step 2: calculate pruned intervals for TP: UB<threshold=near max LB
- // increment global chunkSuit4CPV.modelPointsCursor
- int idx2;
- // note that the first and last points of a chunk are stored in model points
- // there must exist idx2-1 >= startPos, otherwise this chunk won't be processed for the current
- // time span
- while ((idx2 = valueIndex.modelPointIdx_list.get(chunkSuit4CPV.modelPointsCursor)) - 1
- < chunkSuit4CPV.startPos) { // TODO check
- // -1 because idx starting from 1 while pos starting from 0
- chunkSuit4CPV.modelPointsCursor++;
- // pointing to the right end of the first model segment that passes the left endpoint of the
- // current time span
- }
- // increment local cursor starting from chunkSuit4CPV.modelPointsCursor for iterating model
- // segments for the current time span
- // do not increment modelPointsCursor because the model segments for this time span may be
- // iterated multiple times
- int localCursor =
- chunkSuit4CPV.modelPointsCursor; // pointing to the right end of the model segment
- List<Integer> prune_intervals_start = new ArrayList<>();
- List<Integer> prune_intervals_end = new ArrayList<>();
- int interval_start = -1;
- int interval_end = -1;
- int idx1;
- // there must exist idx1-1 <= endPos, otherwise this chunk won't be processed for the current
- // time span
- while (localCursor < valueIndex.modelPointIdx_list.size()
- && (idx1 = valueIndex.modelPointIdx_list.get(localCursor - 1)) - 1
- <= chunkSuit4CPV.endPos) {
- idx2 = valueIndex.modelPointIdx_list.get(localCursor);
- double v1_UB = valueIndex.modelPointVal_list.get(localCursor - 1) + valueIndex.errorBound;
- double v2_UB = valueIndex.modelPointVal_list.get(localCursor) + valueIndex.errorBound;
- if (v1_UB < threshold_LB && v2_UB < threshold_LB) {
- if (interval_start < 0) {
- interval_start = idx1;
+ IOMonitor2.addMeasure(Operation.SEARCH_ARRAY_c_genBPTP, System.nanoTime() - start);
+ }
+
+ public void updateBP_withValueIndex(ChunkSuit4CPV chunkSuit4CPV) {
+ long start = System.nanoTime();
+ if (TSFileDescriptor.getInstance().getConfig().isUseValueIndex()) {
+ // NOTE: get valueIndex from chunkSuit4CPV.getChunkMetadata().getStatistics(), not
+ // chunkSuit4CPV.getStatistics()!
+ ValueIndex valueIndex = chunkSuit4CPV.getChunkMetadata().getStatistics().valueIndex;
+
+ // step 1: find threshold
+ boolean isFound = false;
+ double foundValue = 0;
+ // iterate SDT points from value small to big to find the first point not deleted
+ for (int n = 0; n < valueIndex.sortedModelPoints.size(); n++) { // NOTE from small to big
+ IOMonitor2.DCP_D_traversedPointNum++; // TODO
+ ValuePoint valuePoint = valueIndex.sortedModelPoints.get(n);
+ int idx = valuePoint.index; // index starting from 1
+ int pos = idx - 1; // pos starting from 0
+ long time = timeBuffer.getLong(pos * 8);
+ // check if deleted
+ if ((pos >= chunkSuit4CPV.startPos)
+ && (pos <= chunkSuit4CPV.endPos)
+ && !isDeleted_NoHistoryDeleteCursor(time)) {
+ // startPos&endPos conveys the virtual deletes of the current M4 time span
+ isFound = true;
+ foundValue = valuePoint.value;
+ break;
}
- interval_end = idx2; // continuous
- } else if (v1_UB < threshold_LB && v2_UB >= threshold_LB) {
- if (interval_start < 0) {
- interval_start = idx1;
+ }
+ if (!isFound) { // unfortunately all sdt points are deleted
+ // this includes the case that all points including model points are deleted,
+ // which is handled by updateBPTP
+ updateBPTP(chunkSuit4CPV); // then fall back to baseline method
+ return;
+ }
+ double threshold_UB = foundValue + valueIndex.errorBound; // near min UB
+
+ // step 2: calculate pruned intervals for BP: LB>threshold=near min UB
+ // increment global chunkSuit4CPV.modelPointsCursor
+ int idx2;
+ // note that the first and last points of a chunk are stored in model points
+ // there must exist idx2-1 >= startPos, otherwise this chunk won't be processed for the
+ // current
+ // time span
+ while ((idx2 = valueIndex.modelPointIdx_list.get(chunkSuit4CPV.modelPointsCursor)) - 1
+ < chunkSuit4CPV.startPos) {
+ IOMonitor2.DCP_D_traversedPointNum++; // TODO
+ // -1 because idx starting from 1 while pos starting from 0
+ chunkSuit4CPV.modelPointsCursor++;
+ // pointing to the right end of the first model segment that passes the left endpoint of the
+ // current time span
+ }
+ // increment local cursor starting from chunkSuit4CPV.modelPointsCursor for iterating model
+ // segments for the current time span
+ // do not increment modelPointsCursor because the model segments for this time span may be
+ // iterated multiple times
+ int localCursor =
+ chunkSuit4CPV.modelPointsCursor; // pointing to the right end of the model segment
+ List<Integer> prune_intervals_start = new ArrayList<>();
+ List<Integer> prune_intervals_end = new ArrayList<>();
+ int interval_start = -1;
+ int interval_end = -1;
+ int idx1;
+ // there must exist idx1-1 <= endPos, otherwise this chunk won't be processed for the current
+ // time span
+ while (localCursor < valueIndex.modelPointIdx_list.size()
+ && (idx1 = valueIndex.modelPointIdx_list.get(localCursor - 1)) - 1
+ <= chunkSuit4CPV.endPos) {
+ IOMonitor2.DCP_D_traversedPointNum++; // TODO
+ idx2 = valueIndex.modelPointIdx_list.get(localCursor);
+ double v1_LB = valueIndex.modelPointVal_list.get(localCursor - 1) - valueIndex.errorBound;
+ double v2_LB = valueIndex.modelPointVal_list.get(localCursor) - valueIndex.errorBound;
+ if (v1_LB > threshold_UB && v2_LB > threshold_UB) {
+ if (interval_start < 0) {
+ interval_start = idx1;
+ }
+ interval_end = idx2; // continuous
+ } else if (v1_LB > threshold_UB && v2_LB <= threshold_UB) {
+ if (interval_start < 0) {
+ interval_start = idx1;
+ }
+ prune_intervals_start.add(interval_start);
+ prune_intervals_end.add(
+ (int) Math.floor((threshold_UB - v1_LB) * (idx2 - idx1) / (v2_LB - v1_LB) + idx1));
+ interval_start = -1; // discontinuous
+ } else if (v1_LB <= threshold_UB && v2_LB > threshold_UB) {
+ interval_start =
+ (int) Math.ceil((threshold_UB - v1_LB) * (idx2 - idx1) / (v2_LB - v1_LB) + idx1);
+ interval_end = idx2; // continuous
}
+ localCursor++;
+ }
+ if (interval_start > 0) {
prune_intervals_start.add(interval_start);
- prune_intervals_end.add(
- (int) Math.floor((threshold_LB - v1_UB) * (idx2 - idx1) / (v2_UB - v1_UB) + idx1));
- interval_start = -1; // discontinuous
- } else if (v1_UB >= threshold_LB && v2_UB < threshold_LB) {
- interval_start =
- (int) Math.ceil((threshold_LB - v1_UB) * (idx2 - idx1) / (v2_UB - v1_UB) + idx1);
- interval_end = idx2; // continuous
+ prune_intervals_end.add(interval_end);
}
- localCursor++;
- }
- if (interval_start > 0) {
- prune_intervals_start.add(interval_start);
- prune_intervals_end.add(interval_end);
- }
- // step 3: calculate unpruned intervals
- // TODO deal with time span deletes -> update search_startPos and search_endPos
- // note idx starting from 1, pos starting from 0
- int search_startPos = chunkSuit4CPV.startPos;
- int search_endPos = chunkSuit4CPV.endPos;
- if (prune_intervals_start.size() > 0) {
- // deal with time span left virtual delete -> update search_startPos
- int prune_idx1 = prune_intervals_start.get(0);
- if (prune_idx1 - 1 <= chunkSuit4CPV.startPos) {
- // +1 for included, -1 for starting from 0
- search_startPos = Math.max(search_startPos, prune_intervals_end.get(0) + 1 - 1);
- prune_intervals_start.remove(0);
- prune_intervals_end.remove(0);
+ // step 3: calculate unpruned intervals
+ // deal with time span deletes -> update search_startPos and search_endPos
+ // note idx starting from 1, pos starting from 0
+ int search_startPos = chunkSuit4CPV.startPos;
+ int search_endPos = chunkSuit4CPV.endPos;
+ if (prune_intervals_start.size() > 0) {
+ // deal with time span left virtual delete -> update search_startPos
+ int prune_idx1 = prune_intervals_start.get(0);
+ if (prune_idx1 - 1 <= chunkSuit4CPV.startPos) {
+ // +1 for included, -1 for starting from 0
+ search_startPos = Math.max(search_startPos, prune_intervals_end.get(0) + 1 - 1);
+ prune_intervals_start.remove(0);
+ prune_intervals_end.remove(0);
+ }
}
- }
- if (prune_intervals_start.size() > 0) {
- // deal with time span right virtual delete -> update search_endPos
- int prune_idx2 = prune_intervals_end.get(prune_intervals_end.size() - 1);
- if (prune_idx2 - 1 >= search_endPos) {
- // -1 for included, -1 for starting from 0
- search_endPos =
- Math.min(
- search_endPos, prune_intervals_start.get(prune_intervals_start.size() - 1) - 1 - 1);
- prune_intervals_start.remove(prune_intervals_start.size() - 1);
- prune_intervals_end.remove(prune_intervals_end.size() - 1);
+ if (prune_intervals_start.size() > 0) {
+ // deal with time span right virtual delete -> update search_endPos
+ int prune_idx2 = prune_intervals_end.get(prune_intervals_end.size() - 1);
+ if (prune_idx2 - 1 >= search_endPos) {
+ // -1 for included, -1 for starting from 0
+ search_endPos =
+ Math.min(
+ search_endPos,
+ prune_intervals_start.get(prune_intervals_start.size() - 1) - 1 - 1);
+ prune_intervals_start.remove(prune_intervals_start.size() - 1);
+ prune_intervals_end.remove(prune_intervals_end.size() - 1);
+ }
}
- }
- // add search_endPos+1 to the end of prune_intervals_start
- // turning into search_intervals_end (excluded endpoints)
- // note that idx&prune_interval&search_interval starting from 1, pos starting from 0
- prune_intervals_start.add(search_endPos + 1 + 1); // +1 for starting from 1, +1 for excluded
- // add search_startPos-1 to the start of prune_intervals_end
- // turning into search_intervals_start (excluded endpoints)
- prune_intervals_end.add(0, search_startPos + 1 - 1); // +1 for starting from 1, +1 for excluded
-
- // step 4: search unpruned intervals
- // TODO deal with normal delete intervals
- if (dataType == TSDataType.DOUBLE) {
- double candidateTPvalue = -1;
- long candidateTPtime = -1;
+ // add search_endPos+1 to the end of prune_intervals_start
+ // turning into search_intervals_end (excluded endpoints)
// note that idx&prune_interval&search_interval starting from 1, pos starting from 0
- for (int i = 0; i < prune_intervals_start.size(); i++) {
- int search_interval_start = prune_intervals_end.get(i) + 1; // included
- int search_interval_end = prune_intervals_start.get(i) - 1; // included
- for (int j = search_interval_start; j <= search_interval_end; j++) { // idx starting from 1
- double v = valueBuffer.getDouble(timeBufferLength + (j - 1) * 8); // pos starting from 0
- long t = timeBuffer.getLong((j - 1) * 8); // pos starting from 0
- if (v > candidateTPvalue && !isDeleted_NoHistoryDeleteCursor(t)) {
- candidateTPvalue = v;
- candidateTPtime = t;
+ prune_intervals_start.add(search_endPos + 1 + 1); // +1 for starting from 1, +1 for excluded
+ // add search_startPos-1 to the start of prune_intervals_end
+ // turning into search_intervals_start (excluded endpoints)
+ prune_intervals_end.add(
+ 0, search_startPos + 1 - 1); // +1 for starting from 1, +1 for excluded
+
+ // step 4: search unpruned intervals
+ // also deal with normal delete intervals
+ if (dataType == TSDataType.DOUBLE) {
+ double candidateBPvalue = Double.MAX_VALUE; // NOTE for BP
+ long candidateBPtime = -1;
+ // note that idx&prune_interval&search_interval starting from 1, pos starting from 0
+ for (int i = 0; i < prune_intervals_start.size(); i++) {
+ int search_interval_start = prune_intervals_end.get(i) + 1; // included
+ int search_interval_end = prune_intervals_start.get(i) - 1; // included
+ for (int j = search_interval_start;
+ j <= search_interval_end;
+ j++) { // idx starting from 1
+ IOMonitor2.DCP_D_traversedPointNum++; // TODO
+ double v = valueBuffer.getDouble(timeBufferLength + (j - 1) * 8); // pos starting from 0
+ long t = timeBuffer.getLong((j - 1) * 8); // pos starting from 0
+ if (v < candidateBPvalue && !isDeleted_NoHistoryDeleteCursor(t)) {
+ candidateBPvalue = v;
+ candidateBPtime = t;
+ }
}
}
- }
- chunkSuit4CPV.statistics.setMaxInfo(new MinMaxInfo(candidateTPvalue, candidateTPtime));
- } else if (dataType == TSDataType.INT64) {
- long candidateTPvalue = -1; // NOTE for TP
- long candidateTPtime = -1;
- for (int i = 0; i < prune_intervals_start.size(); i++) {
- int search_interval_start = prune_intervals_end.get(i) + 1; // included
- int search_interval_end = prune_intervals_start.get(i) - 1; // included
- for (int j = search_interval_start; j <= search_interval_end; j++) { // starting from 1
- long v = valueBuffer.getLong(timeBufferLength + (j - 1) * 8);
- long t = timeBuffer.getLong((j - 1) * 8);
- if (v > candidateTPvalue && !isDeleted_NoHistoryDeleteCursor(t)) {
- candidateTPvalue = v;
- candidateTPtime = t;
+ chunkSuit4CPV.statistics.setMinInfo(new MinMaxInfo(candidateBPvalue, candidateBPtime));
+ } else if (dataType == TSDataType.INT64) {
+ long candidateBPvalue = Long.MAX_VALUE; // NOTE for BP
+ long candidateBPtime = -1;
+ for (int i = 0; i < prune_intervals_start.size(); i++) {
+ int search_interval_start = prune_intervals_end.get(i) + 1; // included
+ int search_interval_end = prune_intervals_start.get(i) - 1; // included
+ for (int j = search_interval_start; j <= search_interval_end; j++) { // starting from 1
+ IOMonitor2.DCP_D_traversedPointNum++; // TODO
+ long v = valueBuffer.getLong(timeBufferLength + (j - 1) * 8);
+ long t = timeBuffer.getLong((j - 1) * 8);
+ if (v < candidateBPvalue && !isDeleted_NoHistoryDeleteCursor(t)) {
+ candidateBPvalue = v;
+ candidateBPtime = t;
+ }
}
}
+ chunkSuit4CPV.statistics.setMinInfo(new MinMaxInfo(candidateBPvalue, candidateBPtime));
+ } else {
+ throw new UnSupportedDataTypeException(String.valueOf(dataType));
}
- chunkSuit4CPV.statistics.setMaxInfo(new MinMaxInfo(candidateTPvalue, candidateTPtime));
} else {
- throw new UnSupportedDataTypeException(String.valueOf(dataType));
+ updateBPTP(chunkSuit4CPV);
}
-
- // TODO 注意count=0全部点删掉的情况考虑 难道是在isFound=false的时候回到原来方法执行里处理了
+ IOMonitor2.addMeasure(Operation.SEARCH_ARRAY_c_genBPTP, System.nanoTime() - start);
}
public void updateBPTP(ChunkSuit4CPV chunkSuit4CPV) {
[iotdb] 38/46: fix metric
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 65099ff4384bad3a773aa35452bf5831ceca9d9e
Author: Lei Rui <10...@qq.com>
AuthorDate: Fri Jul 7 10:00:24 2023 +0800
fix metric
---
.../iotdb/tsfile/read/common/ChunkSuit4CPV.java | 1 -
.../iotdb/tsfile/read/reader/page/PageReader.java | 30 ++++++++++++++--------
2 files changed, 19 insertions(+), 12 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
index e0de93e5079..866ae9e513c 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
@@ -315,7 +315,6 @@ public class ChunkSuit4CPV {
pageReader.valueBuffer.position(pageReader.timeBufferLength);
while (pageReader.timeBuffer.remaining() > 0) {
estimatedPos++;
- // IOMonitor.incPointsTravered();
IOMonitor2.DCP_D_timeIndex_traversedPointNum++;
long t = pageReader.timeBuffer.getLong();
if (t >= targetTimestamp) {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
index 6c63f57282b..2b56950bf8f 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
@@ -168,8 +168,8 @@ public class PageReader implements IPageReader {
}
public void updateTP_withValueIndex(ChunkSuit4CPV chunkSuit4CPV) {
- long start = System.nanoTime();
if (TSFileDescriptor.getInstance().getConfig().isUseValueIndex()) {
+ long start = System.nanoTime();
// NOTE: get valueIndex from chunkSuit4CPV.getChunkMetadata().getStatistics(), not
// chunkSuit4CPV.getStatistics()!
ValueIndex valueIndex = chunkSuit4CPV.getChunkMetadata().getStatistics().valueIndex;
@@ -197,7 +197,9 @@ public class PageReader implements IPageReader {
if (!isFound) { // unfortunately all sdt points are deleted
// this includes the case that all points including model points are deleted,
// which is handled by updateBPTP
- updateBPTP(chunkSuit4CPV); // then fall back to baseline method
+ updateBPTP_withoutTimeMeasure(chunkSuit4CPV); // then fall back to baseline method
+ // time measured here, avoid counting twice
+ IOMonitor2.addMeasure(Operation.SEARCH_ARRAY_c_genBPTP, System.nanoTime() - start);
return;
}
double threshold_LB = foundValue - valueIndex.errorBound; // near max LB
@@ -340,15 +342,15 @@ public class PageReader implements IPageReader {
} else {
throw new UnSupportedDataTypeException(String.valueOf(dataType));
}
- } else {
+ IOMonitor2.addMeasure(Operation.SEARCH_ARRAY_c_genBPTP, System.nanoTime() - start);
+ } else { // not use value index
updateBPTP(chunkSuit4CPV);
}
- IOMonitor2.addMeasure(Operation.SEARCH_ARRAY_c_genBPTP, System.nanoTime() - start);
}
public void updateBP_withValueIndex(ChunkSuit4CPV chunkSuit4CPV) {
- long start = System.nanoTime();
if (TSFileDescriptor.getInstance().getConfig().isUseValueIndex()) {
+ long start = System.nanoTime();
// NOTE: get valueIndex from chunkSuit4CPV.getChunkMetadata().getStatistics(), not
// chunkSuit4CPV.getStatistics()!
ValueIndex valueIndex = chunkSuit4CPV.getChunkMetadata().getStatistics().valueIndex;
@@ -376,8 +378,10 @@ public class PageReader implements IPageReader {
if (!isFound) { // unfortunately all sdt points are deleted
// this includes the case that all points including model points are deleted,
// which is handled by updateBPTP
- updateBPTP(chunkSuit4CPV); // then fall back to baseline method
- return;
+ updateBPTP_withoutTimeMeasure(chunkSuit4CPV); // then fall back to baseline method
+ // time measured here, avoid counting twice
+ IOMonitor2.addMeasure(Operation.SEARCH_ARRAY_c_genBPTP, System.nanoTime() - start);
+ return; // note here
}
double threshold_UB = foundValue + valueIndex.errorBound; // near min UB
@@ -520,14 +524,13 @@ public class PageReader implements IPageReader {
} else {
throw new UnSupportedDataTypeException(String.valueOf(dataType));
}
- } else {
+ IOMonitor2.addMeasure(Operation.SEARCH_ARRAY_c_genBPTP, System.nanoTime() - start);
+ } else { // not use value index
updateBPTP(chunkSuit4CPV);
}
- IOMonitor2.addMeasure(Operation.SEARCH_ARRAY_c_genBPTP, System.nanoTime() - start);
}
- public void updateBPTP(ChunkSuit4CPV chunkSuit4CPV) {
- long start = System.nanoTime();
+ public void updateBPTP_withoutTimeMeasure(ChunkSuit4CPV chunkSuit4CPV) {
deleteCursor = 0; // TODO DEBUG
Statistics statistics = null;
switch (dataType) {
@@ -592,6 +595,11 @@ public class PageReader implements IPageReader {
} else {
chunkSuit4CPV.statistics.setCount(0); // otherwise count won't be zero
}
+ }
+
+ public void updateBPTP(ChunkSuit4CPV chunkSuit4CPV) {
+ long start = System.nanoTime();
+ updateBPTP_withoutTimeMeasure(chunkSuit4CPV);
IOMonitor2.addMeasure(Operation.SEARCH_ARRAY_c_genBPTP, System.nanoTime() - start);
}
[iotdb] 32/46: add
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit be822bc3a670c353ff9392e2e6f570a8554ac611
Author: Lei Rui <10...@qq.com>
AuthorDate: Tue Jul 4 17:35:54 2023 +0800
add
---
.../encoding/decoder/DeltaBinaryDecoder.java | 2 +-
.../iotdb/tsfile/file/header/PageHeader.java | 3 +-
.../iotdb/tsfile/file/metadata/ChunkMetadata.java | 2 +-
.../tsfile/file/metadata/TimeseriesMetadata.java | 3 +-
.../file/metadata/statistics/Statistics.java | 154 +++++++++++++++++----
.../file/metadata/statistics/ValueIndex.java | 100 +++++++++++++
.../iotdb/tsfile/encoding/SDTEncoderTest.java | 12 +-
7 files changed, 243 insertions(+), 33 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java
index 7f50cd4bb6a..3f225d354d7 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java
@@ -72,7 +72,7 @@ public abstract class DeltaBinaryDecoder extends Decoder {
}
@Override
- public boolean hasNext(ByteBuffer buffer) throws IOException {
+ public boolean hasNext(ByteBuffer buffer) {
return (nextReadIndex < readIntTotalCount) || buffer.remaining() > 0;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/PageHeader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/PageHeader.java
index 178a0a5ee46..2bc1ad7751b 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/PageHeader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/PageHeader.java
@@ -60,7 +60,8 @@ public class PageHeader {
return new PageHeader(uncompressedSize, compressedSize, statistics);
}
- public static PageHeader deserializeFrom(ByteBuffer buffer, TSDataType dataType) {
+ public static PageHeader deserializeFrom(ByteBuffer buffer, TSDataType dataType)
+ throws IOException {
int uncompressedSize = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
int compressedSize = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
Statistics statistics = Statistics.deserialize(buffer, dataType);
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
index 60494e223af..1770618b1a6 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
@@ -189,7 +189,7 @@ public class ChunkMetadata implements Comparable<ChunkMetadata> {
* @return ChunkMetaData object
*/
public static ChunkMetadata deserializeFrom(
- ByteBuffer buffer, TimeseriesMetadata timeseriesMetadata) {
+ ByteBuffer buffer, TimeseriesMetadata timeseriesMetadata) throws IOException {
ChunkMetadata chunkMetaData = new ChunkMetadata();
chunkMetaData.measurementUid = timeseriesMetadata.getMeasurementId();
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
index 7344c2e77c1..f8bea3156b4 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
@@ -93,7 +93,8 @@ public class TimeseriesMetadata implements Accountable {
this.chunkMetadataList = new ArrayList<>(timeseriesMetadata.chunkMetadataList);
}
- public static TimeseriesMetadata deserializeFrom(ByteBuffer buffer, boolean needChunkMetadata) {
+ public static TimeseriesMetadata deserializeFrom(ByteBuffer buffer, boolean needChunkMetadata)
+ throws IOException {
TimeseriesMetadata timeseriesMetaData = new TimeseriesMetadata();
timeseriesMetaData.setTimeSeriesMetadataType(ReadWriteIOUtils.readByte(buffer));
timeseriesMetaData.setMeasurementId(ReadWriteIOUtils.readVarIntString(buffer));
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java
index 1d22cd29d3b..18e50e7be27 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java
@@ -18,6 +18,9 @@
*/
package org.apache.iotdb.tsfile.file.metadata.statistics;
+import org.apache.iotdb.tsfile.encoding.decoder.Decoder;
+import org.apache.iotdb.tsfile.encoding.decoder.DeltaBinaryDecoder.IntDeltaDecoder;
+import org.apache.iotdb.tsfile.encoding.decoder.DoublePrecisionDecoderV2;
import org.apache.iotdb.tsfile.exception.filter.StatisticsClassException;
import org.apache.iotdb.tsfile.exception.write.UnknownColumnTypeException;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -60,6 +63,8 @@ public abstract class Statistics<T> {
private StepRegress stepRegress = new StepRegress();
+ public ValueIndex valueIndex = new ValueIndex();
+
/** @author Yuyuan Kang */
final String OPERATION_NOT_SUPPORT_FORMAT = "%s statistics does not support operation: %s";
@@ -109,10 +114,13 @@ public abstract class Statistics<T> {
public abstract TSDataType getType();
+ @Deprecated
public int getSerializedSize() {
return ReadWriteForEncodingUtils.uVarIntSize(count) // count
+ 16 // startTime, endTime
+ getStatsSize();
+ // stepRegress not counted
+ // value index not counted
}
public abstract int getStatsSize();
@@ -122,20 +130,40 @@ public abstract class Statistics<T> {
byteLen += ReadWriteForEncodingUtils.writeUnsignedVarInt(count, outputStream);
byteLen += ReadWriteIOUtils.write(startTime, outputStream);
byteLen += ReadWriteIOUtils.write(endTime, outputStream);
- // TODO serialize stepRegress
+ // serialize stepRegress
byteLen += serializeStepRegress(outputStream);
+ // TODO serialize value index
+ byteLen += serializeValueIndex(outputStream);
// value statistics of different data type
byteLen += serializeStats(outputStream);
return byteLen;
}
+ int serializeValueIndex(OutputStream outputStream) throws IOException {
+ valueIndex.learn(); // TODO ensure executed once and only once
+ int byteLen = 0;
+ byteLen += ReadWriteIOUtils.write(valueIndex.idxOut.size(), outputStream);
+ outputStream.write(
+ valueIndex.idxOut.getBuf(), 0, valueIndex.idxOut.size()); // NOTE len is important
+ byteLen += valueIndex.idxOut.getBuf().length;
+
+ byteLen += ReadWriteIOUtils.write(valueIndex.valueOut.size(), outputStream);
+ outputStream.write(
+ valueIndex.valueOut.getBuf(), 0, valueIndex.valueOut.size()); // NOTE len is important
+ byteLen += valueIndex.valueOut.getBuf().length;
+
+ byteLen += ReadWriteIOUtils.write(valueIndex.errorBound, outputStream);
+
+ return byteLen;
+ }
+
/**
* slope, m: the number of segment keys, m-2 segment keys in between when m>=2. The first and the
* last segment keys are not serialized here, because they are minTime and endTime respectively.
*/
int serializeStepRegress(OutputStream outputStream) throws IOException {
+ stepRegress.learn(); // TODO ensure executed once and only once
int byteLen = 0;
- stepRegress.learn(); // TODO ensure excuted once and only once
byteLen += ReadWriteIOUtils.write(stepRegress.getSlope(), outputStream); // K
DoubleArrayList segmentKeys = stepRegress.getSegmentKeys();
// t1 is startTime, tm is endTime, so no need serialize t1 and tm
@@ -223,6 +251,9 @@ public abstract class Statistics<T> {
// TODO M4-LSM if there are more than one chunk in a time series, then access each
// chunkMetadata anyway
this.stepRegress = stats.stepRegress;
+ // TODO
+ this.valueIndex = stats.valueIndex;
+
isEmpty = false;
} else {
String thisClass = this.getClass().toString();
@@ -233,6 +264,7 @@ public abstract class Statistics<T> {
}
}
+ @Deprecated
public void update(long time, boolean value) {
if (time < this.startTime) {
startTime = time;
@@ -242,61 +274,66 @@ public abstract class Statistics<T> {
}
count++;
updateStats(value);
- updateStepRegress(time);
}
/** @author Yuyuan Kang */
public void update(long time, int value) {
+ count++;
if (time < this.startTime) {
startTime = time;
}
if (time > this.endTime) {
endTime = time;
}
- count++;
- updateStats(value, time);
+ // update time index
updateStepRegress(time);
+ updateValueIndex(value);
+ updateStats(value, time);
}
/** @author Yuyuan Kang */
public void update(long time, long value) {
+ count++;
if (time < this.startTime) {
startTime = time;
}
if (time > this.endTime) {
endTime = time;
}
- count++;
- updateStats(value, time);
updateStepRegress(time);
+ updateValueIndex(value);
+ updateStats(value, time);
}
/** @author Yuyuan Kang */
public void update(long time, float value) {
+ count++;
if (time < this.startTime) {
startTime = time;
}
if (time > this.endTime) {
endTime = time;
}
- count++;
- updateStats(value, time);
updateStepRegress(time);
+ updateValueIndex(value);
+ updateStats(value, time);
}
/** @author Yuyuan Kang */
public void update(long time, double value) {
+ count++;
if (time < this.startTime) {
startTime = time;
}
if (time > this.endTime) {
endTime = time;
}
- count++;
- updateStats(value, time);
updateStepRegress(time);
+ updateValueIndex(value);
+ updateStats(value, time);
}
+ @Deprecated
public void update(long time, Binary value) {
if (time < startTime) {
startTime = time;
@@ -306,9 +343,9 @@ public abstract class Statistics<T> {
}
count++;
updateStats(value);
- updateStepRegress(time);
}
+ @Deprecated
public void update(long[] time, boolean[] values, int batchSize) {
if (time[0] < startTime) {
startTime = time[0];
@@ -318,61 +355,65 @@ public abstract class Statistics<T> {
}
count += batchSize;
updateStats(values, batchSize);
- updateStepRegress(time, batchSize);
}
/** @author Yuyuan Kang */
public void update(long[] time, int[] values, int batchSize) {
+ count += batchSize;
if (time[0] < startTime) {
startTime = time[0];
}
if (time[batchSize - 1] > this.endTime) {
endTime = time[batchSize - 1];
}
- count += batchSize;
- updateStats(values, time, batchSize);
updateStepRegress(time, batchSize);
+ updateValueIndex(values, batchSize);
+ updateStats(values, time, batchSize);
}
/** @author Yuyuan Kang */
public void update(long[] time, long[] values, int batchSize) {
+ count += batchSize;
if (time[0] < startTime) {
startTime = time[0];
}
if (time[batchSize - 1] > this.endTime) {
endTime = time[batchSize - 1];
}
- count += batchSize;
- updateStats(values, time, batchSize);
updateStepRegress(time, batchSize);
+ updateValueIndex(values, batchSize);
+ updateStats(values, time, batchSize);
}
/** @author Yuyuan Kang */
public void update(long[] time, float[] values, int batchSize) {
+ count += batchSize;
if (time[0] < startTime) {
startTime = time[0];
}
if (time[batchSize - 1] > this.endTime) {
endTime = time[batchSize - 1];
}
- count += batchSize;
- updateStats(values, time, batchSize);
updateStepRegress(time, batchSize);
+ updateValueIndex(values, batchSize);
+ updateStats(values, time, batchSize);
}
/** @author Yuyuan Kang */
public void update(long[] time, double[] values, int batchSize) {
+ count += batchSize;
if (time[0] < startTime) {
startTime = time[0];
}
if (time[batchSize - 1] > this.endTime) {
endTime = time[batchSize - 1];
}
- count += batchSize;
- updateStats(values, time, batchSize);
updateStepRegress(time, batchSize);
+ updateValueIndex(values, batchSize);
+ updateStats(values, time, batchSize);
}
+ @Deprecated
public void update(long[] time, Binary[] values, int batchSize) {
if (time[0] < startTime) {
startTime = time[0];
@@ -382,7 +423,6 @@ public abstract class Statistics<T> {
}
count += batchSize;
updateStats(values, batchSize);
- updateStepRegress(time, batchSize);
}
protected abstract void mergeStatisticsValue(Statistics stats);
@@ -415,6 +455,46 @@ public abstract class Statistics<T> {
}
}
+ void updateValueIndex(int value) {
+ valueIndex.insert(value);
+ }
+
+ void updateValueIndex(long value) {
+ valueIndex.insert(value);
+ }
+
+ void updateValueIndex(float value) {
+ valueIndex.insert(value);
+ }
+
+ void updateValueIndex(double value) {
+ valueIndex.insert(value);
+ }
+
+ void updateValueIndex(int[] values, int batchSize) {
+ for (int i = 0; i < batchSize; i++) {
+ updateValueIndex(values[i]);
+ }
+ }
+
+ void updateValueIndex(long[] values, int batchSize) {
+ for (int i = 0; i < batchSize; i++) {
+ updateValueIndex(values[i]);
+ }
+ }
+
+ void updateValueIndex(float[] values, int batchSize) {
+ for (int i = 0; i < batchSize; i++) {
+ updateValueIndex(values[i]);
+ }
+ }
+
+ void updateValueIndex(double[] values, int batchSize) {
+ for (int i = 0; i < batchSize; i++) {
+ updateValueIndex(values[i]);
+ }
+ }
+
/** @author Yuyuan Kang */
public void updateStats(int value, long timestamp) {
throw new UnsupportedOperationException();
@@ -477,6 +557,7 @@ public abstract class Statistics<T> {
throw new UnsupportedOperationException();
}
+ @Deprecated
public static Statistics deserialize(InputStream inputStream, TSDataType dataType)
throws IOException {
Statistics statistics = getStatsByType(dataType);
@@ -488,17 +569,42 @@ public abstract class Statistics<T> {
return statistics;
}
- public static Statistics deserialize(ByteBuffer buffer, TSDataType dataType) {
+ public static Statistics deserialize(ByteBuffer buffer, TSDataType dataType) throws IOException {
Statistics statistics = getStatsByType(dataType);
statistics.setCount(ReadWriteForEncodingUtils.readUnsignedVarInt(buffer));
statistics.setStartTime(ReadWriteIOUtils.readLong(buffer));
statistics.setEndTime(ReadWriteIOUtils.readLong(buffer));
- statistics.deserializeStepRegress(buffer); // TODO
+ statistics.deserializeStepRegress(buffer);
+ statistics.deserializeValueIndex(buffer); // TODO
statistics.deserialize(buffer);
statistics.isEmpty = false;
return statistics;
}
+ void deserializeValueIndex(ByteBuffer buffer) throws IOException {
+ int idxSize = ReadWriteIOUtils.readInt(buffer);
+ ByteBuffer idxBuffer = buffer.slice();
+ idxBuffer.limit(idxSize);
+ buffer.position(buffer.position() + idxSize);
+ Decoder idxDecoder = new IntDeltaDecoder();
+ while (idxDecoder.hasNext(idxBuffer)) {
+ int idx = idxDecoder.readInt(idxBuffer);
+ valueIndex.modelPointIdx_list.add(idx);
+ }
+
+ int valueSize = ReadWriteIOUtils.readInt(buffer); // valueOut.size()
+ ByteBuffer valueBuffer = buffer.slice();
+ valueBuffer.limit(valueSize);
+ buffer.position(buffer.position() + valueSize);
+ Decoder valueDecoder = new DoublePrecisionDecoderV2();
+ while (valueDecoder.hasNext(valueBuffer)) {
+ double value = valueDecoder.readDouble(valueBuffer);
+ valueIndex.modelPointVal_list.add(value);
+ }
+
+ valueIndex.errorBound = ReadWriteIOUtils.readDouble(buffer);
+ }
+
void deserializeStepRegress(ByteBuffer byteBuffer) {
this.stepRegress.setSlope(ReadWriteIOUtils.readDouble(byteBuffer)); // K
int m = ReadWriteIOUtils.readInt(byteBuffer); // m
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/ValueIndex.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/ValueIndex.java
new file mode 100644
index 00000000000..24deea0a871
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/ValueIndex.java
@@ -0,0 +1,100 @@
+package org.apache.iotdb.tsfile.file.metadata.statistics;
+
+import org.apache.iotdb.tsfile.encoding.encoder.DeltaBinaryEncoder.IntDeltaEncoder;
+import org.apache.iotdb.tsfile.encoding.encoder.DoublePrecisionEncoderV2;
+import org.apache.iotdb.tsfile.encoding.encoder.SDTEncoder;
+import org.apache.iotdb.tsfile.utils.PublicBAOS;
+
+import org.eclipse.collections.impl.list.mutable.primitive.DoubleArrayList;
+import org.eclipse.collections.impl.list.mutable.primitive.IntArrayList;
+
+public class ValueIndex {
+
+ private DoubleArrayList values = new DoubleArrayList();
+ public SDTEncoder sdtEncoder = new SDTEncoder();
+ public double errorBound = 0; // =std*2 =2*std.compDeviation
+ public IntDeltaEncoder idxEncoder = new IntDeltaEncoder();
+ public DoublePrecisionEncoderV2 valueEncoder = new DoublePrecisionEncoderV2();
+ public PublicBAOS idxOut = new PublicBAOS();
+ public PublicBAOS valueOut = new PublicBAOS();
+ public int lastIdx = 0;
+ public int lastIntValue = 0;
+ public long lastLongValue = 0;
+ public float lastFloatValue = 0;
+ public double lastDoubleValue = 0;
+ public IntArrayList modelPointIdx_list = new IntArrayList();
+ public DoubleArrayList modelPointVal_list = new DoubleArrayList();
+
+ // this is necessary, otherwise serialized twice by timeseriesMetadata and chunkMetadata
+ // causing learn() executed more than once!!
+ private boolean isLearned = false;
+
+ private double stdDev = 0; // standard deviation of intervals
+ private long count = 0;
+ private double sumX2 = 0.0;
+ private double sumX1 = 0.0;
+
+ public void insert(int value) {
+ values.add((double) value);
+ count++;
+ sumX1 += (double) value;
+ sumX2 += (double) value * (double) value;
+ }
+
+ public void insert(long value) {
+ values.add((double) value);
+ count++;
+ sumX1 += (double) value;
+ sumX2 += (double) value * (double) value;
+ }
+
+ public void insert(float value) {
+ values.add((double) value);
+ count++;
+ sumX1 += (double) value;
+ sumX2 += (double) value * (double) value;
+ }
+
+ public void insert(double value) {
+ values.add(value);
+ count++;
+ sumX1 += value;
+ sumX2 += value * value;
+ }
+
+ public double getStdDev() { // sample standard deviation
+ double std = Math.sqrt(this.sumX2 / this.count - Math.pow(this.sumX1 / this.count, 2));
+ return Math.sqrt(Math.pow(std, 2) * this.count / (this.count - 1));
+ }
+
+ private void initForLearn() {
+ this.stdDev = getStdDev();
+ this.errorBound = 2 * stdDev;
+ this.sdtEncoder.setCompDeviation(errorBound / 2); // equals stdDev is best
+ }
+
+ public void learn() {
+ if (isLearned) {
+ // this is necessary, otherwise serialized twice by timeseriesMetadata and chunkMetadata
+ // causing learn() executed more than once!!
+ return;
+ }
+ isLearned = true;
+ initForLearn(); // set self-adapting CompDeviation for sdtEncoder
+ int pos = 0;
+ for (double v : values.toArray()) {
+ pos++; // starting from 1
+ if (sdtEncoder.encodeDouble(pos, v)) {
+ idxEncoder.encode((int) sdtEncoder.getTime(), idxOut);
+ valueEncoder.encode(sdtEncoder.getDoubleValue(), valueOut);
+ }
+ }
+ // add the last point except the first point
+ if (values.size() >= 2) { // means there is last point except the first point
+ idxEncoder.encode(pos, idxOut);
+ valueEncoder.encode(values.getLast(), valueOut);
+ }
+ idxEncoder.flush(idxOut);
+ valueEncoder.flush(valueOut);
+ }
+}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SDTEncoderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SDTEncoderTest.java
index 0a21c23f605..07450c8beae 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SDTEncoderTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SDTEncoderTest.java
@@ -233,7 +233,9 @@ public class SDTEncoderTest {
public void mytest2() throws Exception {
// when e=2*std=160000 is the best
String csvData = "D:\\full-game\\BallSpeed.csv";
- double[] eList = new double[] {500000, 400000, 300000, 200000, 160000, 100000, 50000, 10000};
+ // double[] eList = new double[] {500000, 400000, 300000, 200000, 160000, 100000, 50000,
+ // 10000};
+ double[] eList = new double[] {160000};
int[] startList = new int[] {1, 200000, 300000, 400000, 500000, 600000, 700000, 800000, 900000};
List<Double> elapsedTime_withValueIndex_list = new ArrayList<>();
List<Double> elapsedTime_withoutValueIndex_list = new ArrayList<>();
@@ -295,8 +297,8 @@ public class SDTEncoderTest {
// System.out.println("v=" + valueList + ";");
// System.out.println("at=" + selectTimestamps + ";");
// System.out.println("av=" + selectValues + ";");
- // System.out.println("v.size=" + valueList.size());
- // System.out.println("av.size=" + selectTimestamps.size());
+ System.out.println("v.size=" + valueList.size());
+ System.out.println("av.size=" + selectTimestamps.size());
// System.out.println("start test------");
@@ -379,7 +381,7 @@ public class SDTEncoderTest {
elapsedTime_withValueIndex += elapsedTime;
// System.out.println("search with value index: " + elapsedTime / 1000000.0 + " ms");
// System.out.println("TP=(" + candidateTPidx + "," + candidateTPvalue + ")");
- // System.out.println("search interval number=" + prune_intervals_end.size());
+ System.out.println("search interval number=" + prune_intervals_end.size());
int traversedPoints = 0;
for (int i = 0; i < prune_intervals_start.size(); i++) {
int search_interval_start = prune_intervals_end.get(i) + 1; // included
@@ -389,7 +391,7 @@ public class SDTEncoderTest {
}
}
traversedComplexity += traversedPoints;
- // System.out.println("number of traversed points: " + traversedPoints);
+ System.out.println("number of traversed points: " + traversedPoints);
// System.out.println("start test------");
startTime = System.nanoTime();
[iotdb] 45/46: add
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 3992f77d0b186b9ead32fdb0f377e5862aecbeb1
Author: Lei Rui <10...@qq.com>
AuthorDate: Sun Jul 9 20:59:24 2023 +0800
add
---
.../apache/iotdb/tsfile/encoding/SDTEncoderTest.java | 17 +++++++++++++----
1 file changed, 13 insertions(+), 4 deletions(-)
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SDTEncoderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SDTEncoderTest.java
index 43cdc6d6ae5..7bac3fa4181 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SDTEncoderTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SDTEncoderTest.java
@@ -72,12 +72,18 @@ public class SDTEncoderTest {
@Test
public void mytest1() throws Exception {
String csvData = "D:\\full-game\\BallSpeed.csv";
- int start = 200000;
- int range = 100000;
+ // String csvData = "D:\\desktop\\tmp2.csv";
+ int start = 200000; // counting from 1
+ // int start = 1; // counting from 1
+ int range = 200;
+ // int range = 10000;
int end = start + range;
SDTEncoder encoder = new SDTEncoder();
- double e = 100000; // std/2
+ double e = 40000; // std*2
+ // double e = 5; // std*2
encoder.setCompDeviation(e / 2);
+ boolean hasHeader = false;
+ // boolean hasHeader = true;
long count = 0;
String line;
List<Long> timestampList = new ArrayList<>();
@@ -86,6 +92,9 @@ public class SDTEncoderTest {
List<Long> selectValues = new ArrayList<>();
long idx = 0;
try (BufferedReader reader = new BufferedReader(new FileReader(csvData))) {
+ if (hasHeader) {
+ reader.readLine(); // header
+ }
while ((line = reader.readLine()) != null) {
count++;
if (count >= start && count < end) {
@@ -143,7 +152,7 @@ public class SDTEncoderTest {
+ "% y<thresold ===> (t-t1)*(v2-v1)/(t2-t1)+v1<threshold\n"
+ "% 如果v2>=threshold>v1: 则t<(threshold-v1)*(t2-t1)/(v2-v1)+t1，于是[t1,(threshold-v1)*(t2-t1)/(v2-v1)+t1)内的点可剪枝\n"
+ "% 如果v2<threshold<=v1: 则t>(threshold-v1)*(t2-t1)/(v2-v1)+t1，于是((threshold-v1)*(t2-t1)/(v2-v1)+t1,t2]内的点可剪枝\n"
- + "rank=1\n"
+ + "rank=2\n"
+ "threshold=sortedX(rank)\n"
+ "hold on, yline(threshold);\n"
+ "hold on, plot(at(sortedInds(rank)),threshold,'o')\n"
[iotdb] 22/46: add udf time window TW
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 132ab5fe95290fb1a1f1b63cf0b615f84b5c0ba4
Author: Lei Rui <10...@qq.com>
AuthorDate: Sat Jun 17 16:01:59 2023 +0800
add udf time window TW
---
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 1 -
.../db/query/udf/builtin/BuiltinFunction.java | 1 +
.../apache/iotdb/db/query/udf/builtin/UDTFM4.java | 315 +++++++++++++++++++++
.../iotdb/db/query/udf/builtin/UDTFM4MAC.java | 3 +
.../apache/iotdb/db/integration/m4/MyTest5.java | 281 ++++++++++++++++++
.../iotdb/tsfile/read/common/IOMonitor2.java | 3 +-
6 files changed, 602 insertions(+), 2 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
index ffff5ac268d..b5fde29063a 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
@@ -526,7 +526,6 @@ public class PlanExecutor implements IPlanExecutor {
// no time series are selected, return EmptyDataSet
return new EmptyDataSet();
} else if (queryPlan instanceof UDTFPlan) {
- IOMonitor2.dataSetType = DataSetType.UDTFAlignByTimeDataSet;
UDTFPlan udtfPlan = (UDTFPlan) queryPlan;
queryDataSet = queryRouter.udtfQuery(udtfPlan, context);
} else if (queryPlan instanceof GroupByTimeFillPlan) {
diff --git a/server/src/main/java/org/apache/iotdb/db/query/udf/builtin/BuiltinFunction.java b/server/src/main/java/org/apache/iotdb/db/query/udf/builtin/BuiltinFunction.java
index fc8da6b3a0f..216d912d393 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/udf/builtin/BuiltinFunction.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/udf/builtin/BuiltinFunction.java
@@ -48,6 +48,7 @@ public enum BuiltinFunction {
TOP_K("TOP_K", UDTFTopK.class),
BOTTOM_K("BOTTOM_K", UDTFBottomK.class),
M4("M4", UDTFM4MAC.class),
+ M4_TW("M4_TW", UDTFM4.class);
;
private final String functionName;
diff --git a/server/src/main/java/org/apache/iotdb/db/query/udf/builtin/UDTFM4.java b/server/src/main/java/org/apache/iotdb/db/query/udf/builtin/UDTFM4.java
new file mode 100644
index 00000000000..d4f97ac7679
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/udf/builtin/UDTFM4.java
@@ -0,0 +1,315 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.query.udf.builtin;
+
+import org.apache.iotdb.db.exception.metadata.MetadataException;
+import org.apache.iotdb.db.query.udf.api.UDTF;
+import org.apache.iotdb.db.query.udf.api.access.Row;
+import org.apache.iotdb.db.query.udf.api.access.RowWindow;
+import org.apache.iotdb.db.query.udf.api.collector.PointCollector;
+import org.apache.iotdb.db.query.udf.api.customizer.config.UDTFConfigurations;
+import org.apache.iotdb.db.query.udf.api.customizer.parameter.UDFParameterValidator;
+import org.apache.iotdb.db.query.udf.api.customizer.parameter.UDFParameters;
+import org.apache.iotdb.db.query.udf.api.customizer.strategy.SlidingSizeWindowAccessStrategy;
+import org.apache.iotdb.db.query.udf.api.customizer.strategy.SlidingTimeWindowAccessStrategy;
+import org.apache.iotdb.db.query.udf.api.exception.UDFException;
+import org.apache.iotdb.db.query.udf.api.exception.UDFInputSeriesDataTypeNotValidException;
+import org.apache.iotdb.db.query.udf.api.exception.UDFParameterNotValidException;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.common.IOMonitor2;
+import org.apache.iotdb.tsfile.read.common.IOMonitor2.DataSetType;
+
+import java.io.IOException;
+
+/**
+ * For each sliding window, M4 returns the first, last, bottom, top points. The window can be
+ * controlled by either point size or time interval length. The aggregated points in the output
+ * series has been sorted and deduplicated.
+ *
+ * <p>SlidingSizeWindow usage Example: "select M4(s1,'windowSize'='10','slidingStep'='10') from
+ * root.vehicle.d1" (windowSize is required, slidingStep is optional.)
+ *
+ * <p>SlidingTimeWindow usage Example: "select
+ * M4(s1,'timeInterval'='25','slidingStep'='25','displayWindowBegin'='0','displayWindowEnd'='100')
+ * from root.vehicle.d1" (timeInterval is required, slidingStep/displayWindowBegin/displayWindowEnd
+ * are optional.)
+ */
+public class UDTFM4 implements UDTF {
+
+ enum AccessStrategy {
+ SIZE_WINDOW,
+ TIME_WINDOW
+ }
+
+ protected AccessStrategy accessStrategy;
+ protected TSDataType dataType;
+
+ public static final String WINDOW_SIZE_KEY = "windowSize";
+ public static final String TIME_INTERVAL_KEY = "timeInterval";
+ public static final String SLIDING_STEP_KEY = "slidingStep";
+ public static final String DISPLAY_WINDOW_BEGIN_KEY = "displayWindowBegin";
+ public static final String DISPLAY_WINDOW_END_KEY = "displayWindowEnd";
+
+ @Override
+ public void validate(UDFParameterValidator validator) throws UDFException, MetadataException {
+ IOMonitor2.dataSetType = DataSetType.UDTFAlignByTimeDataSet_M4_TIMEWINDOW;
+ validator
+ .validateInputSeriesNumber(1)
+ .validateInputSeriesDataType(
+ 0, TSDataType.INT32, TSDataType.INT64, TSDataType.FLOAT, TSDataType.DOUBLE);
+
+ if (!validator.getParameters().hasAttribute(WINDOW_SIZE_KEY)
+ && !validator.getParameters().hasAttribute(TIME_INTERVAL_KEY)) {
+ throw new UDFParameterNotValidException(
+ String.format(
+ "attribute \"%s\"/\"%s\" is required but was not provided.",
+ WINDOW_SIZE_KEY, TIME_INTERVAL_KEY));
+ }
+ if (validator.getParameters().hasAttribute(WINDOW_SIZE_KEY)
+ && validator.getParameters().hasAttribute(TIME_INTERVAL_KEY)) {
+ throw new UDFParameterNotValidException(
+ String.format(
+ "use attribute \"%s\" or \"%s\" only one at a time.",
+ WINDOW_SIZE_KEY, TIME_INTERVAL_KEY));
+ }
+ if (validator.getParameters().hasAttribute(WINDOW_SIZE_KEY)) {
+ accessStrategy = AccessStrategy.SIZE_WINDOW;
+ } else {
+ accessStrategy = AccessStrategy.TIME_WINDOW;
+ }
+
+ dataType = validator.getParameters().getDataType(0);
+ }
+
+ @Override
+ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurations)
+ throws MetadataException {
+ // set data type
+ configurations.setOutputDataType(dataType);
+
+ // set access strategy
+ if (accessStrategy == AccessStrategy.SIZE_WINDOW) {
+ int windowSize = parameters.getInt(WINDOW_SIZE_KEY);
+ int slidingStep = parameters.getIntOrDefault(SLIDING_STEP_KEY, windowSize);
+ configurations.setAccessStrategy(
+ new SlidingSizeWindowAccessStrategy(windowSize, slidingStep));
+ } else {
+ long timeInterval = parameters.getLong(TIME_INTERVAL_KEY);
+ long displayWindowBegin =
+ parameters.getLongOrDefault(DISPLAY_WINDOW_BEGIN_KEY, Long.MIN_VALUE);
+ long displayWindowEnd = parameters.getLongOrDefault(DISPLAY_WINDOW_END_KEY, Long.MAX_VALUE);
+ long slidingStep = parameters.getLongOrDefault(SLIDING_STEP_KEY, timeInterval);
+ configurations.setAccessStrategy(
+ new SlidingTimeWindowAccessStrategy(
+ timeInterval, slidingStep, displayWindowBegin, displayWindowEnd));
+ }
+ }
+
+ @Override
+ public void transform(RowWindow rowWindow, PointCollector collector)
+ throws UDFException, IOException {
+ switch (dataType) {
+ case INT32:
+ transformInt(rowWindow, collector);
+ break;
+ case INT64:
+ transformLong(rowWindow, collector);
+ break;
+ case FLOAT:
+ transformFloat(rowWindow, collector);
+ break;
+ case DOUBLE:
+ transformDouble(rowWindow, collector);
+ break;
+ default:
+ // This will not happen
+ throw new UDFInputSeriesDataTypeNotValidException(
+ 0, dataType, TSDataType.INT32, TSDataType.INT64, TSDataType.FLOAT, TSDataType.DOUBLE);
+ }
+ }
+
+ public void transformInt(RowWindow rowWindow, PointCollector collector) throws IOException {
+ if (rowWindow.windowSize() > 0) { // else empty window do nothing
+ int firstValue = rowWindow.getRow(0).getInt(0);
+ int lastValue = rowWindow.getRow(rowWindow.windowSize() - 1).getInt(0);
+
+ int minValue = Math.min(firstValue, lastValue);
+ int maxValue = Math.max(firstValue, lastValue);
+ int minIndex = (firstValue < lastValue) ? 0 : rowWindow.windowSize() - 1;
+ int maxIndex = (firstValue > lastValue) ? 0 : rowWindow.windowSize() - 1;
+
+ for (int i = 1; i < rowWindow.windowSize() - 1; i++) {
+ int value = rowWindow.getRow(i).getInt(0);
+ if (value < minValue) {
+ minValue = value;
+ minIndex = i;
+ }
+ if (value > maxValue) {
+ maxValue = value;
+ maxIndex = i;
+ }
+ }
+
+ Row row = rowWindow.getRow(0);
+ collector.putInt(row.getTime(), row.getInt(0));
+
+ int smallerIndex = Math.min(minIndex, maxIndex);
+ int largerIndex = Math.max(minIndex, maxIndex);
+ if (smallerIndex > 0) {
+ row = rowWindow.getRow(smallerIndex);
+ collector.putInt(row.getTime(), row.getInt(0));
+ }
+ if (largerIndex > smallerIndex) {
+ row = rowWindow.getRow(largerIndex);
+ collector.putInt(row.getTime(), row.getInt(0));
+ }
+ if (largerIndex < rowWindow.windowSize() - 1) {
+ row = rowWindow.getRow(rowWindow.windowSize() - 1);
+ collector.putInt(row.getTime(), row.getInt(0));
+ }
+ }
+ }
+
+ public void transformLong(RowWindow rowWindow, PointCollector collector) throws IOException {
+ if (rowWindow.windowSize() > 0) { // else empty window do nothing
+ long firstValue = rowWindow.getRow(0).getLong(0);
+ long lastValue = rowWindow.getRow(rowWindow.windowSize() - 1).getLong(0);
+
+ long minValue = Math.min(firstValue, lastValue);
+ long maxValue = Math.max(firstValue, lastValue);
+ int minIndex = (firstValue < lastValue) ? 0 : rowWindow.windowSize() - 1;
+ int maxIndex = (firstValue > lastValue) ? 0 : rowWindow.windowSize() - 1;
+
+ for (int i = 1; i < rowWindow.windowSize() - 1; i++) {
+ long value = rowWindow.getRow(i).getLong(0);
+ if (value < minValue) {
+ minValue = value;
+ minIndex = i;
+ }
+ if (value > maxValue) {
+ maxValue = value;
+ maxIndex = i;
+ }
+ }
+
+ Row row = rowWindow.getRow(0);
+ collector.putLong(row.getTime(), row.getLong(0));
+
+ int smallerIndex = Math.min(minIndex, maxIndex);
+ int largerIndex = Math.max(minIndex, maxIndex);
+ if (smallerIndex > 0) {
+ row = rowWindow.getRow(smallerIndex);
+ collector.putLong(row.getTime(), row.getLong(0));
+ }
+ if (largerIndex > smallerIndex) {
+ row = rowWindow.getRow(largerIndex);
+ collector.putLong(row.getTime(), row.getLong(0));
+ }
+ if (largerIndex < rowWindow.windowSize() - 1) {
+ row = rowWindow.getRow(rowWindow.windowSize() - 1);
+ collector.putLong(row.getTime(), row.getLong(0));
+ }
+ }
+ }
+
+ public void transformFloat(RowWindow rowWindow, PointCollector collector) throws IOException {
+ if (rowWindow.windowSize() > 0) { // else empty window do nothing
+ float firstValue = rowWindow.getRow(0).getFloat(0);
+ float lastValue = rowWindow.getRow(rowWindow.windowSize() - 1).getFloat(0);
+
+ float minValue = Math.min(firstValue, lastValue);
+ float maxValue = Math.max(firstValue, lastValue);
+ int minIndex = (firstValue < lastValue) ? 0 : rowWindow.windowSize() - 1;
+ int maxIndex = (firstValue > lastValue) ? 0 : rowWindow.windowSize() - 1;
+
+ for (int i = 1; i < rowWindow.windowSize() - 1; i++) {
+ float value = rowWindow.getRow(i).getFloat(0);
+ if (value < minValue) {
+ minValue = value;
+ minIndex = i;
+ }
+ if (value > maxValue) {
+ maxValue = value;
+ maxIndex = i;
+ }
+ }
+
+ Row row = rowWindow.getRow(0);
+ collector.putFloat(row.getTime(), row.getFloat(0));
+
+ int smallerIndex = Math.min(minIndex, maxIndex);
+ int largerIndex = Math.max(minIndex, maxIndex);
+ if (smallerIndex > 0) {
+ row = rowWindow.getRow(smallerIndex);
+ collector.putFloat(row.getTime(), row.getFloat(0));
+ }
+ if (largerIndex > smallerIndex) {
+ row = rowWindow.getRow(largerIndex);
+ collector.putFloat(row.getTime(), row.getFloat(0));
+ }
+ if (largerIndex < rowWindow.windowSize() - 1) {
+ row = rowWindow.getRow(rowWindow.windowSize() - 1);
+ collector.putFloat(row.getTime(), row.getFloat(0));
+ }
+ }
+ }
+
+ public void transformDouble(RowWindow rowWindow, PointCollector collector) throws IOException {
+ if (rowWindow.windowSize() > 0) { // else empty window do nothing
+ double firstValue = rowWindow.getRow(0).getDouble(0);
+ double lastValue = rowWindow.getRow(rowWindow.windowSize() - 1).getDouble(0);
+
+ double minValue = Math.min(firstValue, lastValue);
+ double maxValue = Math.max(firstValue, lastValue);
+ int minIndex = (firstValue < lastValue) ? 0 : rowWindow.windowSize() - 1;
+ int maxIndex = (firstValue > lastValue) ? 0 : rowWindow.windowSize() - 1;
+
+ for (int i = 1; i < rowWindow.windowSize() - 1; i++) {
+ double value = rowWindow.getRow(i).getDouble(0);
+ if (value < minValue) {
+ minValue = value;
+ minIndex = i;
+ }
+ if (value > maxValue) {
+ maxValue = value;
+ maxIndex = i;
+ }
+ }
+
+ Row row = rowWindow.getRow(0);
+ collector.putDouble(row.getTime(), row.getDouble(0));
+
+ int smallerIndex = Math.min(minIndex, maxIndex);
+ int largerIndex = Math.max(minIndex, maxIndex);
+ if (smallerIndex > 0) {
+ row = rowWindow.getRow(smallerIndex);
+ collector.putDouble(row.getTime(), row.getDouble(0));
+ }
+ if (largerIndex > smallerIndex) {
+ row = rowWindow.getRow(largerIndex);
+ collector.putDouble(row.getTime(), row.getDouble(0));
+ }
+ if (largerIndex < rowWindow.windowSize() - 1) {
+ row = rowWindow.getRow(rowWindow.windowSize() - 1);
+ collector.putDouble(row.getTime(), row.getDouble(0));
+ }
+ }
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/udf/builtin/UDTFM4MAC.java b/server/src/main/java/org/apache/iotdb/db/query/udf/builtin/UDTFM4MAC.java
index 7b2388af2d6..654221963f7 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/udf/builtin/UDTFM4MAC.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/udf/builtin/UDTFM4MAC.java
@@ -29,6 +29,8 @@ import org.apache.iotdb.db.query.udf.api.customizer.parameter.UDFParameters;
import org.apache.iotdb.db.query.udf.api.customizer.strategy.RowByRowAccessStrategy;
import org.apache.iotdb.db.query.udf.api.exception.UDFException;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.common.IOMonitor2;
+import org.apache.iotdb.tsfile.read.common.IOMonitor2.DataSetType;
import java.io.IOException;
@@ -100,6 +102,7 @@ public class UDTFM4MAC implements UDTF {
@Override
public void validate(UDFParameterValidator validator) throws UDFException {
+ IOMonitor2.dataSetType = DataSetType.UDTFAlignByTimeDataSet_M4_POINT;
validator
.validateInputSeriesNumber(1)
.validateInputSeriesDataType(
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest5.java b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest5.java
new file mode 100644
index 00000000000..d7803eac00f
--- /dev/null
+++ b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest5.java
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.integration.m4;
+
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.compaction.CompactionStrategy;
+import org.apache.iotdb.db.utils.EnvironmentUtils;
+import org.apache.iotdb.jdbc.Config;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Locale;
+
+import static org.apache.iotdb.db.query.udf.builtin.UDTFM4.DISPLAY_WINDOW_BEGIN_KEY;
+import static org.apache.iotdb.db.query.udf.builtin.UDTFM4.DISPLAY_WINDOW_END_KEY;
+import static org.apache.iotdb.db.query.udf.builtin.UDTFM4.SLIDING_STEP_KEY;
+import static org.apache.iotdb.db.query.udf.builtin.UDTFM4.TIME_INTERVAL_KEY;
+import static org.apache.iotdb.db.query.udf.builtin.UDTFM4.WINDOW_SIZE_KEY;
+import static org.junit.Assert.fail;
+
+public class MyTest5 {
+
+ private static final String TIMESTAMP_STR = "Time";
+
+ private static String[] creationSqls =
+ new String[] {
+ "SET STORAGE GROUP TO root.vehicle.d0",
+ "CREATE TIMESERIES root.vehicle.d0.s0 WITH DATATYPE=DOUBLE,ENCODING=PLAIN",
+ };
+
+ private final String d0s0 = "root.vehicle.d0.s0";
+
+ private static final String insertTemplate =
+ "INSERT INTO root.vehicle.d0(timestamp,s0)" + " VALUES(%d,%d)";
+
+ private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
+ private static boolean originalEnableCPV;
+ private static CompactionStrategy originalCompactionStrategy;
+ private static boolean originalUseChunkIndex;
+
+ @Before
+ public void setUp() throws Exception {
+ TSFileDescriptor.getInstance().getConfig().setTimeEncoder("PLAIN");
+ originalCompactionStrategy = config.getCompactionStrategy();
+ config.setCompactionStrategy(CompactionStrategy.NO_COMPACTION);
+
+ originalEnableCPV = config.isEnableCPV();
+ // config.setEnableCPV(false); // MOC
+ config.setEnableCPV(true); // CPV
+
+ originalUseChunkIndex = TSFileDescriptor.getInstance().getConfig().isUseChunkIndex();
+ TSFileDescriptor.getInstance().getConfig().setUseChunkIndex(false);
+
+ EnvironmentUtils.envSetUp();
+ Class.forName(Config.JDBC_DRIVER_NAME);
+ config.setTimestampPrecision("ms");
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ EnvironmentUtils.cleanEnv();
+ config.setCompactionStrategy(originalCompactionStrategy);
+ config.setEnableCPV(originalEnableCPV);
+ TSFileDescriptor.getInstance().getConfig().setUseChunkIndex(originalUseChunkIndex);
+ }
+
+ @Test
+ public void testM4Function() {
+ // create timeseries
+ try (Connection connection =
+ DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+ statement.execute("SET STORAGE GROUP TO root.m4");
+ statement.execute("CREATE TIMESERIES root.m4.d1.s1 with datatype=double,encoding=PLAIN");
+ statement.execute("CREATE TIMESERIES root.m4.d1.s2 with datatype=INT32,encoding=PLAIN");
+ } catch (SQLException throwable) {
+ fail(throwable.getMessage());
+ }
+
+ // insert data
+ String insertTemplate = "INSERT INTO root.m4.d1(timestamp,%s)" + " VALUES(%d,%d)";
+ try (Connection connection =
+ DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+ // "root.m4.d1.s1" data illustration:
+ // https://user-images.githubusercontent.com/33376433/151985070-73158010-8ba0-409d-a1c1-df69bad1aaee.png
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, "s1", 1, 5));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, "s1", 2, 15));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, "s1", 20, 1));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, "s1", 25, 8));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, "s1", 54, 3));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, "s1", 120, 8));
+ statement.execute("FLUSH");
+
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, "s1", 5, 10));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, "s1", 8, 8));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, "s1", 10, 30));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, "s1", 20, 20));
+ statement.execute("FLUSH");
+
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, "s1", 27, 20));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, "s1", 30, 40));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, "s1", 35, 10));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, "s1", 40, 20));
+ statement.execute("FLUSH");
+
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, "s1", 33, 9));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, "s1", 45, 30));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, "s1", 52, 8));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, "s1", 54, 18));
+ statement.execute("FLUSH");
+
+ // "root.m4.d1.s2" data: constant value 1
+ for (int i = 0; i < 100; i++) {
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, "s2", i, 1));
+ }
+ statement.execute("FLUSH");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ // query tests
+ test_M4_firstWindowEmpty();
+ test_M4_slidingTimeWindow();
+ test_M4_slidingSizeWindow();
+ test_M4_constantTimeSeries();
+ }
+
+ private void test_M4_firstWindowEmpty() {
+ String[] res = new String[] {"120,8.0"};
+
+ String sql =
+ String.format(
+ "select M4_TW(s1, '%s'='%s','%s'='%s','%s'='%s','%s'='%s') from root.m4.d1",
+ TIME_INTERVAL_KEY,
+ 25,
+ SLIDING_STEP_KEY,
+ 25,
+ DISPLAY_WINDOW_BEGIN_KEY,
+ 75,
+ DISPLAY_WINDOW_END_KEY,
+ 150);
+
+ try (Connection conn =
+ DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
+ Statement statement = conn.createStatement()) {
+ ResultSet resultSet = statement.executeQuery(sql);
+ int count = 0;
+ while (resultSet.next()) {
+ String str = resultSet.getString(1) + "," + resultSet.getString(2);
+ Assert.assertEquals(res[count], str);
+ count++;
+ }
+ Assert.assertEquals(res.length, count);
+ } catch (SQLException throwable) {
+ fail(throwable.getMessage());
+ }
+ }
+
+ private void test_M4_slidingTimeWindow() {
+ String[] res =
+ new String[] {
+ "1,5.0", "10,30.0", "20,20.0", "25,8.0", "30,40.0", "45,30.0", "52,8.0", "54,18.0",
+ "120,8.0"
+ };
+
+ String sql =
+ String.format(
+ "select M4_TW(s1, '%s'='%s','%s'='%s','%s'='%s','%s'='%s') from root.m4.d1",
+ TIME_INTERVAL_KEY,
+ 25,
+ SLIDING_STEP_KEY,
+ 25,
+ DISPLAY_WINDOW_BEGIN_KEY,
+ 0,
+ DISPLAY_WINDOW_END_KEY,
+ 150);
+
+ try (Connection conn =
+ DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
+ Statement statement = conn.createStatement()) {
+ ResultSet resultSet = statement.executeQuery(sql);
+ int count = 0;
+ while (resultSet.next()) {
+ String str = resultSet.getString(1) + "," + resultSet.getString(2);
+ Assert.assertEquals(res[count], str);
+ count++;
+ }
+ Assert.assertEquals(res.length, count);
+ } catch (SQLException throwable) {
+ fail(throwable.getMessage());
+ }
+ }
+
+ private void test_M4_slidingSizeWindow() {
+ String[] res = new String[] {"1,5.0", "30,40.0", "33,9.0", "35,10.0", "45,30.0", "120,8.0"};
+
+ String sql =
+ String.format(
+ "select M4_TW(s1,'%s'='%s','%s'='%s') from root.m4.d1",
+ WINDOW_SIZE_KEY, 10, SLIDING_STEP_KEY, 10);
+
+ try (Connection conn =
+ DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
+ Statement statement = conn.createStatement()) {
+ ResultSet resultSet = statement.executeQuery(sql);
+ int count = 0;
+ while (resultSet.next()) {
+ String str = resultSet.getString(1) + "," + resultSet.getString(2);
+ Assert.assertEquals(res[count], str);
+ count++;
+ }
+ Assert.assertEquals(res.length, count);
+ } catch (SQLException throwable) {
+ fail(throwable.getMessage());
+ }
+ }
+
+ private void test_M4_constantTimeSeries() {
+ /* Result: 0,1 24,1 25,1 49,1 50,1 74,1 75,1 99,1 */
+ String sql =
+ String.format(
+ "select M4_TW(s2, '%s'='%s','%s'='%s','%s'='%s','%s'='%s') from root.m4.d1",
+ TIME_INTERVAL_KEY,
+ 25,
+ SLIDING_STEP_KEY,
+ 25,
+ DISPLAY_WINDOW_BEGIN_KEY,
+ 0,
+ DISPLAY_WINDOW_END_KEY,
+ 100);
+
+ try (Connection conn =
+ DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
+ Statement statement = conn.createStatement()) {
+ ResultSet resultSet = statement.executeQuery(sql);
+ int count = 0;
+ while (resultSet.next()) {
+ String expStr;
+ if (count % 2 == 0) {
+ expStr = 25 * (count / 2) + ",1";
+ } else {
+ expStr = 25 * (count / 2) + 24 + ",1";
+ }
+ String str = resultSet.getString(1) + "," + resultSet.getString(2);
+ Assert.assertEquals(expStr, str);
+ count++;
+ }
+ Assert.assertEquals(8, count);
+ } catch (SQLException throwable) {
+ fail(throwable.getMessage());
+ }
+ }
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
index 7541741b1f4..d9b436b8c5e 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
@@ -24,7 +24,8 @@ public class IOMonitor2 {
public enum DataSetType { // dataSet, executor, reader, file
NONE,
RawQueryDataSetWithoutValueFilter,
- UDTFAlignByTimeDataSet,
+ UDTFAlignByTimeDataSet_M4_POINT,
+ UDTFAlignByTimeDataSet_M4_TIMEWINDOW,
GroupByWithoutValueFilterDataSet_LocalGroupByExecutor4CPV,
GroupByWithoutValueFilterDataSet_LocalGroupByExecutor
[iotdb] 21/46: M4 udf add test
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 48bfd3d54d11d0a126806baf3fba182f45ede0c4
Author: Lei Rui <10...@qq.com>
AuthorDate: Sat Jun 17 14:59:33 2023 +0800
M4 udf add test
---
.../iotdb/db/query/udf/builtin/UDTFM4MAC.java | 199 +++++++++++----------
.../apache/iotdb/db/integration/m4/MyTest3.java | 58 ++++++
.../apache/iotdb/db/integration/m4/MyTmpTest.java | 42 ++---
3 files changed, 181 insertions(+), 118 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/udf/builtin/UDTFM4MAC.java b/server/src/main/java/org/apache/iotdb/db/query/udf/builtin/UDTFM4MAC.java
index 4c3fd5b5690..7b2388af2d6 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/udf/builtin/UDTFM4MAC.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/udf/builtin/UDTFM4MAC.java
@@ -123,7 +123,7 @@ public class UDTFM4MAC implements UDTF {
.setAccessStrategy(new RowByRowAccessStrategy())
.setOutputDataType(TSDataType.TEXT);
init();
- this.idx = 0;
+ this.idx = -1;
result = new String[w];
for (int i = 0; i < w; i++) {
result[i] = "empty";
@@ -134,6 +134,9 @@ public class UDTFM4MAC implements UDTF {
@Override
public void transform(Row row, PointCollector collector)
throws QueryProcessException, IOException {
+ if (idx < 0) {
+ idx = 0; // means at least not all empty
+ }
switch (dataType) {
case INT32:
transformInt(row.getTime(), row.getInt(0));
@@ -358,102 +361,104 @@ public class UDTFM4MAC implements UDTF {
@Override
public void terminate(PointCollector collector) throws IOException, QueryProcessException {
- // record the last interval (not necessarily idx=w-1) in the transform stage
- switch (dataType) {
- case INT32:
- result[idx] =
- "FirstPoint=("
- + minTime
- + ","
- + intFirstV
- + "), "
- + "LastPoint=("
- + maxTime
- + ","
- + intLastV
- + "), "
- + "BottomPoint=("
- + bottomTime
- + ","
- + intMinV
- + "), "
- + "TopPoint=("
- + topTime
- + ","
- + intMaxV
- + ")";
- break;
- case INT64:
- result[idx] =
- "FirstPoint=("
- + minTime
- + ","
- + longFirstV
- + "), "
- + "LastPoint=("
- + maxTime
- + ","
- + longLastV
- + "), "
- + "BottomPoint=("
- + bottomTime
- + ","
- + longMinV
- + "), "
- + "TopPoint=("
- + topTime
- + ","
- + longMaxV
- + ")";
- break;
- case FLOAT:
- result[idx] =
- "FirstPoint=("
- + minTime
- + ","
- + floatFirstV
- + "), "
- + "LastPoint=("
- + maxTime
- + ","
- + floatLastV
- + "), "
- + "BottomPoint=("
- + bottomTime
- + ","
- + floatMinV
- + "), "
- + "TopPoint=("
- + topTime
- + ","
- + floatMaxV
- + ")";
- break;
- case DOUBLE:
- result[idx] =
- "FirstPoint=("
- + minTime
- + ","
- + doubleFirstV
- + "), "
- + "LastPoint=("
- + maxTime
- + ","
- + doubleLastV
- + "), "
- + "BottomPoint=("
- + bottomTime
- + ","
- + doubleMinV
- + "), "
- + "TopPoint=("
- + topTime
- + ","
- + doubleMaxV
- + ")";
- break;
- default:
- break;
+ if (idx >= 0) { // means at least not all empty
+ // record the last interval (not necessarily idx=w-1) in the transform stage
+ switch (dataType) {
+ case INT32:
+ result[idx] =
+ "FirstPoint=("
+ + minTime
+ + ","
+ + intFirstV
+ + "), "
+ + "LastPoint=("
+ + maxTime
+ + ","
+ + intLastV
+ + "), "
+ + "BottomPoint=("
+ + bottomTime
+ + ","
+ + intMinV
+ + "), "
+ + "TopPoint=("
+ + topTime
+ + ","
+ + intMaxV
+ + ")";
+ break;
+ case INT64:
+ result[idx] =
+ "FirstPoint=("
+ + minTime
+ + ","
+ + longFirstV
+ + "), "
+ + "LastPoint=("
+ + maxTime
+ + ","
+ + longLastV
+ + "), "
+ + "BottomPoint=("
+ + bottomTime
+ + ","
+ + longMinV
+ + "), "
+ + "TopPoint=("
+ + topTime
+ + ","
+ + longMaxV
+ + ")";
+ break;
+ case FLOAT:
+ result[idx] =
+ "FirstPoint=("
+ + minTime
+ + ","
+ + floatFirstV
+ + "), "
+ + "LastPoint=("
+ + maxTime
+ + ","
+ + floatLastV
+ + "), "
+ + "BottomPoint=("
+ + bottomTime
+ + ","
+ + floatMinV
+ + "), "
+ + "TopPoint=("
+ + topTime
+ + ","
+ + floatMaxV
+ + ")";
+ break;
+ case DOUBLE:
+ result[idx] =
+ "FirstPoint=("
+ + minTime
+ + ","
+ + doubleFirstV
+ + "), "
+ + "LastPoint=("
+ + maxTime
+ + ","
+ + doubleLastV
+ + "), "
+ + "BottomPoint=("
+ + bottomTime
+ + ","
+ + doubleMinV
+ + "), "
+ + "TopPoint=("
+ + topTime
+ + ","
+ + doubleMaxV
+ + ")";
+ break;
+ default:
+ break;
+ }
}
// collect result
for (int i = 0; i < w; i++) {
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest3.java b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest3.java
index ad3f99f9a78..74923aed64a 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest3.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest3.java
@@ -217,4 +217,62 @@ public class MyTest3 {
fail(e.getMessage());
}
}
+
+ @Test
+ public void test2() { // test UDF MAC extreme case: empty from the beginning
+ prepareData2();
+
+ String[] res = new String[] {"0,empty", "50,empty"};
+ try (Connection connection =
+ DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+ long tqs = 0L;
+ long tqe = 100L;
+ int w = 2;
+ boolean hasResultSet =
+ statement.execute(
+ String.format(
+ "select M4(s0,'tqs'='%1$d','tqe'='%2$d','w'='%3$d') from root.vehicle.d0 where "
+ + "time>=%1$d and time<%2$d",
+ tqs, tqe, w));
+
+ String columnName = "M4(root.vehicle.d0.s0, \"tqs\"=\"%d\", \"tqe\"=\"%d\", \"w\"=\"%d\")";
+ Assert.assertTrue(hasResultSet);
+ try (ResultSet resultSet = statement.getResultSet()) {
+ int i = 0;
+ while (resultSet.next()) {
+ String ans =
+ resultSet.getString(TIMESTAMP_STR)
+ + ","
+ + resultSet.getString(String.format(columnName, tqs, tqe, w));
+ System.out.println(ans);
+ Assert.assertEquals(res[i++], ans);
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ private static void prepareData2() {
+ try (Connection connection =
+ DriverManager.getConnection(
+ Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+
+ for (String sql : creationSqls) {
+ statement.execute(sql);
+ }
+
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 200, 5));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 300, 15));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 400, 1));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 500, 8));
+ statement.execute("FLUSH");
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
}
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTmpTest.java b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTmpTest.java
index 677c79ab535..1c8a47cd729 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTmpTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTmpTest.java
@@ -72,7 +72,7 @@ public class MyTmpTest {
config.setEnableCPV(true); // CPV
originalUseChunkIndex = TSFileDescriptor.getInstance().getConfig().isUseChunkIndex();
- TSFileDescriptor.getInstance().getConfig().setUseChunkIndex(false);
+ TSFileDescriptor.getInstance().getConfig().setUseChunkIndex(true);
originalUseMad = TSFileDescriptor.getInstance().getConfig().isUseMad();
TSFileDescriptor.getInstance().getConfig().setUseMad(true);
@@ -107,14 +107,14 @@ public class MyTmpTest {
Statement statement = connection.createStatement()) {
boolean hasResultSet =
statement.execute(
- "SELECT M4(s0,'tqs'='0','tqe'='100','w'='4')"
+ "SELECT M4(s0,'tqs'='0','tqe'='100','w'='10')"
+ " FROM root.vehicle.d0 where time>=0 and time<100");
Assert.assertTrue(hasResultSet);
try (ResultSet resultSet = statement.getResultSet()) {
int i = 0;
while (resultSet.next()) {
- String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(1);
+ String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(2);
System.out.println(ans);
// Assert.assertEquals(res[i++], ans);
}
@@ -184,30 +184,30 @@ public class MyTmpTest {
statement.execute(sql);
}
- statement.execute(String.format(Locale.ENGLISH, insertTemplate, 1, 5));
- statement.execute(String.format(Locale.ENGLISH, insertTemplate, 2, 15));
- statement.execute(String.format(Locale.ENGLISH, insertTemplate, 20, 1));
- statement.execute(String.format(Locale.ENGLISH, insertTemplate, 25, 8));
- statement.execute(String.format(Locale.ENGLISH, insertTemplate, 54, 3));
- statement.execute(String.format(Locale.ENGLISH, insertTemplate, 120, 8));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 1000, 5));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 2000, 15));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 20000, 1));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 25000, 8));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 54000, 3));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 120000, 8));
statement.execute("FLUSH");
- statement.execute(String.format(Locale.ENGLISH, insertTemplate, 5, 10));
- statement.execute(String.format(Locale.ENGLISH, insertTemplate, 8, 8));
- statement.execute(String.format(Locale.ENGLISH, insertTemplate, 10, 30));
- statement.execute(String.format(Locale.ENGLISH, insertTemplate, 20, 20));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 5000, 10));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 8000, 8));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 10000, 30));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 20000, 20));
statement.execute("FLUSH");
- statement.execute(String.format(Locale.ENGLISH, insertTemplate, 27, 20));
- statement.execute(String.format(Locale.ENGLISH, insertTemplate, 30, 40));
- statement.execute(String.format(Locale.ENGLISH, insertTemplate, 35, 10));
- statement.execute(String.format(Locale.ENGLISH, insertTemplate, 40, 20));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 27000, 20));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 30000, 40));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 35000, 10));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 40000, 20));
statement.execute("FLUSH");
- statement.execute(String.format(Locale.ENGLISH, insertTemplate, 33, 9));
- statement.execute(String.format(Locale.ENGLISH, insertTemplate, 45, 30));
- statement.execute(String.format(Locale.ENGLISH, insertTemplate, 52, 8));
- statement.execute(String.format(Locale.ENGLISH, insertTemplate, 54, 18));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 33000, 9));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 45000, 30));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 52000, 8));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 54000, 18));
statement.execute("FLUSH");
} catch (Exception e) {
[iotdb] 13/46: modify iomonitor2 print
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 7ed7a87606f4c0676f6a13c67f1292a8fe47a647
Author: Lei Rui <10...@qq.com>
AuthorDate: Wed Jun 14 15:44:21 2023 +0800
modify iomonitor2 print
---
.../iotdb/tsfile/read/common/IOMonitor2.java | 180 +++++++++++++--------
1 file changed, 114 insertions(+), 66 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
index 9c17f371e34..496bf9ab678 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
@@ -305,139 +305,187 @@ public class IOMonitor2 {
public static String print() {
StringBuilder stringBuilder = new StringBuilder();
- stringBuilder.append("dataSetType").append(",").append(dataSetType).append("\n");
-
+ // [1] is client elapsed time, not measured by the server side
stringBuilder
- .append("Server_Query_Execute_ns")
+ .append("[2]Server_Query_Execute_ns_cnt")
.append(",")
.append(DCP_Server_Query_Execute_ns)
+ .append(",")
+ .append(DCP_Server_Query_Execute_count)
.append("\n");
stringBuilder
- .append("Server_Query_Fetch_ns")
+ .append("[2]Server_Query_Fetch_ns_cnt")
.append(",")
.append(DCP_Server_Query_Fetch_ns)
+ .append(",")
+ .append(DCP_Server_Query_Fetch_count)
.append("\n");
+ stringBuilder.append("[3]dataSetType").append(",").append(dataSetType).append("\n");
stringBuilder
- .append("M4_LSM_init_loadAllChunkMetadatas_ns")
+ .append("[3]M4_LSM_init_loadAllChunkMetadatas_ns_cnt")
.append(",")
.append(M4_LSM_init_loadAllChunkMetadatas_ns)
+ .append(",")
+ .append(M4_LSM_init_loadAllChunkMetadatas_count)
.append("\n");
stringBuilder
- .append("M4_LSM_merge_M4_time_span_ns")
+ .append("[3]M4_LSM_merge_M4_time_span_ns_cnt")
.append(",")
.append(M4_LSM_merge_M4_time_span_ns)
- .append("\n");
- stringBuilder.append("M4_LSM_FP_ns").append(",").append(M4_LSM_FP_ns).append("\n");
- stringBuilder.append("M4_LSM_LP_ns").append(",").append(M4_LSM_LP_ns).append("\n");
- stringBuilder.append("M4_LSM_BP_ns").append(",").append(M4_LSM_BP_ns).append("\n");
- stringBuilder.append("M4_LSM_TP_ns").append(",").append(M4_LSM_TP_ns).append("\n");
- stringBuilder
- .append("DCP_A_GET_CHUNK_METADATAS_ns")
.append(",")
- .append(DCP_A_GET_CHUNK_METADATAS_ns)
+ .append(M4_LSM_merge_M4_time_span_count)
.append("\n");
stringBuilder
- .append("DCP_B_READ_MEM_CHUNK_ns")
+ .append("[3]M4_LSM_FP_ns_cnt")
.append(",")
- .append(DCP_B_READ_MEM_CHUNK_ns)
- .append("\n");
- stringBuilder
- .append("DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_ns")
+ .append(M4_LSM_FP_ns)
.append(",")
- .append(DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_ns)
+ .append(M4_LSM_FP_count)
.append("\n");
stringBuilder
- .append("DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_ns")
+ .append("[3]M4_LSM_LP_ns_cnt")
.append(",")
- .append(DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_ns)
- .append("\n");
- stringBuilder
- .append("SEARCH_ARRAY_a_verifBPTP_ns")
+ .append(M4_LSM_LP_ns)
.append(",")
- .append(SEARCH_ARRAY_a_verifBPTP_ns)
+ .append(M4_LSM_LP_count)
.append("\n");
stringBuilder
- .append("SEARCH_ARRAY_b_genFP_ns")
+ .append("[3]M4_LSM_BP_ns_cnt")
.append(",")
- .append(SEARCH_ARRAY_b_genFP_ns)
- .append("\n");
- stringBuilder
- .append("SEARCH_ARRAY_b_genLP_ns")
+ .append(M4_LSM_BP_ns)
.append(",")
- .append(SEARCH_ARRAY_b_genLP_ns)
+ .append(M4_LSM_BP_count)
.append("\n");
stringBuilder
- .append("SEARCH_ARRAY_c_genBPTP_ns")
+ .append("[3]M4_LSM_TP_ns_cnt")
.append(",")
- .append(SEARCH_ARRAY_c_genBPTP_ns)
- .append("\n");
-
- stringBuilder
- .append("Server_Query_Execute_cnt")
+ .append(M4_LSM_TP_ns)
.append(",")
- .append(DCP_Server_Query_Execute_count)
+ .append(M4_LSM_TP_count)
.append("\n");
stringBuilder
- .append("Server_Query_Fetch_cnt")
+ .append("[4]DCP_A_GET_CHUNK_METADATAS_ns_cnt")
.append(",")
- .append(DCP_Server_Query_Fetch_count)
- .append("\n");
- stringBuilder
- .append("M4_LSM_init_loadAllChunkMetadatas_cnt")
- .append(",")
- .append(M4_LSM_init_loadAllChunkMetadatas_count)
- .append("\n");
- stringBuilder
- .append("M4_LSM_merge_M4_time_span_cnt")
- .append(",")
- .append(M4_LSM_merge_M4_time_span_count)
- .append("\n");
- stringBuilder.append("M4_LSM_FP_cnt").append(",").append(M4_LSM_FP_count).append("\n");
- stringBuilder.append("M4_LSM_LP_cnt").append(",").append(M4_LSM_LP_count).append("\n");
- stringBuilder.append("M4_LSM_BP_cnt").append(",").append(M4_LSM_BP_count).append("\n");
- stringBuilder.append("M4_LSM_TP_cnt").append(",").append(M4_LSM_TP_count).append("\n");
- stringBuilder
- .append("DCP_A_GET_CHUNK_METADATAS_cnt")
+ .append(DCP_A_GET_CHUNK_METADATAS_ns)
.append(",")
.append(DCP_A_GET_CHUNK_METADATAS_count)
.append("\n");
stringBuilder
- .append("DCP_B_READ_MEM_CHUNK_cnt")
+ .append("[4]DCP_B_READ_MEM_CHUNK_ns_cnt")
+ .append(",")
+ .append(DCP_B_READ_MEM_CHUNK_ns)
.append(",")
.append(DCP_B_READ_MEM_CHUNK_count)
.append("\n");
stringBuilder
- .append("DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_cnt")
+ .append("[4]DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_ns_cnt")
+ .append(",")
+ .append(DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_ns)
.append(",")
.append(DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_count)
.append("\n");
stringBuilder
- .append("DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_cnt")
+ .append("[4]DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_ns_cnt")
+ .append(",")
+ .append(DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_ns)
.append(",")
.append(DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_count)
.append("\n");
stringBuilder
- .append("SEARCH_ARRAY_a_verifBPTP_cnt")
+ .append("[4]SEARCH_ARRAY_a_verifBPTP_ns_cnt")
+ .append(",")
+ .append(SEARCH_ARRAY_a_verifBPTP_ns)
.append(",")
.append(SEARCH_ARRAY_a_verifBPTP_count)
.append("\n");
stringBuilder
- .append("SEARCH_ARRAY_b_genFP_cnt")
+ .append("[4]SEARCH_ARRAY_b_genFP_ns_cnt")
+ .append(",")
+ .append(SEARCH_ARRAY_b_genFP_ns)
.append(",")
.append(SEARCH_ARRAY_b_genFP_count)
.append("\n");
stringBuilder
- .append("SEARCH_ARRAY_b_genLP_cnt")
+ .append("[4]SEARCH_ARRAY_b_genLP_ns_cnt")
+ .append(",")
+ .append(SEARCH_ARRAY_b_genLP_ns)
.append(",")
.append(SEARCH_ARRAY_b_genLP_count)
.append("\n");
stringBuilder
- .append("SEARCH_ARRAY_c_genBPTP_cnt")
+ .append("[4]SEARCH_ARRAY_c_genBPTP_ns_cnt")
+ .append(",")
+ .append(SEARCH_ARRAY_c_genBPTP_ns)
.append(",")
.append(SEARCH_ARRAY_c_genBPTP_count)
.append("\n");
+
+ // stringBuilder
+ // .append("[2]Server_Query_Execute_cnt")
+ // .append(",")
+ // .append(DCP_Server_Query_Execute_count)
+ // .append("\n");
+ // stringBuilder
+ // .append("Server_Query_Fetch_cnt")
+ // .append(",")
+ // .append(DCP_Server_Query_Fetch_count)
+ // .append("\n");
+ // stringBuilder
+ // .append("M4_LSM_init_loadAllChunkMetadatas_cnt")
+ // .append(",")
+ // .append(M4_LSM_init_loadAllChunkMetadatas_count)
+ // .append("\n");
+ // stringBuilder
+ // .append("M4_LSM_merge_M4_time_span_cnt")
+ // .append(",")
+ // .append(M4_LSM_merge_M4_time_span_count)
+ // .append("\n");
+ // stringBuilder.append("M4_LSM_FP_cnt").append(",").append(M4_LSM_FP_count).append("\n");
+ // stringBuilder.append("M4_LSM_LP_cnt").append(",").append(M4_LSM_LP_count).append("\n");
+ // stringBuilder.append("M4_LSM_BP_cnt").append(",").append(M4_LSM_BP_count).append("\n");
+ // stringBuilder.append("M4_LSM_TP_cnt").append(",").append(M4_LSM_TP_count).append("\n");
+ // stringBuilder
+ // .append("DCP_A_GET_CHUNK_METADATAS_cnt")
+ // .append(",")
+ // .append(DCP_A_GET_CHUNK_METADATAS_count)
+ // .append("\n");
+ // stringBuilder
+ // .append("DCP_B_READ_MEM_CHUNK_cnt")
+ // .append(",")
+ // .append(DCP_B_READ_MEM_CHUNK_count)
+ // .append("\n");
+ // stringBuilder
+ // .append("DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_cnt")
+ // .append(",")
+ // .append(DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_count)
+ // .append("\n");
+ // stringBuilder
+ // .append("DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_cnt")
+ // .append(",")
+ // .append(DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_count)
+ // .append("\n");
+ // stringBuilder
+ // .append("SEARCH_ARRAY_a_verifBPTP_cnt")
+ // .append(",")
+ // .append(SEARCH_ARRAY_a_verifBPTP_count)
+ // .append("\n");
+ // stringBuilder
+ // .append("SEARCH_ARRAY_b_genFP_cnt")
+ // .append(",")
+ // .append(SEARCH_ARRAY_b_genFP_count)
+ // .append("\n");
+ // stringBuilder
+ // .append("SEARCH_ARRAY_b_genLP_cnt")
+ // .append(",")
+ // .append(SEARCH_ARRAY_b_genLP_count)
+ // .append("\n");
+ // stringBuilder
+ // .append("SEARCH_ARRAY_c_genBPTP_cnt")
+ // .append(",")
+ // .append(SEARCH_ARRAY_c_genBPTP_count)
+ // .append("\n");
stringBuilder
- .append("DCP_D_traversedPointNum")
+ .append("[4]DCP_D_traversedPointNum")
.append(",")
.append(DCP_D_traversedPointNum)
.append("\n");
[iotdb] 18/46: fix session fetchsize bug
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit cdc4fc55c33e9e72e104c2c36ccd91b55af51096
Author: Lei Rui <10...@qq.com>
AuthorDate: Thu Jun 15 16:39:31 2023 +0800
fix session fetchsize bug
---
client-py/iotdb/Session.py | 1 +
client-py/iotdb/utils/SessionDataSet.py | 3 +-
.../apache/iotdb/session/SessionConnection.java | 6 ++-
.../org/apache/iotdb/session/SessionDataSet.java | 57 ++++++++++++++++++++++
4 files changed, 64 insertions(+), 3 deletions(-)
diff --git a/client-py/iotdb/Session.py b/client-py/iotdb/Session.py
index 7e861ffb247..eb90edea588 100644
--- a/client-py/iotdb/Session.py
+++ b/client-py/iotdb/Session.py
@@ -660,6 +660,7 @@ class Session(object):
self.__session_id,
resp.queryDataSet,
resp.ignoreTimeStamp,
+ self.__fetch_size
)
def execute_non_query_statement(self, sql):
diff --git a/client-py/iotdb/utils/SessionDataSet.py b/client-py/iotdb/utils/SessionDataSet.py
index 7d49c5e691f..f8946cd3c07 100644
--- a/client-py/iotdb/utils/SessionDataSet.py
+++ b/client-py/iotdb/utils/SessionDataSet.py
@@ -43,6 +43,7 @@ class SessionDataSet(object):
session_id,
query_data_set,
ignore_timestamp,
+ fetch_size
):
self.iotdb_rpc_data_set = IoTDBRpcDataSet(
sql,
@@ -55,7 +56,7 @@ class SessionDataSet(object):
statement_id,
session_id,
query_data_set,
- 1024,
+ fetch_size
)
def get_fetch_size(self):
diff --git a/session/src/main/java/org/apache/iotdb/session/SessionConnection.java b/session/src/main/java/org/apache/iotdb/session/SessionConnection.java
index e51d1abe3d9..2d4e846f722 100644
--- a/session/src/main/java/org/apache/iotdb/session/SessionConnection.java
+++ b/session/src/main/java/org/apache/iotdb/session/SessionConnection.java
@@ -383,7 +383,8 @@ public class SessionConnection {
sessionId,
execResp.queryDataSet,
execResp.isIgnoreTimeStamp(),
- timeout);
+ timeout,
+ session.fetchSize);
}
protected void executeNonQueryStatement(String sql)
@@ -443,7 +444,8 @@ public class SessionConnection {
client,
sessionId,
execResp.queryDataSet,
- execResp.isIgnoreTimeStamp());
+ execResp.isIgnoreTimeStamp(),
+ session.fetchSize);
}
protected void insertRecord(TSInsertRecordReq request)
diff --git a/session/src/main/java/org/apache/iotdb/session/SessionDataSet.java b/session/src/main/java/org/apache/iotdb/session/SessionDataSet.java
index 2c72bd8fda8..bb0753223b1 100644
--- a/session/src/main/java/org/apache/iotdb/session/SessionDataSet.java
+++ b/session/src/main/java/org/apache/iotdb/session/SessionDataSet.java
@@ -71,6 +71,34 @@ public class SessionDataSet {
0);
}
+ public SessionDataSet(
+ String sql,
+ List<String> columnNameList,
+ List<String> columnTypeList,
+ Map<String, Integer> columnNameIndex,
+ long queryId,
+ long statementId,
+ TSIService.Iface client,
+ long sessionId,
+ TSQueryDataSet queryDataSet,
+ boolean ignoreTimeStamp,
+ int fetchSize) {
+ this.ioTDBRpcDataSet =
+ new IoTDBRpcDataSet(
+ sql,
+ columnNameList,
+ columnTypeList,
+ columnNameIndex,
+ ignoreTimeStamp,
+ queryId,
+ statementId,
+ client,
+ sessionId,
+ queryDataSet,
+ fetchSize,
+ 0);
+ }
+
public SessionDataSet(String finishResult) {
this.finishResult = finishResult;
this.ioTDBRpcDataSet =
@@ -117,6 +145,35 @@ public class SessionDataSet {
timeout);
}
+ public SessionDataSet(
+ String sql,
+ List<String> columnNameList,
+ List<String> columnTypeList,
+ Map<String, Integer> columnNameIndex,
+ long queryId,
+ long statementId,
+ TSIService.Iface client,
+ long sessionId,
+ TSQueryDataSet queryDataSet,
+ boolean ignoreTimeStamp,
+ long timeout,
+ int fetchSize) {
+ this.ioTDBRpcDataSet =
+ new IoTDBRpcDataSet(
+ sql,
+ columnNameList,
+ columnTypeList,
+ columnNameIndex,
+ ignoreTimeStamp,
+ queryId,
+ statementId,
+ client,
+ sessionId,
+ queryDataSet,
+ fetchSize,
+ timeout);
+ }
+
public int getFetchSize() {
return ioTDBRpcDataSet.fetchSize;
}
[iotdb] 25/46: exportCsv notimeout and fix
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 32254812dd316fd1cf884de8b95ae585e9abd61e
Author: Lei Rui <10...@qq.com>
AuthorDate: Tue Jun 20 20:52:18 2023 +0800
exportCsv notimeout and fix
---
cli/src/main/java/org/apache/iotdb/tool/ExportCsv.java | 2 +-
.../main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java | 1 -
2 files changed, 1 insertion(+), 2 deletions(-)
diff --git a/cli/src/main/java/org/apache/iotdb/tool/ExportCsv.java b/cli/src/main/java/org/apache/iotdb/tool/ExportCsv.java
index d645447db71..23cebcd6147 100644
--- a/cli/src/main/java/org/apache/iotdb/tool/ExportCsv.java
+++ b/cli/src/main/java/org/apache/iotdb/tool/ExportCsv.java
@@ -288,7 +288,7 @@ public class ExportCsv extends AbstractCsvTool {
private static void dumpResult(String sql, int index) {
final String path = targetDirectory + targetFile + index + ".csv";
try {
- SessionDataSet sessionDataSet = session.executeQueryStatement(sql, 10000);
+ SessionDataSet sessionDataSet = session.executeQueryStatement(sql, 0);
writeCsvFile(sessionDataSet, path);
sessionDataSet.closeOperationHandle();
System.out.println("Export completely!");
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
index 453fe2c6e2b..a43563e0410 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
@@ -350,7 +350,6 @@ public class ChunkSuit4CPV {
default:
throw new IOException("Unsupported data type!");
}
- return estimatedPos;
}
IOMonitor2.addMeasure(Operation.SEARCH_ARRAY_b_genFP, System.nanoTime() - start);
return estimatedPos;
[iotdb] 06/46: add log info for plan type
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit bf06145984beee6a75b38ffa19ee0334cd6be4a6
Author: Lei Rui <10...@qq.com>
AuthorDate: Mon Jun 12 15:11:19 2023 +0800
add log info for plan type
---
server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java | 3 +++
1 file changed, 3 insertions(+)
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
index cbd4f75eafa..6d7db03c876 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
@@ -524,12 +524,14 @@ public class PlanExecutor implements IPlanExecutor {
// no time series are selected, return EmptyDataSet
return new EmptyDataSet();
} else if (queryPlan instanceof UDTFPlan) {
+ logger.info("UDTFPlan");
UDTFPlan udtfPlan = (UDTFPlan) queryPlan;
queryDataSet = queryRouter.udtfQuery(udtfPlan, context);
} else if (queryPlan instanceof GroupByTimeFillPlan) {
GroupByTimeFillPlan groupByFillPlan = (GroupByTimeFillPlan) queryPlan;
queryDataSet = queryRouter.groupByFill(groupByFillPlan, context);
} else if (queryPlan instanceof GroupByTimePlan) {
+ logger.info("GroupByTimePlan");
GroupByTimePlan groupByTimePlan = (GroupByTimePlan) queryPlan;
queryDataSet = queryRouter.groupBy(groupByTimePlan, context);
} else if (queryPlan instanceof QueryIndexPlan) {
@@ -543,6 +545,7 @@ public class PlanExecutor implements IPlanExecutor {
} else if (queryPlan instanceof LastQueryPlan) {
queryDataSet = queryRouter.lastQuery((LastQueryPlan) queryPlan, context);
} else {
+ logger.info("RawDataQueryPlan");
queryDataSet = queryRouter.rawDataQuery((RawDataQueryPlan) queryPlan, context);
}
}
[iotdb] 40/46: add log
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit fcd4555e50609b9f318966ef1313fa6e8f0d8b1b
Author: Lei Rui <10...@qq.com>
AuthorDate: Fri Jul 7 11:52:04 2023 +0800
add log
---
.../dataset/groupby/LocalGroupByExecutor4CPV.java | 20 +++++++-----
.../tsfile/file/metadata/TimeseriesMetadata.java | 3 +-
.../file/metadata/statistics/Statistics.java | 36 ++++++++++++++++++----
.../file/metadata/statistics/ValueIndex.java | 3 ++
4 files changed, 47 insertions(+), 15 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java
index 294129523e3..f30f936663f 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java
@@ -153,7 +153,7 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
int BP_v_int = ((IntegerStatistics) statistics).getMinValue();
int TP_v_int = ((IntegerStatistics) statistics).getMaxValue();
M4_CHUNK_METADATA.debug(
- "M4_CHUNK_METADATA,{},{},{},{},{},{},{},{},{},{}",
+ "M4_CHUNK_METADATA,{},{},{},{},{},{},{},{},{},{},{}",
FP_t,
LP_t,
BP_t,
@@ -163,7 +163,8 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
BP_v_int,
TP_v_int,
chunkSuit4CPV.getChunkMetadata().getVersion(),
- chunkSuit4CPV.getChunkMetadata().getOffsetOfChunkHeader());
+ chunkSuit4CPV.getChunkMetadata().getOffsetOfChunkHeader(),
+ statistics.getCount());
break;
case INT64:
long FP_v_long = ((LongStatistics) statistics).getFirstValue();
@@ -171,7 +172,7 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
long BP_v_long = ((LongStatistics) statistics).getMinValue();
long TP_v_long = ((LongStatistics) statistics).getMaxValue();
M4_CHUNK_METADATA.debug(
- "M4_CHUNK_METADATA,{},{},{},{},{},{},{},{},{},{}",
+ "M4_CHUNK_METADATA,{},{},{},{},{},{},{},{},{},{},{}",
FP_t,
LP_t,
BP_t,
@@ -181,7 +182,8 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
BP_v_long,
TP_v_long,
chunkSuit4CPV.getChunkMetadata().getVersion(),
- chunkSuit4CPV.getChunkMetadata().getOffsetOfChunkHeader());
+ chunkSuit4CPV.getChunkMetadata().getOffsetOfChunkHeader(),
+ statistics.getCount());
break;
case FLOAT:
float FP_v_float = ((FloatStatistics) statistics).getFirstValue();
@@ -189,7 +191,7 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
float BP_v_float = ((FloatStatistics) statistics).getMinValue();
float TP_v_float = ((FloatStatistics) statistics).getMaxValue();
M4_CHUNK_METADATA.debug(
- "M4_CHUNK_METADATA,{},{},{},{},{},{},{},{},{},{}",
+ "M4_CHUNK_METADATA,{},{},{},{},{},{},{},{},{},{},{}",
FP_t,
LP_t,
BP_t,
@@ -199,7 +201,8 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
BP_v_float,
TP_v_float,
chunkSuit4CPV.getChunkMetadata().getVersion(),
- chunkSuit4CPV.getChunkMetadata().getOffsetOfChunkHeader());
+ chunkSuit4CPV.getChunkMetadata().getOffsetOfChunkHeader(),
+ statistics.getCount());
break;
case DOUBLE:
double FP_v_double = ((DoubleStatistics) statistics).getFirstValue();
@@ -207,7 +210,7 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
double BP_v_double = ((DoubleStatistics) statistics).getMinValue();
double TP_v_double = ((DoubleStatistics) statistics).getMaxValue();
M4_CHUNK_METADATA.debug(
- "M4_CHUNK_METADATA,{},{},{},{},{},{},{},{},{},{}",
+ "M4_CHUNK_METADATA,{},{},{},{},{},{},{},{},{},{},{}",
FP_t,
LP_t,
BP_t,
@@ -217,7 +220,8 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
BP_v_double,
TP_v_double,
chunkSuit4CPV.getChunkMetadata().getVersion(),
- chunkSuit4CPV.getChunkMetadata().getOffsetOfChunkHeader());
+ chunkSuit4CPV.getChunkMetadata().getOffsetOfChunkHeader(),
+ statistics.getCount());
break;
default:
throw new QueryProcessException("unsupported data type!");
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
index f8bea3156b4..e2301e15b95 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
@@ -131,7 +131,8 @@ public class TimeseriesMetadata implements Accountable {
byteLen += ReadWriteIOUtils.write(dataType, outputStream);
byteLen +=
ReadWriteForEncodingUtils.writeUnsignedVarInt(chunkMetaDataListDataSize, outputStream);
- byteLen += statistics.serialize(outputStream);
+ // log false to avoid interfere with chunk statistics log info
+ byteLen += statistics.serialize(outputStream, false);
chunkMetadataListBuffer.writeTo(outputStream);
byteLen += chunkMetadataListBuffer.size();
return byteLen;
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java
index 804a17d53d4..aa1d544205f 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java
@@ -135,13 +135,27 @@ public abstract class Statistics<T> {
// value statistics of different data type
byteLen += serializeStats(outputStream);
// serialize stepRegress
- byteLen += serializeStepRegress(outputStream);
+ byteLen += serializeStepRegress(outputStream, true);
// serialize value index
- byteLen += serializeValueIndex(outputStream);
+ byteLen += serializeValueIndex(outputStream, true);
return byteLen;
}
- int serializeValueIndex(OutputStream outputStream) throws IOException {
+ public int serialize(OutputStream outputStream, boolean log) throws IOException {
+ int byteLen = 0;
+ byteLen += ReadWriteForEncodingUtils.writeUnsignedVarInt(count, outputStream);
+ byteLen += ReadWriteIOUtils.write(startTime, outputStream);
+ byteLen += ReadWriteIOUtils.write(endTime, outputStream);
+ // value statistics of different data type
+ byteLen += serializeStats(outputStream);
+ // serialize stepRegress
+ byteLen += serializeStepRegress(outputStream, log);
+ // serialize value index
+ byteLen += serializeValueIndex(outputStream, log);
+ return byteLen;
+ }
+
+ int serializeValueIndex(OutputStream outputStream, boolean log) throws IOException {
valueIndex.learn(); // ensure executed once and only once
int byteLen = 0;
byteLen += ReadWriteIOUtils.write(valueIndex.idxOut.size(), outputStream);
@@ -155,7 +169,12 @@ public abstract class Statistics<T> {
byteLen += valueIndex.valueOut.size();
byteLen += ReadWriteIOUtils.write(valueIndex.errorBound, outputStream);
- LOG.info("value_index_serialize_byteLen,{}", byteLen);
+ if (log) {
+ LOG.info(
+ "value_index_serialize_byteLen,{},modelPoint_count_exceptFPvLPv,{}",
+ byteLen,
+ valueIndex.modelPointCount);
+ }
return byteLen;
}
@@ -163,7 +182,7 @@ public abstract class Statistics<T> {
* slope, m: the number of segment keys, m-2 segment keys in between when m>=2. The first and the
* last segment keys are not serialized here, because they are minTime and endTime respectively.
*/
- int serializeStepRegress(OutputStream outputStream) throws IOException {
+ int serializeStepRegress(OutputStream outputStream, boolean log) throws IOException {
stepRegress.learn(); // ensure executed once and only once
int byteLen = 0;
byteLen += ReadWriteIOUtils.write(stepRegress.getSlope(), outputStream); // K
@@ -173,7 +192,12 @@ public abstract class Statistics<T> {
for (int i = 1; i < segmentKeys.size() - 1; i++) { // t2,t3,...,tm-1
byteLen += ReadWriteIOUtils.write(segmentKeys.get(i), outputStream);
}
- LOG.info("time_index_serialize_byteLen,{}", byteLen);
+ if (log) {
+ LOG.info(
+ "time_index_serialize_byteLen,{},segmentKeys_size_includeFPtLPt,{}",
+ byteLen,
+ segmentKeys.size());
+ }
return byteLen;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/ValueIndex.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/ValueIndex.java
index 6cefa6088ec..bd16cdfc074 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/ValueIndex.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/ValueIndex.java
@@ -31,6 +31,8 @@ public class ValueIndex {
// causing learn() executed more than once!!
private boolean isLearned = false;
+ public int modelPointCount = 0; // except the first and last points
+
private double stdDev = 0; // standard deviation of intervals
private long count = 0;
private double sumX2 = 0.0;
@@ -92,6 +94,7 @@ public class ValueIndex {
// the first point value is stored as FirstValue in statistics, so here no need store the
// first point
// the last point won't be checked by the if SDT encode logic
+ modelPointCount++;
idxEncoder.encode((int) sdtEncoder.getTime(), idxOut);
valueEncoder.encode(sdtEncoder.getDoubleValue(), valueOut);
if (!hasDataToFlush) {
[iotdb] 29/46: add test
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 984bfebaf7d6f3223e9c059b4eb2f03f145105f9
Author: Lei Rui <10...@qq.com>
AuthorDate: Mon Jul 3 17:45:16 2023 +0800
add test
---
.../iotdb/tsfile/encoding/SDTEncoderTest.java | 204 +++++++++------------
1 file changed, 89 insertions(+), 115 deletions(-)
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SDTEncoderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SDTEncoderTest.java
index c703ed24d04..9f62a16a794 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SDTEncoderTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SDTEncoderTest.java
@@ -27,7 +27,6 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import org.apache.iotdb.tsfile.encoding.encoder.SDTEncoder;
-import org.apache.iotdb.tsfile.utils.Pair;
import org.junit.Test;
public class SDTEncoderTest {
@@ -113,117 +112,49 @@ public class SDTEncoderTest {
System.out.println("at=" + selectTimestamps + ";");
System.out.println("av=" + selectValues + ";");
- System.out.println("LB=av-e;\n"
- + "UB=av+e;\n"
- + "plot(t,v)\n"
- + "hold on,plot(t,v,'+')\n"
- + "hold on,plot(at,av,'r')\n"
- + "%hold on,plot(at,av,'r+')\n"
- + "hold on,plot(at,UB,'g')\n"
- + "%hold on,plot(at,UB,'g+')\n"
- + "hold on,plot(at,LB,'g')\n"
- + "%hold on,plot(at,LB,'g+')\n"
- + "\n"
- + "% [val,idx]=max(av-e)\n"
- + "% yline(val)\n"
- + "\n"
- + "[sortedX, sortedInds] = sort(LB,'descend');\n"
- + "% yline(sortedX(1))\n"
- + "% yline(sortedX(2))\n"
- + "\n"
- + "% 依次计算UB小于Nth-max-LB的点数\n"
+ System.out.println("LB=av-e;\n" + "UB=av+e;\n" + "plot(t,v)\n" + "hold on,plot(t,v,'+')\n"
+ + "hold on,plot(at,av,'r')\n" + "%hold on,plot(at,av,'r+')\n" + "hold on,plot(at,UB,'g')\n"
+ + "%hold on,plot(at,UB,'g+')\n" + "hold on,plot(at,LB,'g')\n"
+ + "%hold on,plot(at,LB,'g+')\n" + "\n" + "% [val,idx]=max(av-e)\n" + "% yline(val)\n" + "\n"
+ + "[sortedX, sortedInds] = sort(LB,'descend');\n" + "% yline(sortedX(1))\n"
+ + "% yline(sortedX(2))\n" + "\n" + "% 依次计算UB小于Nth-max-LB的点数\n"
+ "% case 1: 两端都小于threshold，则该段内的点的UB都小于threshold，都可以被剪枝\n"
+ "% case 2: 两端都大于等于threshold，则该段内的点的UB都大于等于threshold，则都不可以被剪枝\n"
+ "% case 3: 一端小于一端大于等于threshold，则该段内有部分点的UB小于threshold可以被剪枝\n"
- + "% (t1,v1), (t2,v2), t1<t2\n"
- + "% y=(t-t1)*(v2-v1)/(t2-t1)+v1\n"
+ + "% (t1,v1), (t2,v2), t1<t2\n" + "% y=(t-t1)*(v2-v1)/(t2-t1)+v1\n"
+ "% y<thresold ===> (t-t1)*(v2-v1)/(t2-t1)+v1<threshold\n"
+ "% 如果v2>=threshold>v1: 则t<(threshold-v1)*(t2-t1)/(v2-v1)+t1，于是[t1,(threshold-v1)*(t2-t1)/(v2-v1)+t1)内的点可剪枝\n"
+ "% 如果v2<threshold<=v1: 则t>(threshold-v1)*(t2-t1)/(v2-v1)+t1，于是((threshold-v1)*(t2-t1)/(v2-v1)+t1,t2]内的点可剪枝\n"
- + "rank=5\n"
- + "threshold=sortedX(rank)\n"
- + "hold on, yline(threshold);\n"
- + "hold on, plot(at(sortedInds(rank)),threshold,'o')\n"
- + "\n"
- + "prune_t=[]; % point\n"
- + "prune_v=[]; % point\n"
- + "prune_interval=[]; % interval\n"
- + "interval_start=-1;\n"
- + "for i=2:1:length(av)\n"
- + "\tt1=at(i-1);\n"
- + "\tt2=at(i);\n"
- + "\tv1=UB(i-1);\n"
- + "\tv2=UB(i);\n"
- + "\ta=[];\n"
- + "\tif v1<threshold && v2<threshold\n"
- + "\t\tif i<length(av)\n"
- + "\t\t\ta=t1:1:t2-1;\n"
- + "\t\telse \n"
- + "\t\t\ta=t1:1:t2;\n"
- + "\t\tend\n"
- + "\t\tif interval_start<0\n"
- + "\t\t\tinterval_start=t1;\n"
- + " end\n"
- + " interval_end=t2; % continuous\n"
- + "\telseif v1<threshold && v2>=threshold\n"
+ + "rank=1\n" + "threshold=sortedX(rank)\n" + "hold on, yline(threshold);\n"
+ + "hold on, plot(at(sortedInds(rank)),threshold,'o')\n" + "\n" + "prune_t=[]; % point\n"
+ + "prune_v=[]; % point\n" + "prune_interval=[]; % interval\n" + "interval_start=-1;\n"
+ + "for i=2:1:length(av)\n" + "\tt1=at(i-1);\n" + "\tt2=at(i);\n" + "\tv1=UB(i-1);\n"
+ + "\tv2=UB(i);\n" + "\ta=[];\n" + "\tif v1<threshold && v2<threshold\n"
+ + "\t\tif i<length(av)\n" + "\t\t\ta=t1:1:t2-1;\n" + "\t\telse \n" + "\t\t\ta=t1:1:t2;\n"
+ + "\t\tend\n" + "\t\tif interval_start<0\n" + "\t\t\tinterval_start=t1;\n" + " end\n"
+ + " interval_end=t2; % continuous\n" + "\telseif v1<threshold && v2>=threshold\n"
+ "\t\ta=t1:1:floor((threshold-v1)*(t2-t1)/(v2-v1)+t1); % no need -1 here\n"
- + "\t\tif interval_start<0\n"
- + "\t\t\tinterval_start=t1;\n"
- + " end\n"
+ + "\t\tif interval_start<0\n" + "\t\t\tinterval_start=t1;\n" + " end\n"
+ " prune_interval=[prune_interval;[interval_start,floor((threshold-v1)*(t2-t1)/(v2-v1)+t1)]];\n"
- + "\t\tinterval_start=-1; % discontinuous\n"
- + "\telseif v1>=threshold && v2<threshold\n"
- + "\t\tif i<length(av)\n"
- + "\t\t\ta=ceil((threshold-v1)*(t2-t1)/(v2-v1)+t1):1:t2-1;\n"
- + "\t\telse\n"
- + "\t\t\ta=ceil((threshold-v1)*(t2-t1)/(v2-v1)+t1):1:t2;\n"
- + "\t\tend\n"
- + "\t\tinterval_start=ceil((threshold-v1)*(t2-t1)/(v2-v1)+t1);\n"
- + "\t\tinterval_end=t2;\n"
- + "\tend\n"
- + "\tprune_t=[prune_t,a];\n"
- + "\tprune_v=[prune_v,(a-t1)*(v2-v1)/(t2-t1)+v1];\n"
- + "end\n"
- + "if interval_start>0\n"
- + "\tprune_interval=[prune_interval;[interval_start,interval_end]];\n"
- + "end\n"
- + "\n"
- + "hold on,plot(prune_t,prune_v,'b.')\n"
- + "disp(length(prune_t)/length(t))\n"
- + "\n"
+ + "\t\tinterval_start=-1; % discontinuous\n" + "\telseif v1>=threshold && v2<threshold\n"
+ + "\t\tif i<length(av)\n" + "\t\t\ta=ceil((threshold-v1)*(t2-t1)/(v2-v1)+t1):1:t2-1;\n"
+ + "\t\telse\n" + "\t\t\ta=ceil((threshold-v1)*(t2-t1)/(v2-v1)+t1):1:t2;\n" + "\t\tend\n"
+ + "\t\tinterval_start=ceil((threshold-v1)*(t2-t1)/(v2-v1)+t1);\n" + "\t\tinterval_end=t2;\n"
+ + "\tend\n" + "\tprune_t=[prune_t,a];\n"
+ + "\tprune_v=[prune_v,(a-t1)*(v2-v1)/(t2-t1)+v1];\n" + "end\n" + "if interval_start>0\n"
+ + "\tprune_interval=[prune_interval;[interval_start,interval_end]];\n" + "end\n" + "\n"
+ + "hold on,plot(prune_t,prune_v,'b.')\n" + "disp(length(prune_t)/length(t))\n" + "\n"
+ "%rectangle('Position', [at(1) av(1) at(2) av(2)], 'Facec',[0.5 1 0.5])\n"
- + "prune_interval\n"
- + "\n"
- + "%for i=1:1:length(prune_interval)\n"
- + "%\txline(prune_interval(i,1));\n"
- + "%\txline(prune_interval(i,2));\n"
- + "%end\n"
- + "\n"
- + "% https://se.mathworks.com/help/matlab/ref/xregion.html\n"
- + "% xregion\n"
- + "\n"
- + "for i=1:1:length(prune_interval)\n"
- + "\tx=prune_interval(i,1):1:prune_interval(i,2);\n"
- + "\ty=[];\n"
- + "\tfor j=1:1:length(x)\n"
- + "\t\tidx=x(j);\n"
- + "\t\tfor n=2:1:length(at)\n"
- + "\t\t\tif idx<=at(n)\n"
- + "\t\t\t\tt1=at(n-1);\n"
- + "\t\t\t\tt2=at(n);\n"
- + "\t\t\t\tv1=av(n-1);\n"
- + "\t\t\t\tv2=av(n);\n"
- + "\t\t\t\ty=[y,(idx-t1)*(v2-v1)/(t2-t1)+v1];\n"
- + "\t\t\t\tbreak;\n"
- + "\t\t\tend\n"
- + "\t\tend\n"
- + "\tend\n"
- + "\ty1=y-e;\n"
- + "\ty2=y+e;\n"
- + "\tL1=[x,fliplr(x)];\n"
- + "\tL2=[y1,fliplr(y2)];\n"
- + "\tfill(L1,L2,'b','FaceAlpha',0.1);\n"
- + "end");
+ + "prune_interval\n" + "\n" + "%for i=1:1:length(prune_interval)\n"
+ + "%\txline(prune_interval(i,1));\n" + "%\txline(prune_interval(i,2));\n" + "%end\n" + "\n"
+ + "% https://se.mathworks.com/help/matlab/ref/xregion.html\n" + "% xregion\n" + "\n"
+ + "for i=1:1:length(prune_interval)\n" + "\tx=prune_interval(i,1):1:prune_interval(i,2);\n"
+ + "\ty=[];\n" + "\tfor j=1:1:length(x)\n" + "\t\tidx=x(j);\n" + "\t\tfor n=2:1:length(at)\n"
+ + "\t\t\tif idx<=at(n)\n" + "\t\t\t\tt1=at(n-1);\n" + "\t\t\t\tt2=at(n);\n"
+ + "\t\t\t\tv1=av(n-1);\n" + "\t\t\t\tv2=av(n);\n"
+ + "\t\t\t\ty=[y,(idx-t1)*(v2-v1)/(t2-t1)+v1];\n" + "\t\t\t\tbreak;\n" + "\t\t\tend\n"
+ + "\t\tend\n" + "\tend\n" + "\ty1=y-e;\n" + "\ty2=y+e;\n" + "\tL1=[x,fliplr(x)];\n"
+ + "\tL2=[y1,fliplr(y2)];\n" + "\tfill(L1,L2,'b','FaceAlpha',0.1);\n" + "end");
}
@Test
@@ -275,24 +206,67 @@ public class SDTEncoderTest {
// 计算maxLB
long maxVal = Collections.max(selectValues);
- int maxIdx = selectTimestamps.get(selectValues.indexOf(maxVal));
- System.out.println(maxIdx);
- System.out.println(maxVal);
- double maxLB = maxVal - e;
+// int maxIdx = selectTimestamps.get(selectValues.indexOf(maxVal));
+ double threshold = maxVal - e; // maxLB
+ System.out.println("threshold(maxLB)=" + threshold);
// 计算UB<maxLB的剪枝段
- List<Pair<Integer, Integer>> prune_intervals = new ArrayList<>();
- int interval_start=-1;
+ List<Integer> prune_intervals_start = new ArrayList<>();
+ List<Integer> prune_intervals_end = new ArrayList<>();
+ int interval_start = -1;
+ int interval_end = -1;
for (int i = 1; i < selectTimestamps.size(); i++) {
- int t1=selectTimestamps.get(i-1);
- int t2=selectTimestamps.get(i);
- long v1=selectValues.get(i-1);
- long v2=selectValues.get(i);
-
+ int t1 = selectTimestamps.get(i - 1);
+ int t2 = selectTimestamps.get(i);
+ double v1 = selectValues.get(i - 1) + e; // UB
+ double v2 = selectValues.get(i) + e; // UB
+ if (v1 < threshold && v2 < threshold) {
+ if (interval_start < 0) {
+ interval_start = t1;
+ }
+ interval_end = t2; // continuous
+ } else if (v1 < threshold && v2 >= threshold) {
+ if (interval_start < 0) {
+ interval_start = t1;
+ }
+ prune_intervals_start.add(interval_start);
+ prune_intervals_end.add((int) Math.floor((threshold - v1) * (t2 - t1) / (v2 - v1) + t1));
+ interval_start = -1; // discontinuous
+ } else if (v1 >= threshold && v2 < threshold) {
+ interval_start = (int) Math.ceil((threshold - v1) * (t2 - t1) / (v2 - v1) + t1);
+ interval_end = t2; // continuous
+ }
+ }
+ if (interval_start > 0) {
+ prune_intervals_start.add(interval_start);
+ prune_intervals_end.add(interval_end);
}
+ System.out.println("UB<maxLB prune intervals (included ends):");
+ System.out.println(prune_intervals_start);
+ System.out.println(prune_intervals_end);
// 对剪枝段取余后搜索遍历计算TP
-
+ int startIdx = 1;
+ int endIdx = 200;
+ startIdx = startIdx - 1; // as excluded treated
+ endIdx = endIdx + 1; // as excluded treated
+ prune_intervals_start.add(endIdx); // turn into search_intervals_end (excluded)
+ prune_intervals_end.add(0, startIdx); // turn into search_intervals_start (excluded)
+ long candidateTPvalue = -1;
+ int candidateTPidx = -1;
+ for (int i = 0; i < prune_intervals_start.size(); i++) {
+ int search_interval_start = prune_intervals_end.get(i) + 1; // included
+ int search_interval_end = prune_intervals_start.get(i) - 1; // included
+ System.out.println(search_interval_start + "," + search_interval_end);
+ for (int j = search_interval_start; j <= search_interval_end; j++) {
+ long v = valueList.get(j);
+ if (v > candidateTPvalue) {
+ candidateTPvalue = v;
+ candidateTPidx = j;
+ }
+ }
+ }
+ System.out.println("TP=(" + candidateTPidx + "," + candidateTPvalue + ")");
}
@Test
[iotdb] 35/46: runnable
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 49cd2898b935f888216119d78e60a2594ab26b14
Author: Lei Rui <10...@qq.com>
AuthorDate: Wed Jul 5 16:52:43 2023 +0800
runnable
---
.../dataset/groupby/LocalGroupByExecutor4CPV.java | 78 +++++----
.../apache/iotdb/db/integration/m4/MyTest5.java | 2 +-
.../file/metadata/statistics/Statistics.java | 180 ++++++++-------------
.../file/metadata/statistics/ValueIndex.java | 9 +-
.../iotdb/tsfile/read/common/ChunkSuit4CPV.java | 39 ++---
.../iotdb/tsfile/read/common/ValuePoint.java | 2 +-
.../iotdb/tsfile/read/reader/page/PageReader.java | 151 +++++++++--------
.../iotdb/tsfile/encoding/SDTEncoderTest.java | 29 ++--
8 files changed, 237 insertions(+), 253 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java
index fd95d95c257..5223ec6bde2 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java
@@ -19,14 +19,6 @@
package org.apache.iotdb.db.query.dataset.groupby;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.ListIterator;
-import java.util.Map;
-import java.util.Set;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.exception.query.QueryProcessException;
@@ -53,9 +45,19 @@ import org.apache.iotdb.tsfile.read.filter.GroupByFilter;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.read.reader.page.PageReader;
import org.apache.iotdb.tsfile.utils.Pair;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Map;
+import java.util.Set;
+
/**
* Sql format: SELECT min_time(s0), max_time(s0), first_value(s0), last_value(s0), min_value(s0),
* max_value(s0) ROM root.xx group by ([tqs,tqe),IntervalLength). Requirements: (1) Don't change the
@@ -301,9 +303,9 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
/**
* @param curStartTime closed
- * @param curEndTime open
- * @param startTime closed
- * @param endTime open
+ * @param curEndTime open
+ * @param startTime closed
+ * @param endTime open
*/
@Override
public List<AggregateResult> calcResult(
@@ -354,11 +356,12 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
while (currentChunkList.size() > 0) { // loop 1
// sorted by bottomValue, find BP candidate set
// TODO double check the sort order logic for different aggregations
- currentChunkList.sort((o1, o2) -> {
- return ((Comparable) (o1.getStatistics().getMinValue())).compareTo(
- o2.getStatistics().getMinValue());
- // NOTE here get statistics from ChunkSuit4CPV, not from ChunkSuit4CPV.ChunkMetadata
- });
+ currentChunkList.sort(
+ (o1, o2) -> {
+ return ((Comparable) (o1.getStatistics().getMinValue()))
+ .compareTo(o2.getStatistics().getMinValue());
+ // NOTE here get statistics from ChunkSuit4CPV, not from ChunkSuit4CPV.ChunkMetadata
+ });
// NOTE here get statistics from ChunkSuit4CPV, not from ChunkSuit4CPV.ChunkMetadata
Object value = currentChunkList.get(0).getStatistics().getMinValue();
List<ChunkSuit4CPV> candidateSet = new ArrayList<>();
@@ -374,10 +377,15 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
// TODO check, whether nonLazyLoad remove affects candidateSet
List<ChunkSuit4CPV> nonLazyLoad = new ArrayList<>(candidateSet);
// TODO double check the sort order logic for version
- nonLazyLoad.sort((o1, o2) -> new MergeReaderPriority(o2.getChunkMetadata().getVersion(),
- o2.getChunkMetadata().getOffsetOfChunkHeader())
- .compareTo(new MergeReaderPriority(o1.getChunkMetadata().getVersion(),
- o1.getChunkMetadata().getOffsetOfChunkHeader())));
+ nonLazyLoad.sort(
+ (o1, o2) ->
+ new MergeReaderPriority(
+ o2.getChunkMetadata().getVersion(),
+ o2.getChunkMetadata().getOffsetOfChunkHeader())
+ .compareTo(
+ new MergeReaderPriority(
+ o1.getChunkMetadata().getVersion(),
+ o1.getChunkMetadata().getOffsetOfChunkHeader())));
while (true) { // loop 2
// if there is no chunk for lazy loading, then load all chunks in candidateSet,
// and apply deleteIntervals, deleting BP no matter out of deletion or update
@@ -471,7 +479,7 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
results
.get(4)
.updateResultUsingValues(
- new long[]{candidateTimestamp}, 1, new Object[]{candidateValue});
+ new long[] {candidateTimestamp}, 1, new Object[] {candidateValue});
return; // finished
} else if (!isUpdate) {
// verify whether the candidate point is updated
@@ -500,7 +508,7 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
results
.get(4)
.updateResultUsingValues(
- new long[]{candidateTimestamp}, 1, new Object[]{candidateValue});
+ new long[] {candidateTimestamp}, 1, new Object[] {candidateValue});
return; // finished
} else {
// the candidate point is updated, then label the chunk as already lazy loaded,
@@ -567,8 +575,8 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
new Comparator<ChunkSuit4CPV>() { // TODO double check the sort order logic for version
public int compare(ChunkSuit4CPV o1, ChunkSuit4CPV o2) {
return new MergeReaderPriority(
- o2.getChunkMetadata().getVersion(),
- o2.getChunkMetadata().getOffsetOfChunkHeader())
+ o2.getChunkMetadata().getVersion(),
+ o2.getChunkMetadata().getOffsetOfChunkHeader())
.compareTo(
new MergeReaderPriority(
o1.getChunkMetadata().getVersion(),
@@ -601,7 +609,7 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
.setDeleteIntervalList(chunkSuit4CPV.getChunkMetadata().getDeleteIntervalList());
}
// chunk data read operation (c): get all data points
-// chunkSuit4CPV.getPageReader().updateBPTP(chunkSuit4CPV);
+ // chunkSuit4CPV.getPageReader().updateBPTP(chunkSuit4CPV);
chunkSuit4CPV.getPageReader().updateTP_withValueIndex(chunkSuit4CPV); // TODO
// check if empty
if (chunkSuit4CPV.statistics.getCount() == 0) {
@@ -673,7 +681,7 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
results
.get(5)
.updateResultUsingValues(
- new long[]{candidateTimestamp}, 1, new Object[]{candidateValue});
+ new long[] {candidateTimestamp}, 1, new Object[] {candidateValue});
return; // finished
} else if (!isUpdate) {
// verify whether the candidate point is updated
@@ -702,7 +710,7 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
results
.get(5)
.updateResultUsingValues(
- new long[]{candidateTimestamp}, 1, new Object[]{candidateValue});
+ new long[] {candidateTimestamp}, 1, new Object[] {candidateValue});
return; // finished
} else {
// the candidate point is updated, then label the chunk as already lazy loaded,
@@ -749,8 +757,8 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
return res;
} else {
return new MergeReaderPriority(
- o2.getChunkMetadata().getVersion(),
- o2.getChunkMetadata().getOffsetOfChunkHeader())
+ o2.getChunkMetadata().getVersion(),
+ o2.getChunkMetadata().getOffsetOfChunkHeader())
.compareTo(
new MergeReaderPriority(
o1.getChunkMetadata().getVersion(),
@@ -827,11 +835,11 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
results
.get(0)
.updateResultUsingValues(
- new long[]{candidateTimestamp}, 1, new Object[]{candidateValue});
+ new long[] {candidateTimestamp}, 1, new Object[] {candidateValue});
results
.get(2)
.updateResultUsingValues(
- new long[]{candidateTimestamp}, 1, new Object[]{candidateValue});
+ new long[] {candidateTimestamp}, 1, new Object[] {candidateValue});
return;
}
}
@@ -860,8 +868,8 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
return res;
} else {
return new MergeReaderPriority(
- o2.getChunkMetadata().getVersion(),
- o2.getChunkMetadata().getOffsetOfChunkHeader())
+ o2.getChunkMetadata().getVersion(),
+ o2.getChunkMetadata().getOffsetOfChunkHeader())
.compareTo(
new MergeReaderPriority(
o1.getChunkMetadata().getVersion(),
@@ -939,11 +947,11 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
results
.get(1)
.updateResultUsingValues(
- new long[]{candidateTimestamp}, 1, new Object[]{candidateValue});
+ new long[] {candidateTimestamp}, 1, new Object[] {candidateValue});
results
.get(3)
.updateResultUsingValues(
- new long[]{candidateTimestamp}, 1, new Object[]{candidateValue});
+ new long[] {candidateTimestamp}, 1, new Object[] {candidateValue});
return;
}
}
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest5.java b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest5.java
index d7803eac00f..253dc40dd16 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest5.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest5.java
@@ -99,7 +99,7 @@ public class MyTest5 {
Statement statement = connection.createStatement()) {
statement.execute("SET STORAGE GROUP TO root.m4");
statement.execute("CREATE TIMESERIES root.m4.d1.s1 with datatype=double,encoding=PLAIN");
- statement.execute("CREATE TIMESERIES root.m4.d1.s2 with datatype=INT32,encoding=PLAIN");
+ statement.execute("CREATE TIMESERIES root.m4.d1.s2 with datatype=INT64,encoding=PLAIN");
} catch (SQLException throwable) {
fail(throwable.getMessage());
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java
index 5c0e87cf93d..f823fc2e94f 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java
@@ -18,12 +18,6 @@
*/
package org.apache.iotdb.tsfile.file.metadata.statistics;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.util.Collections;
-import java.util.Objects;
import org.apache.iotdb.tsfile.encoding.decoder.Decoder;
import org.apache.iotdb.tsfile.encoding.decoder.DeltaBinaryDecoder.IntDeltaDecoder;
import org.apache.iotdb.tsfile.encoding.decoder.DoublePrecisionDecoderV2;
@@ -34,15 +28,23 @@ import org.apache.iotdb.tsfile.read.common.ValuePoint;
import org.apache.iotdb.tsfile.utils.Binary;
import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
import org.eclipse.collections.impl.list.mutable.primitive.DoubleArrayList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.Objects;
+
/**
* This class is used for recording statistic information of each measurement in a delta file. While
* writing processing, the processor records the statistics information. Statistics includes
- * maximum, minimum and null value count up to version 0.0.1.<br> Each data type extends this
- * Statistic as super class.<br>
+ * maximum, minimum and null value count up to version 0.0.1.<br>
+ * Each data type extends this Statistic as super class.<br>
* <br>
* For the statistics in the Unseq file TimeSeriesMetadata, only firstValue, lastValue, startTime
* and endTime can be used.</br>
@@ -55,9 +57,7 @@ public abstract class Statistics<T> {
*/
protected boolean isEmpty = true;
- /**
- * number of time-value points
- */
+ /** number of time-value points */
private int count = 0;
private long startTime = Long.MAX_VALUE;
@@ -67,9 +67,7 @@ public abstract class Statistics<T> {
public ValueIndex valueIndex = new ValueIndex();
- /**
- * @author Yuyuan Kang
- */
+ /** @author Yuyuan Kang */
final String OPERATION_NOT_SUPPORT_FORMAT = "%s statistics does not support operation: %s";
/**
@@ -147,14 +145,14 @@ public abstract class Statistics<T> {
valueIndex.learn(); // ensure executed once and only once
int byteLen = 0;
byteLen += ReadWriteIOUtils.write(valueIndex.idxOut.size(), outputStream);
- outputStream.write(
- valueIndex.idxOut.getBuf(), 0, valueIndex.idxOut.size()); // NOTE len is important
- byteLen += valueIndex.idxOut.getBuf().length;
+ // NOTE use size instead of buffer is important
+ outputStream.write(valueIndex.idxOut.getBuf(), 0, valueIndex.idxOut.size());
+ byteLen += valueIndex.idxOut.size();
byteLen += ReadWriteIOUtils.write(valueIndex.valueOut.size(), outputStream);
- outputStream.write(
- valueIndex.valueOut.getBuf(), 0, valueIndex.valueOut.size()); // NOTE len is important
- byteLen += valueIndex.valueOut.getBuf().length;
+ // NOTE use size instead of buffer is important
+ outputStream.write(valueIndex.valueOut.getBuf(), 0, valueIndex.valueOut.size());
+ byteLen += valueIndex.valueOut.size();
byteLen += ReadWriteIOUtils.write(valueIndex.errorBound, outputStream);
@@ -180,23 +178,17 @@ public abstract class Statistics<T> {
abstract int serializeStats(OutputStream outputStream) throws IOException;
- /**
- * read data from the inputStream.
- */
+ /** read data from the inputStream. */
public abstract void deserialize(InputStream inputStream) throws IOException;
public abstract void deserialize(ByteBuffer byteBuffer);
// public abstract void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes);
- /**
- * @author Yuyuan Kang
- */
+ /** @author Yuyuan Kang */
public abstract MinMaxInfo<T> getMinInfo();
- /**
- * @author Yuyuan Kang
- */
+ /** @author Yuyuan Kang */
public abstract MinMaxInfo<T> getMaxInfo();
public abstract T getMinValue();
@@ -207,14 +199,10 @@ public abstract class Statistics<T> {
public abstract T getMaxValue();
- /**
- * @author Yuyuan Kang
- */
+ /** @author Yuyuan Kang */
public abstract long getBottomTimestamp();
- /**
- * @author Yuyuan Kang
- */
+ /** @author Yuyuan Kang */
public abstract long getTopTimestamp();
public abstract T getFirstValue();
@@ -290,9 +278,7 @@ public abstract class Statistics<T> {
updateStats(value);
}
- /**
- * @author Yuyuan Kang
- */
+ /** @author Yuyuan Kang */
public void update(long time, int value) {
count++;
if (time < this.startTime) {
@@ -307,9 +293,7 @@ public abstract class Statistics<T> {
updateStats(value, time);
}
- /**
- * @author Yuyuan Kang
- */
+ /** @author Yuyuan Kang */
public void update(long time, long value) {
count++;
if (time < this.startTime) {
@@ -323,9 +307,7 @@ public abstract class Statistics<T> {
updateStats(value, time);
}
- /**
- * @author Yuyuan Kang
- */
+ /** @author Yuyuan Kang */
public void update(long time, float value) {
count++;
if (time < this.startTime) {
@@ -339,9 +321,7 @@ public abstract class Statistics<T> {
updateStats(value, time);
}
- /**
- * @author Yuyuan Kang
- */
+ /** @author Yuyuan Kang */
public void update(long time, double value) {
count++;
if (time < this.startTime) {
@@ -379,9 +359,7 @@ public abstract class Statistics<T> {
updateStats(values, batchSize);
}
- /**
- * @author Yuyuan Kang
- */
+ /** @author Yuyuan Kang */
public void update(long[] time, int[] values, int batchSize) {
count += batchSize;
if (time[0] < startTime) {
@@ -395,9 +373,7 @@ public abstract class Statistics<T> {
updateStats(values, time, batchSize);
}
- /**
- * @author Yuyuan Kang
- */
+ /** @author Yuyuan Kang */
public void update(long[] time, long[] values, int batchSize) {
count += batchSize;
if (time[0] < startTime) {
@@ -411,9 +387,7 @@ public abstract class Statistics<T> {
updateStats(values, time, batchSize);
}
- /**
- * @author Yuyuan Kang
- */
+ /** @author Yuyuan Kang */
public void update(long[] time, float[] values, int batchSize) {
count += batchSize;
if (time[0] < startTime) {
@@ -427,9 +401,7 @@ public abstract class Statistics<T> {
updateStats(values, time, batchSize);
}
- /**
- * @author Yuyuan Kang
- */
+ /** @author Yuyuan Kang */
public void update(long[] time, double[] values, int batchSize) {
count += batchSize;
if (time[0] < startTime) {
@@ -465,14 +437,10 @@ public abstract class Statistics<T> {
isEmpty = empty;
}
- /**
- * @author Yuyuan Kang
- */
+ /** @author Yuyuan Kang */
public abstract void updateMinInfo(T val, long timestamp);
- /**
- * @author Yuyuan Kang
- */
+ /** @author Yuyuan Kang */
public abstract void updateMaxInfo(T val, long timestamp);
void updateStats(boolean value) {
@@ -529,30 +497,22 @@ public abstract class Statistics<T> {
}
}
- /**
- * @author Yuyuan Kang
- */
+ /** @author Yuyuan Kang */
public void updateStats(int value, long timestamp) {
throw new UnsupportedOperationException();
}
- /**
- * @author Yuyuan Kang
- */
+ /** @author Yuyuan Kang */
public void updateStats(long value, long timestamp) {
throw new UnsupportedOperationException();
}
- /**
- * @author Yuyuan Kang
- */
+ /** @author Yuyuan Kang */
public void updateStats(float value, long timestamp) {
throw new UnsupportedOperationException();
}
- /**
- * @author Yuyuan Kang
- */
+ /** @author Yuyuan Kang */
public void updateStats(double value, long timestamp) {
throw new UnsupportedOperationException();
}
@@ -565,30 +525,22 @@ public abstract class Statistics<T> {
throw new UnsupportedOperationException();
}
- /**
- * @author Yuyuan Kang
- */
+ /** @author Yuyuan Kang */
void updateStats(int[] values, long[] timestamps, int batchSize) {
throw new UnsupportedOperationException();
}
- /**
- * @author Yuyuan Kang
- */
+ /** @author Yuyuan Kang */
void updateStats(long[] values, long[] timestamps, int batchSize) {
throw new UnsupportedOperationException();
}
- /**
- * @author Yuyuan Kang
- */
+ /** @author Yuyuan Kang */
void updateStats(float[] values, long[] timestamps, int batchSize) {
throw new UnsupportedOperationException();
}
- /**
- * @author Yuyuan Kang
- */
+ /** @author Yuyuan Kang */
void updateStats(double[] values, long[] timestamps, int batchSize) {
throw new UnsupportedOperationException();
}
@@ -601,7 +553,7 @@ public abstract class Statistics<T> {
* @param min min timestamp
* @param max max timestamp
* @author Yuyuan Kang This method with two parameters is only used by {@code unsequence} which
- * updates/inserts/deletes timestamp.
+ * updates/inserts/deletes timestamp.
*/
public void updateStats(long min, long bottomTimestamp, long max, long topTimestamp) {
throw new UnsupportedOperationException();
@@ -635,23 +587,23 @@ public abstract class Statistics<T> {
// add the first point
valueIndex.modelPointIdx_list.add(1);
switch (getType()) {
-// case INT32:
-// int intV = (int) getFirstValue();
-// valueIndex.modelPointVal_list.add((double) intV);
-// valueIndex.sortedModelPoints.add(new ValuePoint(1, (double) intV));
-// break;
+ // case INT32:
+ // int intV = (int) getFirstValue();
+ // valueIndex.modelPointVal_list.add((double) intV);
+ // valueIndex.sortedModelPoints.add(new ValuePoint(1, (double) intV));
+ // break;
case INT64:
- long longV = (long) getFirstValue();
+ long longV = (Long) getFirstValue();
valueIndex.modelPointVal_list.add((double) longV);
valueIndex.sortedModelPoints.add(new ValuePoint(1, (double) longV));
break;
-// case FLOAT:
-// float floatV = (float) getFirstValue();
-// valueIndex.modelPointVal_list.add((double) floatV);
-// valueIndex.sortedModelPoints.add(new ValuePoint(1, (double) floatV));
-// break;
+ // case FLOAT:
+ // float floatV = (float) getFirstValue();
+ // valueIndex.modelPointVal_list.add((double) floatV);
+ // valueIndex.sortedModelPoints.add(new ValuePoint(1, (double) floatV));
+ // break;
case DOUBLE:
- double doubleV = (double) getFirstValue();
+ double doubleV = (Double) getFirstValue();
valueIndex.modelPointVal_list.add(doubleV);
valueIndex.sortedModelPoints.add(new ValuePoint(1, doubleV));
break;
@@ -693,23 +645,23 @@ public abstract class Statistics<T> {
if (count >= 2) { // otherwise only one point no need to store again
valueIndex.modelPointIdx_list.add(count);
switch (getType()) {
-// case INT32:
-// int intV = (int) getLastValue();
-// valueIndex.modelPointVal_list.add((double) intV);
-// valueIndex.sortedModelPoints.add(new ValuePoint(count, (double) intV));
-// break;
+ // case INT32:
+ // int intV = (int) getLastValue();
+ // valueIndex.modelPointVal_list.add((double) intV);
+ // valueIndex.sortedModelPoints.add(new ValuePoint(count, (double) intV));
+ // break;
case INT64:
- long longV = (long) getLastValue();
+ long longV = (Long) getLastValue();
valueIndex.modelPointVal_list.add((double) longV);
valueIndex.sortedModelPoints.add(new ValuePoint(count, (double) longV));
break;
-// case FLOAT:
-// float floatV = (float) getLastValue();
-// valueIndex.modelPointVal_list.add((double) floatV);
-// valueIndex.sortedModelPoints.add(new ValuePoint(count, (double) floatV));
-// break;
+ // case FLOAT:
+ // float floatV = (float) getLastValue();
+ // valueIndex.modelPointVal_list.add((double) floatV);
+ // valueIndex.sortedModelPoints.add(new ValuePoint(count, (double) floatV));
+ // break;
case DOUBLE:
- double doubleV = (double) getLastValue();
+ double doubleV = (Double) getLastValue();
valueIndex.modelPointVal_list.add(doubleV);
valueIndex.sortedModelPoints.add(new ValuePoint(count, doubleV));
break;
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/ValueIndex.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/ValueIndex.java
index eb506480840..6cefa6088ec 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/ValueIndex.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/ValueIndex.java
@@ -1,15 +1,17 @@
package org.apache.iotdb.tsfile.file.metadata.statistics;
-import java.util.ArrayList;
-import java.util.List;
import org.apache.iotdb.tsfile.encoding.encoder.DeltaBinaryEncoder.IntDeltaEncoder;
import org.apache.iotdb.tsfile.encoding.encoder.DoublePrecisionEncoderV2;
import org.apache.iotdb.tsfile.encoding.encoder.SDTEncoder;
import org.apache.iotdb.tsfile.read.common.ValuePoint;
import org.apache.iotdb.tsfile.utils.PublicBAOS;
+
import org.eclipse.collections.impl.list.mutable.primitive.DoubleArrayList;
import org.eclipse.collections.impl.list.mutable.primitive.IntArrayList;
+import java.util.ArrayList;
+import java.util.List;
+
public class ValueIndex {
private DoubleArrayList values = new DoubleArrayList();
@@ -22,7 +24,8 @@ public class ValueIndex {
public IntArrayList modelPointIdx_list = new IntArrayList();
public DoubleArrayList modelPointVal_list = new DoubleArrayList();
- public List<ValuePoint> sortedModelPoints = new ArrayList<>(); // sorted by value in ascending order
+ public List<ValuePoint> sortedModelPoints =
+ new ArrayList<>(); // sorted by value in ascending order
// this is necessary, otherwise serialized twice by timeseriesMetadata and chunkMetadata
// causing learn() executed more than once!!
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
index 3b5fd251521..4c1e205781e 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
@@ -19,7 +19,6 @@
package org.apache.iotdb.tsfile.read.common;
-import java.io.IOException;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -33,10 +32,14 @@ import org.apache.iotdb.tsfile.file.metadata.statistics.StepRegress;
import org.apache.iotdb.tsfile.read.common.IOMonitor2.Operation;
import org.apache.iotdb.tsfile.read.reader.page.PageReader;
+import java.io.IOException;
+
public class ChunkSuit4CPV {
private ChunkMetadata chunkMetadata; // fixed info, including version, dataType, stepRegress
- public int modelPointsCursor = 1; // starting from 0, pointing to the right end of the model segment, moving forward as processing time spans
+ public int modelPointsCursor =
+ 1; // starting from 0, pointing to the right end of the model segment, moving forward as
+ // processing time spans
public Statistics statistics; // dynamically updated, includes FP/LP/BP/TP info
@@ -282,10 +285,10 @@ public class ChunkSuit4CPV {
long timestamp = pageReader.timeBuffer.getLong(estimatedPos * 8);
statistics.setStartTime(timestamp);
switch (chunkMetadata.getDataType()) {
- // iotdb的int类型的plain编码用的是自制的不支持random access
- // case INT32:
- // return new MinMaxInfo(pageReader.valueBuffer.getInt(estimatedPos * 4),
- // pageReader.timeBuffer.getLong(estimatedPos * 8));
+ // iotdb的int类型的plain编码用的是自制的不支持random access
+ // case INT32:
+ // return new MinMaxInfo(pageReader.valueBuffer.getInt(estimatedPos * 4),
+ // pageReader.timeBuffer.getLong(estimatedPos * 8));
case INT64:
long longVal =
pageReader.valueBuffer.getLong(pageReader.timeBufferLength + estimatedPos * 8);
@@ -328,10 +331,10 @@ public class ChunkSuit4CPV {
long timestamp = pageReader.timeBuffer.getLong(estimatedPos * 8);
statistics.setStartTime(timestamp);
switch (chunkMetadata.getDataType()) {
- // iotdb的int类型的plain编码用的是自制的不支持random access
- // case INT32:
- // return new MinMaxInfo(pageReader.valueBuffer.getInt(estimatedPos * 4),
- // pageReader.timeBuffer.getLong(estimatedPos * 8));
+ // iotdb的int类型的plain编码用的是自制的不支持random access
+ // case INT32:
+ // return new MinMaxInfo(pageReader.valueBuffer.getInt(estimatedPos * 4),
+ // pageReader.timeBuffer.getLong(estimatedPos * 8));
case INT64:
long longVal =
pageReader.valueBuffer.getLong(pageReader.timeBufferLength + estimatedPos * 8);
@@ -399,10 +402,10 @@ public class ChunkSuit4CPV {
long timestamp = pageReader.timeBuffer.getLong(estimatedPos * 8);
statistics.setEndTime(timestamp);
switch (chunkMetadata.getDataType()) {
- // iotdb的int类型的plain编码用的是自制的不支持random access
- // case INT32:
- // return new MinMaxInfo(pageReader.valueBuffer.getInt(estimatedPos * 4),
- // pageReader.timeBuffer.getLong(estimatedPos * 8));
+ // iotdb的int类型的plain编码用的是自制的不支持random access
+ // case INT32:
+ // return new MinMaxInfo(pageReader.valueBuffer.getInt(estimatedPos * 4),
+ // pageReader.timeBuffer.getLong(estimatedPos * 8));
case INT64:
long longVal =
pageReader.valueBuffer.getLong(pageReader.timeBufferLength + estimatedPos * 8);
@@ -451,10 +454,10 @@ public class ChunkSuit4CPV {
long timestamp = pageReader.timeBuffer.getLong(estimatedPos * 8);
statistics.setEndTime(timestamp);
switch (chunkMetadata.getDataType()) {
- // iotdb的int类型的plain编码用的是自制的不支持random access
- // case INT32:
- // return new MinMaxInfo(pageReader.valueBuffer.getInt(estimatedPos * 4),
- // pageReader.timeBuffer.getLong(estimatedPos * 8));
+ // iotdb的int类型的plain编码用的是自制的不支持random access
+ // case INT32:
+ // return new MinMaxInfo(pageReader.valueBuffer.getInt(estimatedPos * 4),
+ // pageReader.timeBuffer.getLong(estimatedPos * 8));
case INT64:
long longVal =
pageReader.valueBuffer.getLong(pageReader.timeBufferLength + estimatedPos * 8);
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ValuePoint.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ValuePoint.java
index 2e672d7a134..62831c43196 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ValuePoint.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ValuePoint.java
@@ -38,4 +38,4 @@ public class ValuePoint implements Comparable<ValuePoint> {
public String toString() {
return "(" + index + "," + value + ")";
}
-}
\ No newline at end of file
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
index 979678ea5b7..db378202f56 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
@@ -18,11 +18,6 @@
*/
package org.apache.iotdb.tsfile.read.reader.page;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
import org.apache.iotdb.tsfile.encoding.decoder.Decoder;
import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException;
import org.apache.iotdb.tsfile.file.header.PageHeader;
@@ -46,39 +41,35 @@ import org.apache.iotdb.tsfile.read.reader.IPageReader;
import org.apache.iotdb.tsfile.utils.Binary;
import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
public class PageReader implements IPageReader {
private PageHeader pageHeader;
protected TSDataType dataType;
- /**
- * decoder for value column
- */
+ /** decoder for value column */
protected Decoder valueDecoder;
- /**
- * decoder for time column
- */
+ /** decoder for time column */
protected Decoder timeDecoder;
- /**
- * time column in memory
- */
+ /** time column in memory */
public ByteBuffer timeBuffer;
- /**
- * value column in memory
- */
+ /** value column in memory */
public ByteBuffer valueBuffer;
public int timeBufferLength;
protected Filter filter;
- /**
- * A list of deleted intervals.
- */
+ /** A list of deleted intervals. */
private List<TimeRange> deleteIntervalList;
private int deleteCursor = 0;
@@ -122,9 +113,7 @@ public class PageReader implements IPageReader {
valueBuffer.position(timeBufferLength);
}
- /**
- * the chunk partially overlaps in time with the current M4 interval Ii
- */
+ /** the chunk partially overlaps in time with the current M4 interval Ii */
public void split4CPV(
long startTime,
long endTime,
@@ -137,10 +126,10 @@ public class PageReader implements IPageReader {
// endTime is excluded so -1
int numberOfSpans =
(int)
- Math.floor(
- (Math.min(chunkMetadata.getEndTime(), endTime - 1) - curStartTime)
- * 1.0
- / interval)
+ Math.floor(
+ (Math.min(chunkMetadata.getEndTime(), endTime - 1) - curStartTime)
+ * 1.0
+ / interval)
+ 1;
for (int n = 0; n < numberOfSpans; n++) {
long leftEndIncluded = curStartTime + n * interval;
@@ -178,20 +167,23 @@ public class PageReader implements IPageReader {
}
public void updateTP_withValueIndex(ChunkSuit4CPV chunkSuit4CPV) {
- // NOTE: get valueIndex from chunkSuit4CPV.getChunkMetadata().getStatistics(), not chunkSuit4CPV.getStatistics()!
+ // NOTE: get valueIndex from chunkSuit4CPV.getChunkMetadata().getStatistics(), not
+ // chunkSuit4CPV.getStatistics()!
ValueIndex valueIndex = chunkSuit4CPV.getChunkMetadata().getStatistics().valueIndex;
// step 1: find threshold
- // iterate SDT points from value big to small to find the first point not deleted
boolean isFound = false;
double foundValue = 0;
- for (ValuePoint valuePoint : valueIndex.sortedModelPoints) {
+ // iterate SDT points from value big to small to find the first point not deleted
+ for (int n = valueIndex.sortedModelPoints.size() - 1; n >= 0; n--) { // NOTE from big to small
+ ValuePoint valuePoint = valueIndex.sortedModelPoints.get(n);
int idx = valuePoint.index; // index starting from 1
int pos = idx - 1; // pos starting from 0
long time = timeBuffer.getLong(pos * 8);
// check if deleted
- deleteCursor = 0; // TODO check
- if ((pos >= chunkSuit4CPV.startPos) && (pos <= chunkSuit4CPV.endPos) && !isDeleted(time)) {
+ if ((pos >= chunkSuit4CPV.startPos)
+ && (pos <= chunkSuit4CPV.endPos)
+ && !isDeleted_NoHistoryDeleteCursor(time)) {
// startPos&endPos conveys the virtual deletes of the current M4 time span
isFound = true;
foundValue = valuePoint.value;
@@ -208,23 +200,31 @@ public class PageReader implements IPageReader {
// increment global chunkSuit4CPV.modelPointsCursor
int idx2;
// note that the first and last points of a chunk are stored in model points
- // there must exist idx2-1 >= startPos, otherwise this chunk won't be processed for the current time span
- // there must exist idx1-1 <= endPos, otherwise this chunk won't be processed for the current time span
+ // there must exist idx2-1 >= startPos, otherwise this chunk won't be processed for the current
+ // time span
while ((idx2 = valueIndex.modelPointIdx_list.get(chunkSuit4CPV.modelPointsCursor)) - 1
< chunkSuit4CPV.startPos) { // TODO check
// -1 because idx starting from 1 while pos starting from 0
chunkSuit4CPV.modelPointsCursor++;
+ // pointing to the right end of the first model segment that passes the left endpoint of the
+ // current time span
}
- // increment local cursor starting from chunkSuit4CPV.modelPointsCursor for iterating model segments for the current time span
- // do not increment modelPointsCursor because the model segments for this time span may be iterated multiple times
- int localCursor = chunkSuit4CPV.modelPointsCursor;
+ // increment local cursor starting from chunkSuit4CPV.modelPointsCursor for iterating model
+ // segments for the current time span
+ // do not increment modelPointsCursor because the model segments for this time span may be
+ // iterated multiple times
+ int localCursor =
+ chunkSuit4CPV.modelPointsCursor; // pointing to the right end of the model segment
List<Integer> prune_intervals_start = new ArrayList<>();
List<Integer> prune_intervals_end = new ArrayList<>();
int interval_start = -1;
int interval_end = -1;
int idx1;
- while ((idx1 = valueIndex.modelPointIdx_list.get(localCursor - 1)) - 1
- <= chunkSuit4CPV.endPos) {
+ // there must exist idx1-1 <= endPos, otherwise this chunk won't be processed for the current
+ // time span
+ while (localCursor < valueIndex.modelPointIdx_list.size()
+ && (idx1 = valueIndex.modelPointIdx_list.get(localCursor - 1)) - 1
+ <= chunkSuit4CPV.endPos) {
idx2 = valueIndex.modelPointIdx_list.get(localCursor);
double v1_UB = valueIndex.modelPointVal_list.get(localCursor - 1) + valueIndex.errorBound;
double v2_UB = valueIndex.modelPointVal_list.get(localCursor) + valueIndex.errorBound;
@@ -242,8 +242,8 @@ public class PageReader implements IPageReader {
(int) Math.floor((threshold_LB - v1_UB) * (idx2 - idx1) / (v2_UB - v1_UB) + idx1));
interval_start = -1; // discontinuous
} else if (v1_UB >= threshold_LB && v2_UB < threshold_LB) {
- interval_start = (int) Math.ceil(
- (threshold_LB - v1_UB) * (idx2 - idx1) / (v2_UB - v1_UB) + idx1);
+ interval_start =
+ (int) Math.ceil((threshold_LB - v1_UB) * (idx2 - idx1) / (v2_UB - v1_UB) + idx1);
interval_end = idx2; // continuous
}
localCursor++;
@@ -273,31 +273,34 @@ public class PageReader implements IPageReader {
int prune_idx2 = prune_intervals_end.get(prune_intervals_end.size() - 1);
if (prune_idx2 - 1 >= search_endPos) {
// -1 for included, -1 for starting from 0
- search_endPos = Math.min(search_endPos,
- prune_intervals_start.get(prune_intervals_start.size() - 1) - 1 - 1);
+ search_endPos =
+ Math.min(
+ search_endPos, prune_intervals_start.get(prune_intervals_start.size() - 1) - 1 - 1);
prune_intervals_start.remove(prune_intervals_start.size() - 1);
prune_intervals_end.remove(prune_intervals_end.size() - 1);
}
}
// add search_endPos+1 to the end of prune_intervals_start
// turning into search_intervals_end (excluded endpoints)
- prune_intervals_start.add(search_endPos + 1);
+ // note that idx&prune_interval&search_interval starting from 1, pos starting from 0
+ prune_intervals_start.add(search_endPos + 1 + 1); // +1 for starting from 1, +1 for excluded
// add search_startPos-1 to the start of prune_intervals_end
// turning into search_intervals_start (excluded endpoints)
- prune_intervals_end.add(0, search_startPos - 1);
+ prune_intervals_end.add(0, search_startPos + 1 - 1); // +1 for starting from 1, +1 for excluded
// step 4: search unpruned intervals
// TODO deal with normal delete intervals
if (dataType == TSDataType.DOUBLE) {
double candidateTPvalue = -1;
long candidateTPtime = -1;
+ // note that idx&prune_interval&search_interval starting from 1, pos starting from 0
for (int i = 0; i < prune_intervals_start.size(); i++) {
int search_interval_start = prune_intervals_end.get(i) + 1; // included
int search_interval_end = prune_intervals_start.get(i) - 1; // included
- for (int j = search_interval_start; j <= search_interval_end; j++) { // starting from 1
- double v = valueBuffer.getDouble(timeBufferLength + (j - 1) * 8);
- long t = timeBuffer.getLong((j - 1) * 8);
- if (v > candidateTPvalue && !isDeleted(t)) {
+ for (int j = search_interval_start; j <= search_interval_end; j++) { // idx starting from 1
+ double v = valueBuffer.getDouble(timeBufferLength + (j - 1) * 8); // pos starting from 0
+ long t = timeBuffer.getLong((j - 1) * 8); // pos starting from 0
+ if (v > candidateTPvalue && !isDeleted_NoHistoryDeleteCursor(t)) {
candidateTPvalue = v;
candidateTPtime = t;
}
@@ -305,7 +308,7 @@ public class PageReader implements IPageReader {
}
chunkSuit4CPV.statistics.setMaxInfo(new MinMaxInfo(candidateTPvalue, candidateTPtime));
} else if (dataType == TSDataType.INT64) {
- long candidateTPvalue = -1;
+ long candidateTPvalue = -1; // NOTE for TP
long candidateTPtime = -1;
for (int i = 0; i < prune_intervals_start.size(); i++) {
int search_interval_start = prune_intervals_end.get(i) + 1; // included
@@ -313,7 +316,7 @@ public class PageReader implements IPageReader {
for (int j = search_interval_start; j <= search_interval_end; j++) { // starting from 1
long v = valueBuffer.getLong(timeBufferLength + (j - 1) * 8);
long t = timeBuffer.getLong((j - 1) * 8);
- if (v > candidateTPvalue && !isDeleted(t)) {
+ if (v > candidateTPvalue && !isDeleted_NoHistoryDeleteCursor(t)) {
candidateTPvalue = v;
candidateTPtime = t;
}
@@ -335,9 +338,9 @@ public class PageReader implements IPageReader {
case INT64:
statistics = new LongStatistics();
break;
-// case FLOAT:
-// statistics = new FloatStatistics();
-// break;
+ // case FLOAT:
+ // statistics = new FloatStatistics();
+ // break;
case DOUBLE:
statistics = new DoubleStatistics();
break;
@@ -361,16 +364,18 @@ public class PageReader implements IPageReader {
// only updateStats, actually only need to update BP and TP
}
break;
-// case FLOAT:
-// float aFloat = valueBuffer.getFloat(timeBufferLength + pos * 8);
-// if (!isDeleted(timestamp) && (filter == null || filter.satisfy(timestamp, aFloat))) {
-// // update statistics of chunkMetadata1
-// statistics.updateStats(aFloat, timestamp);
-// count++;
-// // ATTENTION: do not use update() interface which will also update StepRegress!
-// // only updateStats, actually only need to update BP and TP
-// }
-// break;
+ // case FLOAT:
+ // float aFloat = valueBuffer.getFloat(timeBufferLength + pos * 8);
+ // if (!isDeleted(timestamp) && (filter == null || filter.satisfy(timestamp,
+ // aFloat))) {
+ // // update statistics of chunkMetadata1
+ // statistics.updateStats(aFloat, timestamp);
+ // count++;
+ // // ATTENTION: do not use update() interface which will also update
+ // StepRegress!
+ // // only updateStats, actually only need to update BP and TP
+ // }
+ // break;
case DOUBLE:
double aDouble = valueBuffer.getDouble(timeBufferLength + pos * 8);
if (!isDeleted(timestamp) && (filter == null || filter.satisfy(timestamp, aDouble))) {
@@ -394,9 +399,7 @@ public class PageReader implements IPageReader {
IOMonitor2.addMeasure(Operation.SEARCH_ARRAY_c_genBPTP, System.nanoTime() - start);
}
- /**
- * @return the returned BatchData may be empty, but never be null
- */
+ /** @return the returned BatchData may be empty, but never be null */
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
@Override
public BatchData getAllSatisfiedPageData(boolean ascending) throws IOException {
@@ -493,6 +496,20 @@ public class PageReader implements IPageReader {
return false;
}
+ protected boolean isDeleted_NoHistoryDeleteCursor(long timestamp) {
+ deleteCursor = 0;
+ while (deleteIntervalList != null && deleteCursor < deleteIntervalList.size()) {
+ if (deleteIntervalList.get(deleteCursor).contains(timestamp)) {
+ return true;
+ } else if (deleteIntervalList.get(deleteCursor).getMax() < timestamp) {
+ deleteCursor++;
+ } else {
+ return false;
+ }
+ }
+ return false;
+ }
+
public static boolean isDeleted(long timestamp, List<TimeRange> deleteIntervalList) {
int deleteCursor = 0;
while (deleteIntervalList != null && deleteCursor < deleteIntervalList.size()) {
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SDTEncoderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SDTEncoderTest.java
index f859ddab83d..43cdc6d6ae5 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SDTEncoderTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SDTEncoderTest.java
@@ -19,7 +19,10 @@
package org.apache.iotdb.tsfile.encoding;
-import static org.junit.Assert.assertEquals;
+import org.apache.iotdb.tsfile.encoding.encoder.SDTEncoder;
+import org.apache.iotdb.tsfile.read.common.ValuePoint;
+
+import org.junit.Test;
import java.io.BufferedReader;
import java.io.FileReader;
@@ -27,9 +30,8 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
-import org.apache.iotdb.tsfile.encoding.encoder.SDTEncoder;
-import org.apache.iotdb.tsfile.read.common.ValuePoint;
-import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
public class SDTEncoderTest {
@@ -233,8 +235,8 @@ public class SDTEncoderTest {
String csvData = "D:\\full-game\\BallSpeed.csv";
// double[] eList = new double[] {500000, 400000, 300000, 200000, 160000, 100000, 50000,
// 10000};
- double[] eList = new double[]{160000};
- int[] startList = new int[]{1, 200000, 300000, 400000, 500000, 600000, 700000, 800000, 900000};
+ double[] eList = new double[] {160000};
+ int[] startList = new int[] {1, 200000, 300000, 400000, 500000, 600000, 700000, 800000, 900000};
List<Double> elapsedTime_withValueIndex_list = new ArrayList<>();
List<Double> elapsedTime_withoutValueIndex_list = new ArrayList<>();
List<Double> traversedComplexity_list = new ArrayList<>();
@@ -302,12 +304,12 @@ public class SDTEncoderTest {
// ValuePoint[] myArray = (ValuePoint[]) points.toArray();
// Arrays.sort(points.toArray());
-// points.sort(
-// new Comparator<ValuePoint>() {
-// public int compare(ValuePoint o1, ValuePoint o2) {
-// return ((Comparable) (o1.value)).compareTo(o2.value);
-// }
-// });
+ // points.sort(
+ // new Comparator<ValuePoint>() {
+ // public int compare(ValuePoint o1, ValuePoint o2) {
+ // return ((Comparable) (o1.value)).compareTo(o2.value);
+ // }
+ // });
Collections.sort(points);
long startTime = System.nanoTime();
@@ -407,8 +409,7 @@ public class SDTEncoderTest {
elapsedTime_withoutValueIndex += elapsedTime;
// System.out.println("search without value index: " + elapsedTime / 1000000.0 + "
// ms");
- System.out.println("TP=(" + candidateTPidx_raw + "," + candidateTPvalue_raw +
- ")");
+ System.out.println("TP=(" + candidateTPidx_raw + "," + candidateTPvalue_raw + ")");
}
elapsedTime_withValueIndex_list.add(
elapsedTime_withValueIndex / startList.length / 1000000.0);
[iotdb] 07/46: add log info for plan type
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit dbeeaccfecbb86d29b1372319b9e8fa882b81d59
Author: Lei Rui <10...@qq.com>
AuthorDate: Mon Jun 12 16:07:52 2023 +0800
add log info for plan type
---
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 12 +-
.../apache/iotdb/db/integration/m4/MyTmpTest.java | 675 +++++++++++++++++++++
.../iotdb/tsfile/read/common/IOMonitor2.java | 12 +
3 files changed, 695 insertions(+), 4 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
index 6d7db03c876..a37e1ddf3a2 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
@@ -135,6 +135,8 @@ import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Field;
+import org.apache.iotdb.tsfile.read.common.IOMonitor2;
+import org.apache.iotdb.tsfile.read.common.IOMonitor2.DataSetType;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.common.RowRecord;
import org.apache.iotdb.tsfile.read.query.dataset.EmptyDataSet;
@@ -524,14 +526,14 @@ public class PlanExecutor implements IPlanExecutor {
// no time series are selected, return EmptyDataSet
return new EmptyDataSet();
} else if (queryPlan instanceof UDTFPlan) {
- logger.info("UDTFPlan");
+ IOMonitor2.dataSetType = DataSetType.UDTFAlignByTimeDataSet;
UDTFPlan udtfPlan = (UDTFPlan) queryPlan;
queryDataSet = queryRouter.udtfQuery(udtfPlan, context);
} else if (queryPlan instanceof GroupByTimeFillPlan) {
GroupByTimeFillPlan groupByFillPlan = (GroupByTimeFillPlan) queryPlan;
queryDataSet = queryRouter.groupByFill(groupByFillPlan, context);
} else if (queryPlan instanceof GroupByTimePlan) {
- logger.info("GroupByTimePlan");
+ IOMonitor2.dataSetType = DataSetType.GroupByWithoutValueFilterDataSet;
GroupByTimePlan groupByTimePlan = (GroupByTimePlan) queryPlan;
queryDataSet = queryRouter.groupBy(groupByTimePlan, context);
} else if (queryPlan instanceof QueryIndexPlan) {
@@ -545,7 +547,7 @@ public class PlanExecutor implements IPlanExecutor {
} else if (queryPlan instanceof LastQueryPlan) {
queryDataSet = queryRouter.lastQuery((LastQueryPlan) queryPlan, context);
} else {
- logger.info("RawDataQueryPlan");
+ IOMonitor2.dataSetType = DataSetType.RawQueryDataSetWithoutValueFilter;
queryDataSet = queryRouter.rawDataQuery((RawDataQueryPlan) queryPlan, context);
}
}
@@ -1045,7 +1047,9 @@ public class PlanExecutor implements IPlanExecutor {
Collections.sort(
tsfiles,
(o1, o2) -> {
- if (establishTime[o1] == establishTime[o2]) return 0;
+ if (establishTime[o1] == establishTime[o2]) {
+ return 0;
+ }
return establishTime[o1] < establishTime[o2] ? -1 : 1;
});
for (Integer i : tsfiles) {
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTmpTest.java b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTmpTest.java
new file mode 100644
index 00000000000..677c79ab535
--- /dev/null
+++ b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTmpTest.java
@@ -0,0 +1,675 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.integration.m4;
+
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.compaction.CompactionStrategy;
+import org.apache.iotdb.db.utils.EnvironmentUtils;
+import org.apache.iotdb.jdbc.Config;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.Locale;
+
+import static org.junit.Assert.fail;
+
+public class MyTmpTest {
+
+ private static final String TIMESTAMP_STR = "Time";
+
+ private static String[] creationSqls =
+ new String[] {
+ "SET STORAGE GROUP TO root.vehicle.d0",
+ "CREATE TIMESERIES root.vehicle.d0.s0 WITH DATATYPE=INT64, ENCODING=PLAIN",
+ // IoTDB int data type does not support plain encoding, so use long data type
+ };
+
+ private final String d0s0 = "root.vehicle.d0.s0";
+
+ private static final String insertTemplate =
+ "INSERT INTO root.vehicle.d0(timestamp,s0)" + " VALUES(%d,%d)";
+
+ private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
+ private static boolean originalEnableCPV;
+ private static boolean originalUseChunkIndex;
+
+ private static boolean originalUseMad;
+ private static CompactionStrategy originalCompactionStrategy;
+
+ @Before
+ public void setUp() throws Exception {
+ TSFileDescriptor.getInstance().getConfig().setTimeEncoder("PLAIN");
+ originalCompactionStrategy = config.getCompactionStrategy();
+ config.setCompactionStrategy(CompactionStrategy.NO_COMPACTION);
+
+ originalEnableCPV = config.isEnableCPV();
+ config.setEnableCPV(true); // CPV
+
+ originalUseChunkIndex = TSFileDescriptor.getInstance().getConfig().isUseChunkIndex();
+ TSFileDescriptor.getInstance().getConfig().setUseChunkIndex(false);
+
+ originalUseMad = TSFileDescriptor.getInstance().getConfig().isUseMad();
+ TSFileDescriptor.getInstance().getConfig().setUseMad(true);
+
+ EnvironmentUtils.envSetUp();
+ Class.forName(Config.JDBC_DRIVER_NAME);
+ config.setTimestampPrecision("ms");
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ EnvironmentUtils.cleanEnv();
+ config.setCompactionStrategy(originalCompactionStrategy);
+ config.setEnableCPV(originalEnableCPV);
+ TSFileDescriptor.getInstance().getConfig().setUseChunkIndex(originalUseChunkIndex);
+ TSFileDescriptor.getInstance().getConfig().setUseMad(originalUseMad);
+ }
+
+ @Test
+ public void test1() throws Exception {
+ prepareData1();
+
+ String[] res =
+ new String[] {
+ "0,1,20,5,20,5[1],30[10]",
+ "25,25,45,8,30,8[25],40[30]",
+ "50,52,54,8,18,8[52],18[54]",
+ "75,null,null,null,null,null,null"
+ };
+ try (Connection connection =
+ DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+ boolean hasResultSet =
+ statement.execute(
+ "SELECT M4(s0,'tqs'='0','tqe'='100','w'='4')"
+ + " FROM root.vehicle.d0 where time>=0 and time<100");
+
+ Assert.assertTrue(hasResultSet);
+ try (ResultSet resultSet = statement.getResultSet()) {
+ int i = 0;
+ while (resultSet.next()) {
+ String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(1);
+ System.out.println(ans);
+ // Assert.assertEquals(res[i++], ans);
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void test5() throws Exception {
+ prepareData5();
+
+ String[] res =
+ new String[] {
+ "0,1,20,5,20,5[1],30[10]",
+ "25,25,45,8,30,8[25],40[30]",
+ "50,null,null,null,null,null,null",
+ "75,null,null,null,null,null,null"
+ };
+ try (Connection connection =
+ DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+ boolean hasResultSet =
+ statement.execute(
+ "SELECT min_time(s0), max_time(s0), first_value(s0), last_value(s0), min_value(s0), max_value(s0)"
+ + " FROM root.vehicle.d0 group by ([0,100),25ms)");
+
+ Assert.assertTrue(hasResultSet);
+ try (ResultSet resultSet = statement.getResultSet()) {
+ int i = 0;
+ while (resultSet.next()) {
+ String ans =
+ resultSet.getString(TIMESTAMP_STR)
+ + ","
+ + resultSet.getString(String.format("min_time(%s)", d0s0))
+ + ","
+ + resultSet.getString(String.format("max_time(%s)", d0s0))
+ + ","
+ + resultSet.getString(String.format("first_value(%s)", d0s0))
+ + ","
+ + resultSet.getString(String.format("last_value(%s)", d0s0))
+ + ","
+ + resultSet.getString(String.format("min_value(%s)", d0s0))
+ + ","
+ + resultSet.getString(String.format("max_value(%s)", d0s0));
+ System.out.println(ans);
+ Assert.assertEquals(res[i++], ans);
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ private static void prepareData1() {
+ // data:
+ // https://user-images.githubusercontent.com/33376433/151985070-73158010-8ba0-409d-a1c1-df69bad1aaee.png
+ try (Connection connection =
+ DriverManager.getConnection(
+ Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+
+ for (String sql : creationSqls) {
+ statement.execute(sql);
+ }
+
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 1, 5));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 2, 15));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 20, 1));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 25, 8));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 54, 3));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 120, 8));
+ statement.execute("FLUSH");
+
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 5, 10));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 8, 8));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 10, 30));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 20, 20));
+ statement.execute("FLUSH");
+
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 27, 20));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 30, 40));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 35, 10));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 40, 20));
+ statement.execute("FLUSH");
+
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 33, 9));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 45, 30));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 52, 8));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 54, 18));
+ statement.execute("FLUSH");
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ private static void prepareData5() {
+ // data:
+ // https://user-images.githubusercontent.com/33376433/151985070-73158010-8ba0-409d-a1c1-df69bad1aaee.png
+ try (Connection connection =
+ DriverManager.getConnection(
+ Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+
+ for (String sql : creationSqls) {
+ statement.execute(sql);
+ }
+
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 1, 5));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 2, 15));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 20, 1));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 25, 8));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 54, 3));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 120, 8));
+ statement.execute("FLUSH");
+
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 5, 10));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 8, 8));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 10, 30));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 20, 20));
+ statement.execute("FLUSH");
+
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 27, 20));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 30, 40));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 35, 10));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 40, 20));
+ statement.execute("FLUSH");
+
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 33, 9));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 45, 30));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 52, 8));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 54, 18));
+ statement.execute("FLUSH");
+
+ statement.execute("delete from root.vehicle.d0.s0 where time>=52 and time<=54");
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Test
+ public void test2() { // add deletes
+ prepareData2();
+
+ String[] res =
+ new String[] {
+ "0,1,20,5,20,5[1],30[10]",
+ "25,25,27,8,20,8[25],20[27]",
+ "50,null,null,null,null,null,null",
+ "75,null,null,null,null,null,null"
+ };
+ try (Connection connection =
+ DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+ boolean hasResultSet =
+ statement.execute(
+ "SELECT min_time(s0), max_time(s0), first_value(s0), last_value(s0), min_value(s0), max_value(s0)"
+ + " FROM root.vehicle.d0 group by ([0,100),25ms)"); // don't change the
+ // sequence!!!
+
+ Assert.assertTrue(hasResultSet);
+ try (ResultSet resultSet = statement.getResultSet()) {
+ int i = 0;
+ while (resultSet.next()) {
+ String ans =
+ resultSet.getString(TIMESTAMP_STR)
+ + ","
+ + resultSet.getString(String.format("min_time(%s)", d0s0))
+ + ","
+ + resultSet.getString(String.format("max_time(%s)", d0s0))
+ + ","
+ + resultSet.getString(String.format("first_value(%s)", d0s0))
+ + ","
+ + resultSet.getString(String.format("last_value(%s)", d0s0))
+ + ","
+ + resultSet.getString(String.format("min_value(%s)", d0s0))
+ + ","
+ + resultSet.getString(String.format("max_value(%s)", d0s0));
+ System.out.println(ans);
+ Assert.assertEquals(res[i++], ans);
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ private static void prepareData2() {
+ // data:
+ // https://user-images.githubusercontent.com/33376433/151995378-07a2f8df-5cac-499a-ae88-e3b017eee07a.png
+ try (Connection connection =
+ DriverManager.getConnection(
+ Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+
+ for (String sql : creationSqls) {
+ statement.execute(sql);
+ }
+
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 1, 5));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 2, 15));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 20, 1));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 25, 8));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 54, 3));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 120, 8));
+ statement.execute("FLUSH");
+
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 5, 10));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 8, 8));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 10, 30));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 20, 20));
+ statement.execute("FLUSH");
+
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 27, 20));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 30, 40));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 35, 10));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 40, 20));
+ statement.execute("FLUSH");
+
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 33, 9));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 45, 30));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 52, 8));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 54, 18));
+ statement.execute("FLUSH");
+
+ statement.execute("delete from root.vehicle.d0.s0 where time>=28 and time<=60");
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Test
+ public void test2_2() { // use data2 but change the sql from group by ([0,100),25ms) to group by
+ // ([0,150),25ms)
+ prepareData2();
+
+ String[] res =
+ new String[] {
+ "0,1,20,5,20,5[1],30[10]",
+ "25,25,27,8,20,8[25],20[27]",
+ "50,null,null,null,null,null,null",
+ "75,null,null,null,null,null,null",
+ "100,120,120,8,8,8[120],8[120]",
+ "125,null,null,null,null,null,null"
+ };
+ try (Connection connection =
+ DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+ boolean hasResultSet =
+ statement.execute(
+ "SELECT min_time(s0), max_time(s0), first_value(s0), last_value(s0), min_value(s0), max_value(s0)"
+ + " FROM root.vehicle.d0 group by ([0,150),25ms)"); // don't change the
+ // sequence!!!
+
+ Assert.assertTrue(hasResultSet);
+ try (ResultSet resultSet = statement.getResultSet()) {
+ int i = 0;
+ while (resultSet.next()) {
+ String ans =
+ resultSet.getString(TIMESTAMP_STR)
+ + ","
+ + resultSet.getString(String.format("min_time(%s)", d0s0))
+ + ","
+ + resultSet.getString(String.format("max_time(%s)", d0s0))
+ + ","
+ + resultSet.getString(String.format("first_value(%s)", d0s0))
+ + ","
+ + resultSet.getString(String.format("last_value(%s)", d0s0))
+ + ","
+ + resultSet.getString(String.format("min_value(%s)", d0s0))
+ + ","
+ + resultSet.getString(String.format("max_value(%s)", d0s0));
+ System.out.println(ans);
+ Assert.assertEquals(res[i++], ans);
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void test3() { // all seq
+ prepareData3();
+
+ String[] res =
+ new String[] {
+ "0,1,22,5,4,1[10],10[2]",
+ "25,30,40,8,2,2[40],8[30]",
+ "50,55,72,5,4,4[72],20[62]",
+ "75,80,90,11,1,1[90],11[80]"
+ };
+ try (Connection connection =
+ DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+ boolean hasResultSet =
+ statement.execute(
+ "SELECT min_time(s0), max_time(s0), first_value(s0), last_value(s0), min_value(s0), max_value(s0)"
+ + " FROM root.vehicle.d0 group by ([0,100),25ms)"); // don't change the
+ // sequence!!!
+
+ Assert.assertTrue(hasResultSet);
+ try (ResultSet resultSet = statement.getResultSet()) {
+ int i = 0;
+ while (resultSet.next()) {
+ String ans =
+ resultSet.getString(TIMESTAMP_STR)
+ + ","
+ + resultSet.getString(String.format("min_time(%s)", d0s0))
+ + ","
+ + resultSet.getString(String.format("max_time(%s)", d0s0))
+ + ","
+ + resultSet.getString(String.format("first_value(%s)", d0s0))
+ + ","
+ + resultSet.getString(String.format("last_value(%s)", d0s0))
+ + ","
+ + resultSet.getString(String.format("min_value(%s)", d0s0))
+ + ","
+ + resultSet.getString(String.format("max_value(%s)", d0s0));
+ System.out.println(ans);
+ Assert.assertEquals(res[i++], ans);
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ private static void prepareData3() {
+ // data:
+ // https://user-images.githubusercontent.com/33376433/152003603-6b4e7494-00ff-47e4-bf6e-cab3c8600ce2.png
+ try (Connection connection =
+ DriverManager.getConnection(
+ Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+
+ for (String sql : creationSqls) {
+ statement.execute(sql);
+ }
+
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 1, 5));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 2, 10));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 10, 1));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 20, 5));
+ statement.execute("FLUSH");
+
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 22, 4));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 30, 8));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 40, 2));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 55, 5));
+ statement.execute("FLUSH");
+
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 60, 15));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 62, 20));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 65, 8));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 70, 18));
+ statement.execute("FLUSH");
+
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 72, 4));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 80, 11));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 90, 1));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 105, 7));
+ statement.execute("FLUSH");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Test
+ public void test3_2() { // not seq but no overlap
+ prepareData3_2();
+
+ String[] res =
+ new String[] {
+ "0,1,22,5,4,1[10],10[2]",
+ "25,30,40,8,2,2[40],8[30]",
+ "50,55,72,5,4,4[72],20[62]",
+ "75,80,90,11,1,1[90],11[80]"
+ };
+ try (Connection connection =
+ DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+ boolean hasResultSet =
+ statement.execute(
+ "SELECT min_time(s0), max_time(s0), first_value(s0), last_value(s0), min_value(s0), max_value(s0)"
+ + " FROM root.vehicle.d0 group by ([0,100),25ms)"); // don't change the
+ // sequence!!!
+
+ Assert.assertTrue(hasResultSet);
+ try (ResultSet resultSet = statement.getResultSet()) {
+ int i = 0;
+ while (resultSet.next()) {
+ String ans =
+ resultSet.getString(TIMESTAMP_STR)
+ + ","
+ + resultSet.getString(String.format("min_time(%s)", d0s0))
+ + ","
+ + resultSet.getString(String.format("max_time(%s)", d0s0))
+ + ","
+ + resultSet.getString(String.format("first_value(%s)", d0s0))
+ + ","
+ + resultSet.getString(String.format("last_value(%s)", d0s0))
+ + ","
+ + resultSet.getString(String.format("min_value(%s)", d0s0))
+ + ","
+ + resultSet.getString(String.format("max_value(%s)", d0s0));
+ System.out.println(ans);
+ Assert.assertEquals(res[i++], ans);
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ private static void prepareData3_2() {
+ // data:
+ // https://user-images.githubusercontent.com/33376433/152003603-6b4e7494-00ff-47e4-bf6e-cab3c8600ce2.png
+ try (Connection connection =
+ DriverManager.getConnection(
+ Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+
+ for (String sql : creationSqls) {
+ statement.execute(sql);
+ }
+
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 72, 4));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 80, 11));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 90, 1));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 105, 7));
+ statement.execute("FLUSH");
+
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 1, 5));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 2, 10));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 10, 1));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 20, 5));
+ statement.execute("FLUSH");
+
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 22, 4));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 30, 8));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 40, 2));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 55, 5));
+ statement.execute("FLUSH");
+
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 60, 15));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 62, 20));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 65, 8));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 70, 18));
+ statement.execute("FLUSH");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Test
+ public void test4() { // delete sequence move forward
+ prepareData4();
+
+ String[] res =
+ new String[] {
+ "0,1,20,5,20,5[1],30[10]",
+ "25,25,45,8,30,8[25],30[45]",
+ "50,52,54,8,18,8[52],18[54]",
+ "75,null,null,null,null,null,null"
+ };
+ try (Connection connection =
+ DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+ boolean hasResultSet =
+ statement.execute(
+ "SELECT min_time(s0), max_time(s0), first_value(s0), last_value(s0), min_value(s0), max_value(s0)"
+ + " FROM root.vehicle.d0 group by ([0,100),25ms)"); // don't change the
+ // sequence!!!
+
+ Assert.assertTrue(hasResultSet);
+ try (ResultSet resultSet = statement.getResultSet()) {
+ int i = 0;
+ while (resultSet.next()) {
+ String ans =
+ resultSet.getString(TIMESTAMP_STR)
+ + ","
+ + resultSet.getString(String.format("min_time(%s)", d0s0))
+ + ","
+ + resultSet.getString(String.format("max_time(%s)", d0s0))
+ + ","
+ + resultSet.getString(String.format("first_value(%s)", d0s0))
+ + ","
+ + resultSet.getString(String.format("last_value(%s)", d0s0))
+ + ","
+ + resultSet.getString(String.format("min_value(%s)", d0s0))
+ + ","
+ + resultSet.getString(String.format("max_value(%s)", d0s0));
+ System.out.println(ans);
+ Assert.assertEquals(res[i++], ans);
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ private static void prepareData4() {
+ // data:
+ // https://user-images.githubusercontent.com/33376433/152006061-f1d95952-3f5c-4d88-b34e-45d3bb61b600.png
+ try (Connection connection =
+ DriverManager.getConnection(
+ Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+
+ for (String sql : creationSqls) {
+ statement.execute(sql);
+ }
+
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 1, 5));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 2, 15));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 20, 1));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 25, 8));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 54, 3));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 120, 8));
+ statement.execute("FLUSH");
+
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 5, 10));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 8, 8));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 10, 30));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 20, 20));
+ statement.execute("FLUSH");
+
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 27, 20));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 30, 40));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 35, 10));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 40, 20));
+ statement.execute("FLUSH");
+
+ statement.execute("delete from root.vehicle.d0.s0 where time>=28 and time<=42");
+
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 33, 9));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 45, 30));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 52, 8));
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, 54, 18));
+ statement.execute("FLUSH");
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
index 5af7b162f4c..d9eeaf30b6c 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
@@ -23,6 +23,12 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class IOMonitor2 {
+ public enum DataSetType { // dataSet, executor, reader, file
+ NONE,
+ RawQueryDataSetWithoutValueFilter,
+ UDTFAlignByTimeDataSet,
+ GroupByWithoutValueFilterDataSet
+ }
public enum Operation {
DCP_Server_Query_Execute("DCP_Server_Query_Execute"),
@@ -76,6 +82,8 @@ public class IOMonitor2 {
public static long DCP_D_traversedPointNum = 0;
+ public static DataSetType dataSetType = DataSetType.NONE;
+
private static final Logger DEBUG_LOGGER = LoggerFactory.getLogger("IOMonitor2");
private static void reset() {
@@ -101,6 +109,8 @@ public class IOMonitor2 {
DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_ns = 0;
DCP_D_traversedPointNum = 0;
+
+ dataSetType = DataSetType.NONE;
}
public static void addMeasure(Operation operation, long elapsedTimeInNanosecond) {
@@ -179,6 +189,8 @@ public class IOMonitor2 {
public static String print() {
StringBuilder stringBuilder = new StringBuilder();
+ stringBuilder.append("dataSetType").append(",").append(dataSetType).append("\n");
+
stringBuilder
.append("Server_Query_Execute_ns")
.append(",")
[iotdb] 34/46: ni
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit c7dfd4887e65f00346c7aa5f60baf9b72c565b71
Author: Lei Rui <10...@qq.com>
AuthorDate: Wed Jul 5 01:47:54 2023 +0800
ni
---
.../dataset/groupby/LocalGroupByExecutor4CPV.java | 94 ++++-----
.../file/metadata/statistics/Statistics.java | 175 +++++++++++-----
.../file/metadata/statistics/ValueIndex.java | 6 +-
.../iotdb/tsfile/read/common/ChunkSuit4CPV.java | 36 ++--
.../iotdb/tsfile/read/common/IOMonitor2.java | 16 --
.../iotdb/tsfile/read/common/ValuePoint.java | 41 ++++
.../iotdb/tsfile/read/reader/page/PageReader.java | 227 ++++++++++++++++++---
.../iotdb/tsfile/encoding/SDTEncoderTest.java | 37 ++--
8 files changed, 439 insertions(+), 193 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java
index 55f7629159a..fd95d95c257 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java
@@ -19,6 +19,14 @@
package org.apache.iotdb.db.query.dataset.groupby;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Map;
+import java.util.Set;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.exception.query.QueryProcessException;
@@ -45,19 +53,9 @@ import org.apache.iotdb.tsfile.read.filter.GroupByFilter;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.read.reader.page.PageReader;
import org.apache.iotdb.tsfile.utils.Pair;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.ListIterator;
-import java.util.Map;
-import java.util.Set;
-
/**
* Sql format: SELECT min_time(s0), max_time(s0), first_value(s0), last_value(s0), min_value(s0),
* max_value(s0) ROM root.xx group by ([tqs,tqe),IntervalLength). Requirements: (1) Don't change the
@@ -253,7 +251,7 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
}
// iterate futureChunkList
- ListIterator itr = futureChunkList.listIterator();
+ ListIterator<ChunkSuit4CPV> itr = futureChunkList.listIterator();
while (itr.hasNext()) {
ChunkSuit4CPV chunkSuit4CPV = (ChunkSuit4CPV) (itr.next());
ChunkMetadata chunkMetadata = chunkSuit4CPV.getChunkMetadata();
@@ -303,9 +301,9 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
/**
* @param curStartTime closed
- * @param curEndTime open
- * @param startTime closed
- * @param endTime open
+ * @param curEndTime open
+ * @param startTime closed
+ * @param endTime open
*/
@Override
public List<AggregateResult> calcResult(
@@ -355,15 +353,12 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
// currentChunkList
while (currentChunkList.size() > 0) { // loop 1
// sorted by bottomValue, find BP candidate set
- currentChunkList.sort(
- new Comparator<ChunkSuit4CPV>() { // TODO double check the sort order logic for different
- // aggregations
- public int compare(ChunkSuit4CPV o1, ChunkSuit4CPV o2) {
- return ((Comparable) (o1.getStatistics().getMinValue()))
- .compareTo(o2.getStatistics().getMinValue());
- // NOTE here get statistics from ChunkSuit4CPV, not from ChunkSuit4CPV.ChunkMetadata
- }
- });
+ // TODO double check the sort order logic for different aggregations
+ currentChunkList.sort((o1, o2) -> {
+ return ((Comparable) (o1.getStatistics().getMinValue())).compareTo(
+ o2.getStatistics().getMinValue());
+ // NOTE here get statistics from ChunkSuit4CPV, not from ChunkSuit4CPV.ChunkMetadata
+ });
// NOTE here get statistics from ChunkSuit4CPV, not from ChunkSuit4CPV.ChunkMetadata
Object value = currentChunkList.get(0).getStatistics().getMinValue();
List<ChunkSuit4CPV> candidateSet = new ArrayList<>();
@@ -376,21 +371,13 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
}
}
- List<ChunkSuit4CPV> nonLazyLoad =
- new ArrayList<>(
- candidateSet); // TODO check, whether nonLazyLoad remove affects candidateSet
- nonLazyLoad.sort(
- new Comparator<ChunkSuit4CPV>() { // TODO double check the sort order logic for version
- public int compare(ChunkSuit4CPV o1, ChunkSuit4CPV o2) {
- return new MergeReaderPriority(
- o2.getChunkMetadata().getVersion(),
- o2.getChunkMetadata().getOffsetOfChunkHeader())
- .compareTo(
- new MergeReaderPriority(
- o1.getChunkMetadata().getVersion(),
- o1.getChunkMetadata().getOffsetOfChunkHeader()));
- }
- });
+ // TODO check, whether nonLazyLoad remove affects candidateSet
+ List<ChunkSuit4CPV> nonLazyLoad = new ArrayList<>(candidateSet);
+ // TODO double check the sort order logic for version
+ nonLazyLoad.sort((o1, o2) -> new MergeReaderPriority(o2.getChunkMetadata().getVersion(),
+ o2.getChunkMetadata().getOffsetOfChunkHeader())
+ .compareTo(new MergeReaderPriority(o1.getChunkMetadata().getVersion(),
+ o1.getChunkMetadata().getOffsetOfChunkHeader())));
while (true) { // loop 2
// if there is no chunk for lazy loading, then load all chunks in candidateSet,
// and apply deleteIntervals, deleting BP no matter out of deletion or update
@@ -484,7 +471,7 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
results
.get(4)
.updateResultUsingValues(
- new long[] {candidateTimestamp}, 1, new Object[] {candidateValue});
+ new long[]{candidateTimestamp}, 1, new Object[]{candidateValue});
return; // finished
} else if (!isUpdate) {
// verify whether the candidate point is updated
@@ -513,7 +500,7 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
results
.get(4)
.updateResultUsingValues(
- new long[] {candidateTimestamp}, 1, new Object[] {candidateValue});
+ new long[]{candidateTimestamp}, 1, new Object[]{candidateValue});
return; // finished
} else {
// the candidate point is updated, then label the chunk as already lazy loaded,
@@ -580,8 +567,8 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
new Comparator<ChunkSuit4CPV>() { // TODO double check the sort order logic for version
public int compare(ChunkSuit4CPV o1, ChunkSuit4CPV o2) {
return new MergeReaderPriority(
- o2.getChunkMetadata().getVersion(),
- o2.getChunkMetadata().getOffsetOfChunkHeader())
+ o2.getChunkMetadata().getVersion(),
+ o2.getChunkMetadata().getOffsetOfChunkHeader())
.compareTo(
new MergeReaderPriority(
o1.getChunkMetadata().getVersion(),
@@ -614,7 +601,8 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
.setDeleteIntervalList(chunkSuit4CPV.getChunkMetadata().getDeleteIntervalList());
}
// chunk data read operation (c): get all data points
- chunkSuit4CPV.getPageReader().updateBPTP(chunkSuit4CPV);
+// chunkSuit4CPV.getPageReader().updateBPTP(chunkSuit4CPV);
+ chunkSuit4CPV.getPageReader().updateTP_withValueIndex(chunkSuit4CPV); // TODO
// check if empty
if (chunkSuit4CPV.statistics.getCount() == 0) {
currentChunkList.remove(chunkSuit4CPV);
@@ -685,7 +673,7 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
results
.get(5)
.updateResultUsingValues(
- new long[] {candidateTimestamp}, 1, new Object[] {candidateValue});
+ new long[]{candidateTimestamp}, 1, new Object[]{candidateValue});
return; // finished
} else if (!isUpdate) {
// verify whether the candidate point is updated
@@ -714,7 +702,7 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
results
.get(5)
.updateResultUsingValues(
- new long[] {candidateTimestamp}, 1, new Object[] {candidateValue});
+ new long[]{candidateTimestamp}, 1, new Object[]{candidateValue});
return; // finished
} else {
// the candidate point is updated, then label the chunk as already lazy loaded,
@@ -761,8 +749,8 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
return res;
} else {
return new MergeReaderPriority(
- o2.getChunkMetadata().getVersion(),
- o2.getChunkMetadata().getOffsetOfChunkHeader())
+ o2.getChunkMetadata().getVersion(),
+ o2.getChunkMetadata().getOffsetOfChunkHeader())
.compareTo(
new MergeReaderPriority(
o1.getChunkMetadata().getVersion(),
@@ -839,11 +827,11 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
results
.get(0)
.updateResultUsingValues(
- new long[] {candidateTimestamp}, 1, new Object[] {candidateValue});
+ new long[]{candidateTimestamp}, 1, new Object[]{candidateValue});
results
.get(2)
.updateResultUsingValues(
- new long[] {candidateTimestamp}, 1, new Object[] {candidateValue});
+ new long[]{candidateTimestamp}, 1, new Object[]{candidateValue});
return;
}
}
@@ -872,8 +860,8 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
return res;
} else {
return new MergeReaderPriority(
- o2.getChunkMetadata().getVersion(),
- o2.getChunkMetadata().getOffsetOfChunkHeader())
+ o2.getChunkMetadata().getVersion(),
+ o2.getChunkMetadata().getOffsetOfChunkHeader())
.compareTo(
new MergeReaderPriority(
o1.getChunkMetadata().getVersion(),
@@ -951,11 +939,11 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
results
.get(1)
.updateResultUsingValues(
- new long[] {candidateTimestamp}, 1, new Object[] {candidateValue});
+ new long[]{candidateTimestamp}, 1, new Object[]{candidateValue});
results
.get(3)
.updateResultUsingValues(
- new long[] {candidateTimestamp}, 1, new Object[] {candidateValue});
+ new long[]{candidateTimestamp}, 1, new Object[]{candidateValue});
return;
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java
index c712d1967f0..5c0e87cf93d 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java
@@ -18,31 +18,31 @@
*/
package org.apache.iotdb.tsfile.file.metadata.statistics;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.Objects;
import org.apache.iotdb.tsfile.encoding.decoder.Decoder;
import org.apache.iotdb.tsfile.encoding.decoder.DeltaBinaryDecoder.IntDeltaDecoder;
import org.apache.iotdb.tsfile.encoding.decoder.DoublePrecisionDecoderV2;
import org.apache.iotdb.tsfile.exception.filter.StatisticsClassException;
import org.apache.iotdb.tsfile.exception.write.UnknownColumnTypeException;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.common.ValuePoint;
import org.apache.iotdb.tsfile.utils.Binary;
import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
-
import org.eclipse.collections.impl.list.mutable.primitive.DoubleArrayList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.util.Objects;
-
/**
* This class is used for recording statistic information of each measurement in a delta file. While
* writing processing, the processor records the statistics information. Statistics includes
- * maximum, minimum and null value count up to version 0.0.1.<br>
- * Each data type extends this Statistic as super class.<br>
+ * maximum, minimum and null value count up to version 0.0.1.<br> Each data type extends this
+ * Statistic as super class.<br>
* <br>
* For the statistics in the Unseq file TimeSeriesMetadata, only firstValue, lastValue, startTime
* and endTime can be used.</br>
@@ -55,7 +55,9 @@ public abstract class Statistics<T> {
*/
protected boolean isEmpty = true;
- /** number of time-value points */
+ /**
+ * number of time-value points
+ */
private int count = 0;
private long startTime = Long.MAX_VALUE;
@@ -65,7 +67,9 @@ public abstract class Statistics<T> {
public ValueIndex valueIndex = new ValueIndex();
- /** @author Yuyuan Kang */
+ /**
+ * @author Yuyuan Kang
+ */
final String OPERATION_NOT_SUPPORT_FORMAT = "%s statistics does not support operation: %s";
/**
@@ -176,17 +180,23 @@ public abstract class Statistics<T> {
abstract int serializeStats(OutputStream outputStream) throws IOException;
- /** read data from the inputStream. */
+ /**
+ * read data from the inputStream.
+ */
public abstract void deserialize(InputStream inputStream) throws IOException;
public abstract void deserialize(ByteBuffer byteBuffer);
// public abstract void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes);
- /** @author Yuyuan Kang */
+ /**
+ * @author Yuyuan Kang
+ */
public abstract MinMaxInfo<T> getMinInfo();
- /** @author Yuyuan Kang */
+ /**
+ * @author Yuyuan Kang
+ */
public abstract MinMaxInfo<T> getMaxInfo();
public abstract T getMinValue();
@@ -197,10 +207,14 @@ public abstract class Statistics<T> {
public abstract T getMaxValue();
- /** @author Yuyuan Kang */
+ /**
+ * @author Yuyuan Kang
+ */
public abstract long getBottomTimestamp();
- /** @author Yuyuan Kang */
+ /**
+ * @author Yuyuan Kang
+ */
public abstract long getTopTimestamp();
public abstract T getFirstValue();
@@ -276,7 +290,9 @@ public abstract class Statistics<T> {
updateStats(value);
}
- /** @author Yuyuan Kang */
+ /**
+ * @author Yuyuan Kang
+ */
public void update(long time, int value) {
count++;
if (time < this.startTime) {
@@ -291,7 +307,9 @@ public abstract class Statistics<T> {
updateStats(value, time);
}
- /** @author Yuyuan Kang */
+ /**
+ * @author Yuyuan Kang
+ */
public void update(long time, long value) {
count++;
if (time < this.startTime) {
@@ -305,7 +323,9 @@ public abstract class Statistics<T> {
updateStats(value, time);
}
- /** @author Yuyuan Kang */
+ /**
+ * @author Yuyuan Kang
+ */
public void update(long time, float value) {
count++;
if (time < this.startTime) {
@@ -319,7 +339,9 @@ public abstract class Statistics<T> {
updateStats(value, time);
}
- /** @author Yuyuan Kang */
+ /**
+ * @author Yuyuan Kang
+ */
public void update(long time, double value) {
count++;
if (time < this.startTime) {
@@ -357,7 +379,9 @@ public abstract class Statistics<T> {
updateStats(values, batchSize);
}
- /** @author Yuyuan Kang */
+ /**
+ * @author Yuyuan Kang
+ */
public void update(long[] time, int[] values, int batchSize) {
count += batchSize;
if (time[0] < startTime) {
@@ -371,7 +395,9 @@ public abstract class Statistics<T> {
updateStats(values, time, batchSize);
}
- /** @author Yuyuan Kang */
+ /**
+ * @author Yuyuan Kang
+ */
public void update(long[] time, long[] values, int batchSize) {
count += batchSize;
if (time[0] < startTime) {
@@ -385,7 +411,9 @@ public abstract class Statistics<T> {
updateStats(values, time, batchSize);
}
- /** @author Yuyuan Kang */
+ /**
+ * @author Yuyuan Kang
+ */
public void update(long[] time, float[] values, int batchSize) {
count += batchSize;
if (time[0] < startTime) {
@@ -399,7 +427,9 @@ public abstract class Statistics<T> {
updateStats(values, time, batchSize);
}
- /** @author Yuyuan Kang */
+ /**
+ * @author Yuyuan Kang
+ */
public void update(long[] time, double[] values, int batchSize) {
count += batchSize;
if (time[0] < startTime) {
@@ -435,10 +465,14 @@ public abstract class Statistics<T> {
isEmpty = empty;
}
- /** @author Yuyuan Kang */
+ /**
+ * @author Yuyuan Kang
+ */
public abstract void updateMinInfo(T val, long timestamp);
- /** @author Yuyuan Kang */
+ /**
+ * @author Yuyuan Kang
+ */
public abstract void updateMaxInfo(T val, long timestamp);
void updateStats(boolean value) {
@@ -495,22 +529,30 @@ public abstract class Statistics<T> {
}
}
- /** @author Yuyuan Kang */
+ /**
+ * @author Yuyuan Kang
+ */
public void updateStats(int value, long timestamp) {
throw new UnsupportedOperationException();
}
- /** @author Yuyuan Kang */
+ /**
+ * @author Yuyuan Kang
+ */
public void updateStats(long value, long timestamp) {
throw new UnsupportedOperationException();
}
- /** @author Yuyuan Kang */
+ /**
+ * @author Yuyuan Kang
+ */
public void updateStats(float value, long timestamp) {
throw new UnsupportedOperationException();
}
- /** @author Yuyuan Kang */
+ /**
+ * @author Yuyuan Kang
+ */
public void updateStats(double value, long timestamp) {
throw new UnsupportedOperationException();
}
@@ -523,22 +565,30 @@ public abstract class Statistics<T> {
throw new UnsupportedOperationException();
}
- /** @author Yuyuan Kang */
+ /**
+ * @author Yuyuan Kang
+ */
void updateStats(int[] values, long[] timestamps, int batchSize) {
throw new UnsupportedOperationException();
}
- /** @author Yuyuan Kang */
+ /**
+ * @author Yuyuan Kang
+ */
void updateStats(long[] values, long[] timestamps, int batchSize) {
throw new UnsupportedOperationException();
}
- /** @author Yuyuan Kang */
+ /**
+ * @author Yuyuan Kang
+ */
void updateStats(float[] values, long[] timestamps, int batchSize) {
throw new UnsupportedOperationException();
}
- /** @author Yuyuan Kang */
+ /**
+ * @author Yuyuan Kang
+ */
void updateStats(double[] values, long[] timestamps, int batchSize) {
throw new UnsupportedOperationException();
}
@@ -551,7 +601,7 @@ public abstract class Statistics<T> {
* @param min min timestamp
* @param max max timestamp
* @author Yuyuan Kang This method with two parameters is only used by {@code unsequence} which
- * updates/inserts/deletes timestamp.
+ * updates/inserts/deletes timestamp.
*/
public void updateStats(long min, long bottomTimestamp, long max, long topTimestamp) {
throw new UnsupportedOperationException();
@@ -585,26 +635,31 @@ public abstract class Statistics<T> {
// add the first point
valueIndex.modelPointIdx_list.add(1);
switch (getType()) {
- case INT32:
- int intV = (int) getFirstValue();
- valueIndex.modelPointVal_list.add((double) intV);
- break;
+// case INT32:
+// int intV = (int) getFirstValue();
+// valueIndex.modelPointVal_list.add((double) intV);
+// valueIndex.sortedModelPoints.add(new ValuePoint(1, (double) intV));
+// break;
case INT64:
long longV = (long) getFirstValue();
valueIndex.modelPointVal_list.add((double) longV);
+ valueIndex.sortedModelPoints.add(new ValuePoint(1, (double) longV));
break;
- case FLOAT:
- float floatV = (float) getFirstValue();
- valueIndex.modelPointVal_list.add((double) floatV);
- break;
+// case FLOAT:
+// float floatV = (float) getFirstValue();
+// valueIndex.modelPointVal_list.add((double) floatV);
+// valueIndex.sortedModelPoints.add(new ValuePoint(1, (double) floatV));
+// break;
case DOUBLE:
double doubleV = (double) getFirstValue();
valueIndex.modelPointVal_list.add(doubleV);
+ valueIndex.sortedModelPoints.add(new ValuePoint(1, doubleV));
break;
default:
- throw new IOException("unsupported");
+ throw new IOException("unsupported data type");
}
+ // sdt point idx of the points in between the first and the last points
int idxSize = ReadWriteIOUtils.readInt(buffer);
if (idxSize > 0) {
ByteBuffer idxBuffer = buffer.slice();
@@ -613,18 +668,24 @@ public abstract class Statistics<T> {
while (idxDecoder.hasNext(idxBuffer)) {
int idx = idxDecoder.readInt(idxBuffer);
valueIndex.modelPointIdx_list.add(idx);
+ valueIndex.sortedModelPoints.add(new ValuePoint(idx, 0));
}
}
+ // sdt point value of the points in between the first and the last points
buffer.position(buffer.position() + idxSize);
int valueSize = ReadWriteIOUtils.readInt(buffer);
if (valueSize > 0) {
ByteBuffer valueBuffer = buffer.slice();
valueBuffer.limit(valueSize);
Decoder valueDecoder = new DoublePrecisionDecoderV2();
+ int n = 0;
while (valueDecoder.hasNext(valueBuffer)) {
double value = valueDecoder.readDouble(valueBuffer);
valueIndex.modelPointVal_list.add(value);
+ // NOTE: n+1 because the first point already added
+ valueIndex.sortedModelPoints.get(n + 1).value = value;
+ n++;
}
}
@@ -632,24 +693,28 @@ public abstract class Statistics<T> {
if (count >= 2) { // otherwise only one point no need to store again
valueIndex.modelPointIdx_list.add(count);
switch (getType()) {
- case INT32:
- int intV = (int) getLastValue();
- valueIndex.modelPointVal_list.add((double) intV);
- break;
+// case INT32:
+// int intV = (int) getLastValue();
+// valueIndex.modelPointVal_list.add((double) intV);
+// valueIndex.sortedModelPoints.add(new ValuePoint(count, (double) intV));
+// break;
case INT64:
long longV = (long) getLastValue();
valueIndex.modelPointVal_list.add((double) longV);
+ valueIndex.sortedModelPoints.add(new ValuePoint(count, (double) longV));
break;
- case FLOAT:
- float floatV = (float) getLastValue();
- valueIndex.modelPointVal_list.add((double) floatV);
- break;
+// case FLOAT:
+// float floatV = (float) getLastValue();
+// valueIndex.modelPointVal_list.add((double) floatV);
+// valueIndex.sortedModelPoints.add(new ValuePoint(count, (double) floatV));
+// break;
case DOUBLE:
double doubleV = (double) getLastValue();
valueIndex.modelPointVal_list.add(doubleV);
+ valueIndex.sortedModelPoints.add(new ValuePoint(count, doubleV));
break;
default:
- throw new IOException("unsupported");
+ throw new IOException("unsupported data type");
}
}
@@ -657,8 +722,8 @@ public abstract class Statistics<T> {
buffer.position(buffer.position() + valueSize);
valueIndex.errorBound = ReadWriteIOUtils.readDouble(buffer);
- // System.out.println(valueIndex.modelPointIdx_list);
- // System.out.println(valueIndex.modelPointVal_list);
+ // sort by value from small to big
+ Collections.sort(valueIndex.sortedModelPoints);
}
void deserializeStepRegress(ByteBuffer byteBuffer) {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/ValueIndex.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/ValueIndex.java
index 1ad935e6ef8..eb506480840 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/ValueIndex.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/ValueIndex.java
@@ -1,10 +1,12 @@
package org.apache.iotdb.tsfile.file.metadata.statistics;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.iotdb.tsfile.encoding.encoder.DeltaBinaryEncoder.IntDeltaEncoder;
import org.apache.iotdb.tsfile.encoding.encoder.DoublePrecisionEncoderV2;
import org.apache.iotdb.tsfile.encoding.encoder.SDTEncoder;
+import org.apache.iotdb.tsfile.read.common.ValuePoint;
import org.apache.iotdb.tsfile.utils.PublicBAOS;
-
import org.eclipse.collections.impl.list.mutable.primitive.DoubleArrayList;
import org.eclipse.collections.impl.list.mutable.primitive.IntArrayList;
@@ -20,6 +22,8 @@ public class ValueIndex {
public IntArrayList modelPointIdx_list = new IntArrayList();
public DoubleArrayList modelPointVal_list = new DoubleArrayList();
+ public List<ValuePoint> sortedModelPoints = new ArrayList<>(); // sorted by value in ascending order
+
// this is necessary, otherwise serialized twice by timeseriesMetadata and chunkMetadata
// causing learn() executed more than once!!
private boolean isLearned = false;
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
index a43563e0410..3b5fd251521 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.tsfile.read.common;
+import java.io.IOException;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -32,11 +33,10 @@ import org.apache.iotdb.tsfile.file.metadata.statistics.StepRegress;
import org.apache.iotdb.tsfile.read.common.IOMonitor2.Operation;
import org.apache.iotdb.tsfile.read.reader.page.PageReader;
-import java.io.IOException;
-
public class ChunkSuit4CPV {
private ChunkMetadata chunkMetadata; // fixed info, including version, dataType, stepRegress
+ public int modelPointsCursor = 1; // starting from 0, pointing to the right end of the model segment, moving forward as processing time spans
public Statistics statistics; // dynamically updated, includes FP/LP/BP/TP info
@@ -282,10 +282,10 @@ public class ChunkSuit4CPV {
long timestamp = pageReader.timeBuffer.getLong(estimatedPos * 8);
statistics.setStartTime(timestamp);
switch (chunkMetadata.getDataType()) {
- // iotdb的int类型的plain编码用的是自制的不支持random access
- // case INT32:
- // return new MinMaxInfo(pageReader.valueBuffer.getInt(estimatedPos * 4),
- // pageReader.timeBuffer.getLong(estimatedPos * 8));
+ // iotdb的int类型的plain编码用的是自制的不支持random access
+ // case INT32:
+ // return new MinMaxInfo(pageReader.valueBuffer.getInt(estimatedPos * 4),
+ // pageReader.timeBuffer.getLong(estimatedPos * 8));
case INT64:
long longVal =
pageReader.valueBuffer.getLong(pageReader.timeBufferLength + estimatedPos * 8);
@@ -328,10 +328,10 @@ public class ChunkSuit4CPV {
long timestamp = pageReader.timeBuffer.getLong(estimatedPos * 8);
statistics.setStartTime(timestamp);
switch (chunkMetadata.getDataType()) {
- // iotdb的int类型的plain编码用的是自制的不支持random access
- // case INT32:
- // return new MinMaxInfo(pageReader.valueBuffer.getInt(estimatedPos * 4),
- // pageReader.timeBuffer.getLong(estimatedPos * 8));
+ // iotdb的int类型的plain编码用的是自制的不支持random access
+ // case INT32:
+ // return new MinMaxInfo(pageReader.valueBuffer.getInt(estimatedPos * 4),
+ // pageReader.timeBuffer.getLong(estimatedPos * 8));
case INT64:
long longVal =
pageReader.valueBuffer.getLong(pageReader.timeBufferLength + estimatedPos * 8);
@@ -399,10 +399,10 @@ public class ChunkSuit4CPV {
long timestamp = pageReader.timeBuffer.getLong(estimatedPos * 8);
statistics.setEndTime(timestamp);
switch (chunkMetadata.getDataType()) {
- // iotdb的int类型的plain编码用的是自制的不支持random access
- // case INT32:
- // return new MinMaxInfo(pageReader.valueBuffer.getInt(estimatedPos * 4),
- // pageReader.timeBuffer.getLong(estimatedPos * 8));
+ // iotdb的int类型的plain编码用的是自制的不支持random access
+ // case INT32:
+ // return new MinMaxInfo(pageReader.valueBuffer.getInt(estimatedPos * 4),
+ // pageReader.timeBuffer.getLong(estimatedPos * 8));
case INT64:
long longVal =
pageReader.valueBuffer.getLong(pageReader.timeBufferLength + estimatedPos * 8);
@@ -451,10 +451,10 @@ public class ChunkSuit4CPV {
long timestamp = pageReader.timeBuffer.getLong(estimatedPos * 8);
statistics.setEndTime(timestamp);
switch (chunkMetadata.getDataType()) {
- // iotdb的int类型的plain编码用的是自制的不支持random access
- // case INT32:
- // return new MinMaxInfo(pageReader.valueBuffer.getInt(estimatedPos * 4),
- // pageReader.timeBuffer.getLong(estimatedPos * 8));
+ // iotdb的int类型的plain编码用的是自制的不支持random access
+ // case INT32:
+ // return new MinMaxInfo(pageReader.valueBuffer.getInt(estimatedPos * 4),
+ // pageReader.timeBuffer.getLong(estimatedPos * 8));
case INT64:
long longVal =
pageReader.valueBuffer.getLong(pageReader.timeBufferLength + estimatedPos * 8);
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
index cd39d9af1e0..62e79e40fac 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
@@ -247,22 +247,6 @@ public class IOMonitor2 {
M4_LSM_TP_SEARCH_ARRAY_c_genBPTP_cnt = 0;
}
- public static class ValuePoint implements Comparable<ValuePoint> {
- public final int index;
- public final long value;
-
- public ValuePoint(int index, long value) {
- this.index = index;
- this.value = value;
- }
-
- @Override
- public int compareTo(ValuePoint other) {
- // multiplied to -1 as the author need descending sort order
- return -1 * Long.valueOf(this.value).compareTo(other.value);
- }
- }
-
public static void addMeasure(Operation operation, long elapsedTimeInNanosecond) {
switch (operation) {
case DCP_Server_Query_Execute:
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ValuePoint.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ValuePoint.java
new file mode 100644
index 00000000000..2e672d7a134
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ValuePoint.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.read.common;
+
+public class ValuePoint implements Comparable<ValuePoint> {
+
+ public int index;
+ public double value;
+
+ public ValuePoint(int index, double value) {
+ this.index = index;
+ this.value = value;
+ }
+
+ @Override
+ public int compareTo(ValuePoint other) {
+ // ascending order
+ return Double.valueOf(this.value).compareTo(other.value);
+ }
+
+ public String toString() {
+ return "(" + index + "," + value + ")";
+ }
+}
\ No newline at end of file
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
index a75b368c00d..979678ea5b7 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
@@ -18,56 +18,67 @@
*/
package org.apache.iotdb.tsfile.read.reader.page;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
import org.apache.iotdb.tsfile.encoding.decoder.Decoder;
import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException;
import org.apache.iotdb.tsfile.file.header.PageHeader;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.statistics.DoubleStatistics;
-import org.apache.iotdb.tsfile.file.metadata.statistics.FloatStatistics;
import org.apache.iotdb.tsfile.file.metadata.statistics.LongStatistics;
+import org.apache.iotdb.tsfile.file.metadata.statistics.MinMaxInfo;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
+import org.apache.iotdb.tsfile.file.metadata.statistics.ValueIndex;
import org.apache.iotdb.tsfile.read.common.BatchData;
import org.apache.iotdb.tsfile.read.common.BatchDataFactory;
import org.apache.iotdb.tsfile.read.common.ChunkSuit4CPV;
import org.apache.iotdb.tsfile.read.common.IOMonitor2;
import org.apache.iotdb.tsfile.read.common.IOMonitor2.Operation;
import org.apache.iotdb.tsfile.read.common.TimeRange;
+import org.apache.iotdb.tsfile.read.common.ValuePoint;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.read.filter.operator.AndFilter;
import org.apache.iotdb.tsfile.read.reader.IPageReader;
import org.apache.iotdb.tsfile.utils.Binary;
import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
public class PageReader implements IPageReader {
private PageHeader pageHeader;
protected TSDataType dataType;
- /** decoder for value column */
+ /**
+ * decoder for value column
+ */
protected Decoder valueDecoder;
- /** decoder for time column */
+ /**
+ * decoder for time column
+ */
protected Decoder timeDecoder;
- /** time column in memory */
+ /**
+ * time column in memory
+ */
public ByteBuffer timeBuffer;
- /** value column in memory */
+ /**
+ * value column in memory
+ */
public ByteBuffer valueBuffer;
public int timeBufferLength;
protected Filter filter;
- /** A list of deleted intervals. */
+ /**
+ * A list of deleted intervals.
+ */
private List<TimeRange> deleteIntervalList;
private int deleteCursor = 0;
@@ -111,7 +122,9 @@ public class PageReader implements IPageReader {
valueBuffer.position(timeBufferLength);
}
- /** the chunk partially overlaps in time with the current M4 interval Ii */
+ /**
+ * the chunk partially overlaps in time with the current M4 interval Ii
+ */
public void split4CPV(
long startTime,
long endTime,
@@ -124,10 +137,10 @@ public class PageReader implements IPageReader {
// endTime is excluded so -1
int numberOfSpans =
(int)
- Math.floor(
- (Math.min(chunkMetadata.getEndTime(), endTime - 1) - curStartTime)
- * 1.0
- / interval)
+ Math.floor(
+ (Math.min(chunkMetadata.getEndTime(), endTime - 1) - curStartTime)
+ * 1.0
+ / interval)
+ 1;
for (int n = 0; n < numberOfSpans; n++) {
long leftEndIncluded = curStartTime + n * interval;
@@ -164,6 +177,156 @@ public class PageReader implements IPageReader {
}
}
+ public void updateTP_withValueIndex(ChunkSuit4CPV chunkSuit4CPV) {
+ // NOTE: get valueIndex from chunkSuit4CPV.getChunkMetadata().getStatistics(), not chunkSuit4CPV.getStatistics()!
+ ValueIndex valueIndex = chunkSuit4CPV.getChunkMetadata().getStatistics().valueIndex;
+
+ // step 1: find threshold
+ // iterate SDT points from value big to small to find the first point not deleted
+ boolean isFound = false;
+ double foundValue = 0;
+ for (ValuePoint valuePoint : valueIndex.sortedModelPoints) {
+ int idx = valuePoint.index; // index starting from 1
+ int pos = idx - 1; // pos starting from 0
+ long time = timeBuffer.getLong(pos * 8);
+ // check if deleted
+ deleteCursor = 0; // TODO check
+ if ((pos >= chunkSuit4CPV.startPos) && (pos <= chunkSuit4CPV.endPos) && !isDeleted(time)) {
+ // startPos&endPos conveys the virtual deletes of the current M4 time span
+ isFound = true;
+ foundValue = valuePoint.value;
+ break;
+ }
+ }
+ if (!isFound) { // unfortunately all sdt points are deleted
+ updateBPTP(chunkSuit4CPV); // then fall back to baseline method
+ return;
+ }
+ double threshold_LB = foundValue - valueIndex.errorBound; // near max LB
+
+ // step 2: calculate pruned intervals for TP: UB<threshold=near max LB
+ // increment global chunkSuit4CPV.modelPointsCursor
+ int idx2;
+ // note that the first and last points of a chunk are stored in model points
+ // there must exist idx2-1 >= startPos, otherwise this chunk won't be processed for the current time span
+ // there must exist idx1-1 <= endPos, otherwise this chunk won't be processed for the current time span
+ while ((idx2 = valueIndex.modelPointIdx_list.get(chunkSuit4CPV.modelPointsCursor)) - 1
+ < chunkSuit4CPV.startPos) { // TODO check
+ // -1 because idx starting from 1 while pos starting from 0
+ chunkSuit4CPV.modelPointsCursor++;
+ }
+ // increment local cursor starting from chunkSuit4CPV.modelPointsCursor for iterating model segments for the current time span
+ // do not increment modelPointsCursor because the model segments for this time span may be iterated multiple times
+ int localCursor = chunkSuit4CPV.modelPointsCursor;
+ List<Integer> prune_intervals_start = new ArrayList<>();
+ List<Integer> prune_intervals_end = new ArrayList<>();
+ int interval_start = -1;
+ int interval_end = -1;
+ int idx1;
+ while ((idx1 = valueIndex.modelPointIdx_list.get(localCursor - 1)) - 1
+ <= chunkSuit4CPV.endPos) {
+ idx2 = valueIndex.modelPointIdx_list.get(localCursor);
+ double v1_UB = valueIndex.modelPointVal_list.get(localCursor - 1) + valueIndex.errorBound;
+ double v2_UB = valueIndex.modelPointVal_list.get(localCursor) + valueIndex.errorBound;
+ if (v1_UB < threshold_LB && v2_UB < threshold_LB) {
+ if (interval_start < 0) {
+ interval_start = idx1;
+ }
+ interval_end = idx2; // continuous
+ } else if (v1_UB < threshold_LB && v2_UB >= threshold_LB) {
+ if (interval_start < 0) {
+ interval_start = idx1;
+ }
+ prune_intervals_start.add(interval_start);
+ prune_intervals_end.add(
+ (int) Math.floor((threshold_LB - v1_UB) * (idx2 - idx1) / (v2_UB - v1_UB) + idx1));
+ interval_start = -1; // discontinuous
+ } else if (v1_UB >= threshold_LB && v2_UB < threshold_LB) {
+ interval_start = (int) Math.ceil(
+ (threshold_LB - v1_UB) * (idx2 - idx1) / (v2_UB - v1_UB) + idx1);
+ interval_end = idx2; // continuous
+ }
+ localCursor++;
+ }
+ if (interval_start > 0) {
+ prune_intervals_start.add(interval_start);
+ prune_intervals_end.add(interval_end);
+ }
+
+ // step 3: calculate unpruned intervals
+ // TODO deal with time span deletes -> update search_startPos and search_endPos
+ // note idx starting from 1, pos starting from 0
+ int search_startPos = chunkSuit4CPV.startPos;
+ int search_endPos = chunkSuit4CPV.endPos;
+ if (prune_intervals_start.size() > 0) {
+ // deal with time span left virtual delete -> update search_startPos
+ int prune_idx1 = prune_intervals_start.get(0);
+ if (prune_idx1 - 1 <= chunkSuit4CPV.startPos) {
+ // +1 for included, -1 for starting from 0
+ search_startPos = Math.max(search_startPos, prune_intervals_end.get(0) + 1 - 1);
+ prune_intervals_start.remove(0);
+ prune_intervals_end.remove(0);
+ }
+ }
+ if (prune_intervals_start.size() > 0) {
+ // deal with time span right virtual delete -> update search_endPos
+ int prune_idx2 = prune_intervals_end.get(prune_intervals_end.size() - 1);
+ if (prune_idx2 - 1 >= search_endPos) {
+ // -1 for included, -1 for starting from 0
+ search_endPos = Math.min(search_endPos,
+ prune_intervals_start.get(prune_intervals_start.size() - 1) - 1 - 1);
+ prune_intervals_start.remove(prune_intervals_start.size() - 1);
+ prune_intervals_end.remove(prune_intervals_end.size() - 1);
+ }
+ }
+ // add search_endPos+1 to the end of prune_intervals_start
+ // turning into search_intervals_end (excluded endpoints)
+ prune_intervals_start.add(search_endPos + 1);
+ // add search_startPos-1 to the start of prune_intervals_end
+ // turning into search_intervals_start (excluded endpoints)
+ prune_intervals_end.add(0, search_startPos - 1);
+
+ // step 4: search unpruned intervals
+ // TODO deal with normal delete intervals
+ if (dataType == TSDataType.DOUBLE) {
+ double candidateTPvalue = -1;
+ long candidateTPtime = -1;
+ for (int i = 0; i < prune_intervals_start.size(); i++) {
+ int search_interval_start = prune_intervals_end.get(i) + 1; // included
+ int search_interval_end = prune_intervals_start.get(i) - 1; // included
+ for (int j = search_interval_start; j <= search_interval_end; j++) { // starting from 1
+ double v = valueBuffer.getDouble(timeBufferLength + (j - 1) * 8);
+ long t = timeBuffer.getLong((j - 1) * 8);
+ if (v > candidateTPvalue && !isDeleted(t)) {
+ candidateTPvalue = v;
+ candidateTPtime = t;
+ }
+ }
+ }
+ chunkSuit4CPV.statistics.setMaxInfo(new MinMaxInfo(candidateTPvalue, candidateTPtime));
+ } else if (dataType == TSDataType.INT64) {
+ long candidateTPvalue = -1;
+ long candidateTPtime = -1;
+ for (int i = 0; i < prune_intervals_start.size(); i++) {
+ int search_interval_start = prune_intervals_end.get(i) + 1; // included
+ int search_interval_end = prune_intervals_start.get(i) - 1; // included
+ for (int j = search_interval_start; j <= search_interval_end; j++) { // starting from 1
+ long v = valueBuffer.getLong(timeBufferLength + (j - 1) * 8);
+ long t = timeBuffer.getLong((j - 1) * 8);
+ if (v > candidateTPvalue && !isDeleted(t)) {
+ candidateTPvalue = v;
+ candidateTPtime = t;
+ }
+ }
+ }
+ chunkSuit4CPV.statistics.setMaxInfo(new MinMaxInfo(candidateTPvalue, candidateTPtime));
+ } else {
+ throw new UnSupportedDataTypeException(String.valueOf(dataType));
+ }
+
+ // TODO 注意count=0全部点删掉的情况考虑 难道是在isFound=false的时候回到原来方法执行里处理了
+ }
+
public void updateBPTP(ChunkSuit4CPV chunkSuit4CPV) {
long start = System.nanoTime();
deleteCursor = 0; // TODO DEBUG
@@ -172,9 +335,9 @@ public class PageReader implements IPageReader {
case INT64:
statistics = new LongStatistics();
break;
- case FLOAT:
- statistics = new FloatStatistics();
- break;
+// case FLOAT:
+// statistics = new FloatStatistics();
+// break;
case DOUBLE:
statistics = new DoubleStatistics();
break;
@@ -198,16 +361,16 @@ public class PageReader implements IPageReader {
// only updateStats, actually only need to update BP and TP
}
break;
- case FLOAT:
- float aFloat = valueBuffer.getFloat(timeBufferLength + pos * 8);
- if (!isDeleted(timestamp) && (filter == null || filter.satisfy(timestamp, aFloat))) {
- // update statistics of chunkMetadata1
- statistics.updateStats(aFloat, timestamp);
- count++;
- // ATTENTION: do not use update() interface which will also update StepRegress!
- // only updateStats, actually only need to update BP and TP
- }
- break;
+// case FLOAT:
+// float aFloat = valueBuffer.getFloat(timeBufferLength + pos * 8);
+// if (!isDeleted(timestamp) && (filter == null || filter.satisfy(timestamp, aFloat))) {
+// // update statistics of chunkMetadata1
+// statistics.updateStats(aFloat, timestamp);
+// count++;
+// // ATTENTION: do not use update() interface which will also update StepRegress!
+// // only updateStats, actually only need to update BP and TP
+// }
+// break;
case DOUBLE:
double aDouble = valueBuffer.getDouble(timeBufferLength + pos * 8);
if (!isDeleted(timestamp) && (filter == null || filter.satisfy(timestamp, aDouble))) {
@@ -231,7 +394,9 @@ public class PageReader implements IPageReader {
IOMonitor2.addMeasure(Operation.SEARCH_ARRAY_c_genBPTP, System.nanoTime() - start);
}
- /** @return the returned BatchData may be empty, but never be null */
+ /**
+ * @return the returned BatchData may be empty, but never be null
+ */
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
@Override
public BatchData getAllSatisfiedPageData(boolean ascending) throws IOException {
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SDTEncoderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SDTEncoderTest.java
index 07450c8beae..f859ddab83d 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SDTEncoderTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SDTEncoderTest.java
@@ -19,19 +19,17 @@
package org.apache.iotdb.tsfile.encoding;
-import org.apache.iotdb.tsfile.encoding.encoder.SDTEncoder;
-import org.apache.iotdb.tsfile.read.common.IOMonitor2.ValuePoint;
-
-import org.junit.Test;
+import static org.junit.Assert.assertEquals;
import java.io.BufferedReader;
import java.io.FileReader;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Comparator;
+import java.util.Collections;
import java.util.List;
-
-import static org.junit.Assert.assertEquals;
+import org.apache.iotdb.tsfile.encoding.encoder.SDTEncoder;
+import org.apache.iotdb.tsfile.read.common.ValuePoint;
+import org.junit.Test;
public class SDTEncoderTest {
@@ -235,8 +233,8 @@ public class SDTEncoderTest {
String csvData = "D:\\full-game\\BallSpeed.csv";
// double[] eList = new double[] {500000, 400000, 300000, 200000, 160000, 100000, 50000,
// 10000};
- double[] eList = new double[] {160000};
- int[] startList = new int[] {1, 200000, 300000, 400000, 500000, 600000, 700000, 800000, 900000};
+ double[] eList = new double[]{160000};
+ int[] startList = new int[]{1, 200000, 300000, 400000, 500000, 600000, 700000, 800000, 900000};
List<Double> elapsedTime_withValueIndex_list = new ArrayList<>();
List<Double> elapsedTime_withoutValueIndex_list = new ArrayList<>();
List<Double> traversedComplexity_list = new ArrayList<>();
@@ -304,18 +302,19 @@ public class SDTEncoderTest {
// ValuePoint[] myArray = (ValuePoint[]) points.toArray();
// Arrays.sort(points.toArray());
- points.sort(
- new Comparator<ValuePoint>() {
- public int compare(ValuePoint o1, ValuePoint o2) {
- return ((Comparable) (o1.value)).compareTo(o2.value);
- }
- });
+// points.sort(
+// new Comparator<ValuePoint>() {
+// public int compare(ValuePoint o1, ValuePoint o2) {
+// return ((Comparable) (o1.value)).compareTo(o2.value);
+// }
+// });
+ Collections.sort(points);
long startTime = System.nanoTime();
// 计算maxLB
traversedComplexity += selectValues.size();
- long maxVal = points.get(points.size() - 2).value;
+ double maxVal = points.get(points.size() - 2).value;
double threshold = maxVal - e; // maxLB
// System.out.println("threshold(maxLB)=" + threshold);
@@ -380,7 +379,7 @@ public class SDTEncoderTest {
long elapsedTime = System.nanoTime() - startTime;
elapsedTime_withValueIndex += elapsedTime;
// System.out.println("search with value index: " + elapsedTime / 1000000.0 + " ms");
- // System.out.println("TP=(" + candidateTPidx + "," + candidateTPvalue + ")");
+ System.out.println("TP=(" + candidateTPidx + "," + candidateTPvalue + ")");
System.out.println("search interval number=" + prune_intervals_end.size());
int traversedPoints = 0;
for (int i = 0; i < prune_intervals_start.size(); i++) {
@@ -408,8 +407,8 @@ public class SDTEncoderTest {
elapsedTime_withoutValueIndex += elapsedTime;
// System.out.println("search without value index: " + elapsedTime / 1000000.0 + "
// ms");
- // System.out.println("TP=(" + candidateTPidx_raw + "," + candidateTPvalue_raw +
- // ")");
+ System.out.println("TP=(" + candidateTPidx_raw + "," + candidateTPvalue_raw +
+ ")");
}
elapsedTime_withValueIndex_list.add(
elapsedTime_withValueIndex / startList.length / 1000000.0);
[iotdb] 11/46: [important] four-layer metrics design done, from client, server execute&fetch, dataset&executor, basic ops ABCD, improved D search array 4 functions
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit b24dbd33b034bf71f505476f9b9f97e481ef318f
Author: Lei Rui <10...@qq.com>
AuthorDate: Tue Jun 13 22:08:35 2023 +0800
[important] four-layer metrics design done, from client, server execute&fetch, dataset&executor, basic ops ABCD, improved D search array 4 functions
---
.../dataset/groupby/LocalGroupByExecutor4CPV.java | 18 ++
.../iotdb/tsfile/read/common/ChunkSuit4CPV.java | 40 ++-
.../iotdb/tsfile/read/common/IOMonitor2.java | 282 +++++++++++++++++----
.../iotdb/tsfile/read/reader/page/PageReader.java | 3 +
4 files changed, 289 insertions(+), 54 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java
index b4c59643688..92026420e17 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java
@@ -38,6 +38,8 @@ import org.apache.iotdb.tsfile.file.metadata.statistics.IntegerStatistics;
import org.apache.iotdb.tsfile.file.metadata.statistics.LongStatistics;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.common.ChunkSuit4CPV;
+import org.apache.iotdb.tsfile.read.common.IOMonitor2;
+import org.apache.iotdb.tsfile.read.common.IOMonitor2.Operation;
import org.apache.iotdb.tsfile.read.common.TimeRange;
import org.apache.iotdb.tsfile.read.filter.GroupByFilter;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
@@ -93,6 +95,7 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
TsFileFilter fileFilter,
boolean ascending)
throws StorageEngineException, QueryProcessException {
+ long start = System.nanoTime();
this.tsDataType = dataType;
// this.mergeReader = new PriorityMergeReader();
@@ -225,6 +228,8 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
} catch (IOException e) {
throw new QueryProcessException(e.getMessage());
}
+
+ IOMonitor2.addMeasure(Operation.M4_LSM_INIT_LOAD_ALL_CHUNKMETADATAS, System.nanoTime() - start);
}
@Override
@@ -309,16 +314,29 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
result.reset();
}
+ long start = System.nanoTime();
getCurrentChunkListFromFutureChunkList(curStartTime, curEndTime, startTime, endTime, interval);
+ IOMonitor2.addMeasure(Operation.M4_LSM_MERGE_M4_TIME_SPAN, System.nanoTime() - start);
if (currentChunkList.size() == 0) {
return results;
}
+ start = System.nanoTime();
calculateFirstPoint(currentChunkList, startTime, endTime, interval, curStartTime);
+ IOMonitor2.addMeasure(Operation.M4_LSM_FP, System.nanoTime() - start);
+
+ start = System.nanoTime();
calculateLastPoint(currentChunkList, startTime, endTime, interval, curStartTime);
+ IOMonitor2.addMeasure(Operation.M4_LSM_LP, System.nanoTime() - start);
+
+ start = System.nanoTime();
calculateBottomPoint(currentChunkList, startTime, endTime, interval, curStartTime);
+ IOMonitor2.addMeasure(Operation.M4_LSM_BP, System.nanoTime() - start);
+
+ start = System.nanoTime();
calculateTopPoint(currentChunkList, startTime, endTime, interval, curStartTime);
+ IOMonitor2.addMeasure(Operation.M4_LSM_TP, System.nanoTime() - start);
return results;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
index 0834348a268..67520439d22 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
@@ -29,6 +29,7 @@ import org.apache.iotdb.tsfile.file.metadata.statistics.LongStatistics;
import org.apache.iotdb.tsfile.file.metadata.statistics.MinMaxInfo;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.file.metadata.statistics.StepRegress;
+import org.apache.iotdb.tsfile.read.common.IOMonitor2.Operation;
import org.apache.iotdb.tsfile.read.reader.page.PageReader;
import java.io.IOException;
@@ -245,10 +246,12 @@ public class ChunkSuit4CPV {
* @return the position of the point, starting from 0
*/
public int updateFPwithTheClosetPointEqualOrAfter(long targetTimestamp) throws IOException {
+ long start = System.nanoTime();
+ int estimatedPos;
if (TSFileDescriptor.getInstance().getConfig().isUseChunkIndex()) {
StepRegress stepRegress = chunkMetadata.getStatistics().getStepRegress();
// infer position starts from 1, so minus 1 here
- int estimatedPos = (int) Math.round(stepRegress.infer(targetTimestamp)) - 1;
+ estimatedPos = (int) Math.round(stepRegress.infer(targetTimestamp)) - 1;
// search from estimatePos in the timeBuffer to find the closet timestamp equal to or larger
// than the given timestamp
@@ -256,15 +259,18 @@ public class ChunkSuit4CPV {
while (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
estimatedPos++;
IOMonitor.incPointsTravered();
+ IOMonitor2.DCP_D_traversedPointNum++;
}
} else if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
while (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
estimatedPos--;
IOMonitor.incPointsTravered();
+ IOMonitor2.DCP_D_traversedPointNum++;
}
if (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
estimatedPos++;
IOMonitor.incPointsTravered();
+ IOMonitor2.DCP_D_traversedPointNum++;
} // else equal
} // else equal
this.startPos = estimatedPos; // note this
@@ -298,16 +304,16 @@ public class ChunkSuit4CPV {
default:
throw new IOException("Unsupported data type!");
}
- return estimatedPos;
} else {
// search from estimatePos in the timeBuffer to find the closet timestamp equal to or larger
// than the given timestamp
- int estimatedPos = -1;
+ estimatedPos = -1;
pageReader.timeBuffer.position(0);
pageReader.valueBuffer.position(pageReader.timeBufferLength);
while (pageReader.timeBuffer.remaining() > 0) {
estimatedPos++;
IOMonitor.incPointsTravered();
+ IOMonitor2.DCP_D_traversedPointNum++;
long t = pageReader.timeBuffer.getLong();
if (t >= targetTimestamp) {
break;
@@ -346,6 +352,8 @@ public class ChunkSuit4CPV {
}
return estimatedPos;
}
+ IOMonitor2.addMeasure(Operation.SEARCH_ARRAY_b_genFP, System.nanoTime() - start);
+ return estimatedPos;
}
/**
@@ -356,10 +364,12 @@ public class ChunkSuit4CPV {
* @return the position of the point, starting from 0
*/
public int updateLPwithTheClosetPointEqualOrBefore(long targetTimestamp) throws IOException {
+ long start = System.nanoTime();
+ int estimatedPos;
if (TSFileDescriptor.getInstance().getConfig().isUseChunkIndex()) {
StepRegress stepRegress = chunkMetadata.getStatistics().getStepRegress();
// infer position starts from 1, so minus 1 here
- int estimatedPos = (int) Math.round(stepRegress.infer(targetTimestamp)) - 1;
+ estimatedPos = (int) Math.round(stepRegress.infer(targetTimestamp)) - 1;
// search from estimatePos in the timeBuffer to find the closet timestamp equal to or smaller
// than the given timestamp
@@ -367,15 +377,18 @@ public class ChunkSuit4CPV {
while (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
estimatedPos--;
IOMonitor.incPointsTravered();
+ IOMonitor2.DCP_D_traversedPointNum++;
}
} else if (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
while (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
estimatedPos++;
IOMonitor.incPointsTravered();
+ IOMonitor2.DCP_D_traversedPointNum++;
}
if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
estimatedPos--;
IOMonitor.incPointsTravered();
+ IOMonitor2.DCP_D_traversedPointNum++;
} // else equal
} // else equal
this.endPos = estimatedPos; // note this
@@ -413,12 +426,13 @@ public class ChunkSuit4CPV {
} else {
// to find the closet timestamp equal to or smaller
// than the given timestamp
- int estimatedPos = -1;
+ estimatedPos = -1;
pageReader.timeBuffer.position(0);
pageReader.valueBuffer.position(pageReader.timeBufferLength);
while (pageReader.timeBuffer.remaining() > 0) {
estimatedPos++;
IOMonitor.incPointsTravered();
+ IOMonitor2.DCP_D_traversedPointNum++;
long t = pageReader.timeBuffer.getLong();
if (t >= targetTimestamp) {
break;
@@ -426,6 +440,7 @@ public class ChunkSuit4CPV {
}
if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
IOMonitor.incPointsTravered();
+ IOMonitor2.DCP_D_traversedPointNum++;
estimatedPos--;
} // else equals no need to minus 1
@@ -460,8 +475,9 @@ public class ChunkSuit4CPV {
default:
throw new IOException("Unsupported data type!");
}
- return estimatedPos;
}
+ IOMonitor2.addMeasure(Operation.SEARCH_ARRAY_b_genLP, System.nanoTime() - start);
+ return estimatedPos;
}
/**
@@ -471,6 +487,8 @@ public class ChunkSuit4CPV {
* @return true if exists; false not exist
*/
public boolean checkIfExist(long targetTimestamp) throws IOException {
+ long start = System.nanoTime();
+ boolean exist;
if (TSFileDescriptor.getInstance().getConfig().isUseChunkIndex()) {
StepRegress stepRegress = chunkMetadata.getStatistics().getStepRegress();
// infer position starts from 1, so minus 1 here
@@ -483,22 +501,25 @@ public class ChunkSuit4CPV {
while (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
estimatedPos--;
IOMonitor.incPointsTravered();
+ IOMonitor2.DCP_D_traversedPointNum++;
}
} else if (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
while (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
estimatedPos++;
IOMonitor.incPointsTravered();
+ IOMonitor2.DCP_D_traversedPointNum++;
}
if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
estimatedPos--;
IOMonitor.incPointsTravered();
+ IOMonitor2.DCP_D_traversedPointNum++;
} // else equal
} // else equal
// since we have constrained that targetTimestamp must be within the chunk time range
// [startTime, endTime],
// estimatedPos will not be out of range.
- return pageReader.timeBuffer.getLong(estimatedPos * 8) == targetTimestamp;
+ exist = pageReader.timeBuffer.getLong(estimatedPos * 8) == targetTimestamp;
} else {
// search from estimatePos in the timeBuffer to find the closet timestamp equal to or smaller
// than the given timestamp
@@ -510,6 +531,7 @@ public class ChunkSuit4CPV {
if (!flag) {
estimatedPos++;
IOMonitor.incPointsTravered();
+ IOMonitor2.DCP_D_traversedPointNum++;
}
long t = pageReader.timeBuffer.getLong();
if (t >= targetTimestamp) {
@@ -520,8 +542,10 @@ public class ChunkSuit4CPV {
// since we have constrained that targetTimestamp must be within the chunk time range
// [startTime, endTime],
// estimatedPos will not be out of range.
- return pageReader.timeBuffer.getLong(estimatedPos * 8) == targetTimestamp;
+ exist = pageReader.timeBuffer.getLong(estimatedPos * 8) == targetTimestamp;
}
+ IOMonitor2.addMeasure(Operation.SEARCH_ARRAY_a_verifBPTP, System.nanoTime() - start);
+ return exist;
}
public void updateFP(MinMaxInfo point) {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
index d9eeaf30b6c..9c17f371e34 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
@@ -19,10 +19,8 @@
package org.apache.iotdb.tsfile.read.common;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
public class IOMonitor2 {
+
public enum DataSetType { // dataSet, executor, reader, file
NONE,
RawQueryDataSetWithoutValueFilter,
@@ -33,12 +31,21 @@ public class IOMonitor2 {
public enum Operation {
DCP_Server_Query_Execute("DCP_Server_Query_Execute"),
DCP_Server_Query_Fetch("DCP_Server_Query_Fetch"),
- DCP_SeriesScanOperator_hasNext("DCP_SeriesScanOperator_hasNext"),
DCP_A_GET_CHUNK_METADATAS("DCP_A_GET_CHUNK_METADATAS"),
DCP_B_READ_MEM_CHUNK("DCP_B_READ_MEM_CHUNK"),
DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA(
"DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA"),
- DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS("DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS");
+ DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS("DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS"),
+ SEARCH_ARRAY_a_verifBPTP("SEARCH_ARRAY_a_verifBPTP"),
+ SEARCH_ARRAY_b_genFP("SEARCH_ARRAY_b_genFP"),
+ SEARCH_ARRAY_b_genLP("SEARCH_ARRAY_b_genLP"),
+ SEARCH_ARRAY_c_genBPTP("SEARCH_ARRAY_c_genBPTP"),
+ M4_LSM_INIT_LOAD_ALL_CHUNKMETADATAS("M4_LSM_INIT_LOAD_ALL_CHUNKMETADATAS"),
+ M4_LSM_MERGE_M4_TIME_SPAN("M4_LSM_MERGE_M4_TIME_SPAN"),
+ M4_LSM_FP("M4_LSM_FP"),
+ M4_LSM_LP("M4_LSM_LP"),
+ M4_LSM_BP("M4_LSM_BP"),
+ M4_LSM_TP("M4_LSM_TP");
public String getName() {
return name;
@@ -51,23 +58,14 @@ public class IOMonitor2 {
}
}
- /**
- * [Raw data query metrics] - server端execute&fetch的次数和耗时 -
- * SeriesScanOperator.hasNext的次数和耗时（问题：root.hasNext执行次数等于decode
- * page次数，但是SeriesScanOperator.hasNext次数大于那个次数？破案了：是root.isFinished->SeriesScanOperator.isFinished里面调用了hasNext，和root.hasNext里调用SeriesScanOperator.hasNext的次数加起来就对了。）
- * - A：从磁盘解读chunk元数据的次数和耗时 - B：从磁盘加载chunk数据的次数和耗时 - C：解压缩所有page构造pageReaderList的次数和耗时 -
- * D：解码page并遍历点的次数和耗时 （遍历点数不用统计，因为一定是全量点）
- */
- public static int DCP_Server_Query_Execute_count = 0;
+ public static DataSetType dataSetType = DataSetType.NONE;
+ public static int DCP_Server_Query_Execute_count = 0;
public static long DCP_Server_Query_Execute_ns = 0;
public static int DCP_Server_Query_Fetch_count = 0;
public static long DCP_Server_Query_Fetch_ns = 0;
- public static int DCP_SeriesScanOperator_hasNext_count = 0;
- public static long DCP_SeriesScanOperator_hasNext_ns = 0;
-
public static int DCP_A_GET_CHUNK_METADATAS_count = 0;
public static long DCP_A_GET_CHUNK_METADATAS_ns = 0;
@@ -77,25 +75,49 @@ public class IOMonitor2 {
public static int DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_count = 0;
public static long DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_ns = 0;
+ public static long DCP_D_traversedPointNum = 0;
public static int DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_count = 0;
public static long DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_ns = 0;
- public static long DCP_D_traversedPointNum = 0;
+ public static int SEARCH_ARRAY_a_verifBPTP_count = 0;
+ public static long SEARCH_ARRAY_a_verifBPTP_ns = 0;
- public static DataSetType dataSetType = DataSetType.NONE;
+ public static int SEARCH_ARRAY_b_genFP_count = 0;
+ public static long SEARCH_ARRAY_b_genFP_ns = 0;
+
+ public static int SEARCH_ARRAY_b_genLP_count = 0;
+ public static long SEARCH_ARRAY_b_genLP_ns = 0;
+
+ public static int SEARCH_ARRAY_c_genBPTP_count = 0;
+ public static long SEARCH_ARRAY_c_genBPTP_ns = 0;
+
+ public static int M4_LSM_init_loadAllChunkMetadatas_count = 0;
+ public static long M4_LSM_init_loadAllChunkMetadatas_ns = 0;
- private static final Logger DEBUG_LOGGER = LoggerFactory.getLogger("IOMonitor2");
+ public static int M4_LSM_merge_M4_time_span_count = 0;
+ public static long M4_LSM_merge_M4_time_span_ns = 0;
+
+ public static int M4_LSM_FP_count = 0;
+ public static long M4_LSM_FP_ns = 0;
+
+ public static int M4_LSM_LP_count = 0;
+ public static long M4_LSM_LP_ns = 0;
+
+ public static int M4_LSM_BP_count = 0;
+ public static long M4_LSM_BP_ns = 0;
+
+ public static int M4_LSM_TP_count = 0;
+ public static long M4_LSM_TP_ns = 0;
private static void reset() {
+ dataSetType = DataSetType.NONE;
+
DCP_Server_Query_Execute_count = 0;
DCP_Server_Query_Execute_ns = 0;
DCP_Server_Query_Fetch_count = 0;
DCP_Server_Query_Fetch_ns = 0;
- DCP_SeriesScanOperator_hasNext_count = 0;
- DCP_SeriesScanOperator_hasNext_ns = 0;
-
DCP_A_GET_CHUNK_METADATAS_count = 0;
DCP_A_GET_CHUNK_METADATAS_ns = 0;
@@ -105,17 +127,39 @@ public class IOMonitor2 {
DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_count = 0;
DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_ns = 0;
+ DCP_D_traversedPointNum = 0;
DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_count = 0;
DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_ns = 0;
- DCP_D_traversedPointNum = 0;
+ SEARCH_ARRAY_a_verifBPTP_count = 0;
+ SEARCH_ARRAY_a_verifBPTP_ns = 0;
+ SEARCH_ARRAY_b_genFP_count = 0;
+ SEARCH_ARRAY_b_genFP_ns = 0;
+ SEARCH_ARRAY_b_genLP_count = 0;
+ SEARCH_ARRAY_b_genLP_ns = 0;
+ SEARCH_ARRAY_c_genBPTP_count = 0;
+ SEARCH_ARRAY_c_genBPTP_ns = 0;
- dataSetType = DataSetType.NONE;
+ M4_LSM_init_loadAllChunkMetadatas_count = 0;
+ M4_LSM_init_loadAllChunkMetadatas_ns = 0;
+
+ M4_LSM_merge_M4_time_span_count = 0;
+ M4_LSM_merge_M4_time_span_ns = 0;
+
+ M4_LSM_FP_count = 0;
+ M4_LSM_FP_ns = 0;
+
+ M4_LSM_LP_count = 0;
+ M4_LSM_LP_ns = 0;
+
+ M4_LSM_BP_count = 0;
+ M4_LSM_BP_ns = 0;
+
+ M4_LSM_TP_count = 0;
+ M4_LSM_TP_ns = 0;
}
public static void addMeasure(Operation operation, long elapsedTimeInNanosecond) {
- // TODO tmp for debug
- // DEBUG_LOGGER.info(operation.getName() + ": " + elapsedTimeInNanosecond + " ns");
switch (operation) {
case DCP_Server_Query_Execute:
DCP_Server_Query_Execute_count++;
@@ -125,10 +169,6 @@ public class IOMonitor2 {
DCP_Server_Query_Fetch_count++;
DCP_Server_Query_Fetch_ns += elapsedTimeInNanosecond;
break;
- case DCP_SeriesScanOperator_hasNext:
- DCP_SeriesScanOperator_hasNext_count++;
- DCP_SeriesScanOperator_hasNext_ns += elapsedTimeInNanosecond;
- break;
case DCP_A_GET_CHUNK_METADATAS:
DCP_A_GET_CHUNK_METADATAS_count++;
DCP_A_GET_CHUNK_METADATAS_ns += elapsedTimeInNanosecond;
@@ -145,6 +185,46 @@ public class IOMonitor2 {
DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_count++;
DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_ns += elapsedTimeInNanosecond;
break;
+ case SEARCH_ARRAY_a_verifBPTP:
+ SEARCH_ARRAY_a_verifBPTP_count++;
+ SEARCH_ARRAY_a_verifBPTP_ns += elapsedTimeInNanosecond;
+ break;
+ case SEARCH_ARRAY_b_genFP:
+ SEARCH_ARRAY_b_genFP_count++;
+ SEARCH_ARRAY_b_genFP_ns += elapsedTimeInNanosecond;
+ break;
+ case SEARCH_ARRAY_b_genLP:
+ SEARCH_ARRAY_b_genLP_count++;
+ SEARCH_ARRAY_b_genLP_ns += elapsedTimeInNanosecond;
+ break;
+ case SEARCH_ARRAY_c_genBPTP:
+ SEARCH_ARRAY_c_genBPTP_count++;
+ SEARCH_ARRAY_c_genBPTP_ns += elapsedTimeInNanosecond;
+ break;
+ case M4_LSM_INIT_LOAD_ALL_CHUNKMETADATAS:
+ M4_LSM_init_loadAllChunkMetadatas_count++;
+ M4_LSM_init_loadAllChunkMetadatas_ns += elapsedTimeInNanosecond;
+ break;
+ case M4_LSM_MERGE_M4_TIME_SPAN:
+ M4_LSM_merge_M4_time_span_count++;
+ M4_LSM_merge_M4_time_span_ns += elapsedTimeInNanosecond;
+ break;
+ case M4_LSM_FP:
+ M4_LSM_FP_count++;
+ M4_LSM_FP_ns += elapsedTimeInNanosecond;
+ break;
+ case M4_LSM_LP:
+ M4_LSM_LP_count++;
+ M4_LSM_LP_ns += elapsedTimeInNanosecond;
+ break;
+ case M4_LSM_BP:
+ M4_LSM_BP_count++;
+ M4_LSM_BP_ns += elapsedTimeInNanosecond;
+ break;
+ case M4_LSM_TP:
+ M4_LSM_TP_count++;
+ M4_LSM_TP_ns += elapsedTimeInNanosecond;
+ break;
default:
System.out.println("not supported operation type"); // this will not happen
break;
@@ -161,10 +241,6 @@ public class IOMonitor2 {
DCP_Server_Query_Fetch_count += count;
DCP_Server_Query_Fetch_ns += elapsedTimeInNanosecond;
break;
- case DCP_SeriesScanOperator_hasNext:
- DCP_SeriesScanOperator_hasNext_count += count;
- DCP_SeriesScanOperator_hasNext_ns += elapsedTimeInNanosecond;
- break;
case DCP_A_GET_CHUNK_METADATAS:
DCP_A_GET_CHUNK_METADATAS_count += count;
DCP_A_GET_CHUNK_METADATAS_ns += elapsedTimeInNanosecond;
@@ -181,6 +257,46 @@ public class IOMonitor2 {
DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_count += count;
DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_ns += elapsedTimeInNanosecond;
break;
+ case SEARCH_ARRAY_a_verifBPTP:
+ SEARCH_ARRAY_a_verifBPTP_count += count;
+ SEARCH_ARRAY_a_verifBPTP_ns += elapsedTimeInNanosecond;
+ break;
+ case SEARCH_ARRAY_b_genFP:
+ SEARCH_ARRAY_b_genFP_count += count;
+ SEARCH_ARRAY_b_genFP_ns += elapsedTimeInNanosecond;
+ break;
+ case SEARCH_ARRAY_b_genLP:
+ SEARCH_ARRAY_b_genLP_count += count;
+ SEARCH_ARRAY_b_genLP_ns += elapsedTimeInNanosecond;
+ break;
+ case SEARCH_ARRAY_c_genBPTP:
+ SEARCH_ARRAY_c_genBPTP_count += count;
+ SEARCH_ARRAY_c_genBPTP_ns += elapsedTimeInNanosecond;
+ break;
+ case M4_LSM_INIT_LOAD_ALL_CHUNKMETADATAS:
+ M4_LSM_init_loadAllChunkMetadatas_count += count;
+ M4_LSM_init_loadAllChunkMetadatas_ns += elapsedTimeInNanosecond;
+ break;
+ case M4_LSM_MERGE_M4_TIME_SPAN:
+ M4_LSM_merge_M4_time_span_count += count;
+ M4_LSM_merge_M4_time_span_ns += elapsedTimeInNanosecond;
+ break;
+ case M4_LSM_FP:
+ M4_LSM_FP_count += count;
+ M4_LSM_FP_ns += elapsedTimeInNanosecond;
+ break;
+ case M4_LSM_LP:
+ M4_LSM_LP_count += count;
+ M4_LSM_LP_ns += elapsedTimeInNanosecond;
+ break;
+ case M4_LSM_BP:
+ M4_LSM_BP_count += count;
+ M4_LSM_BP_ns += elapsedTimeInNanosecond;
+ break;
+ case M4_LSM_TP:
+ M4_LSM_TP_count += count;
+ M4_LSM_TP_ns += elapsedTimeInNanosecond;
+ break;
default:
System.out.println("not supported operation type"); // this will not happen
break;
@@ -202,22 +318,59 @@ public class IOMonitor2 {
.append(DCP_Server_Query_Fetch_ns)
.append("\n");
stringBuilder
- .append("SeriesScanOperator_hasNext_ns")
+ .append("M4_LSM_init_loadAllChunkMetadatas_ns")
+ .append(",")
+ .append(M4_LSM_init_loadAllChunkMetadatas_ns)
+ .append("\n");
+ stringBuilder
+ .append("M4_LSM_merge_M4_time_span_ns")
+ .append(",")
+ .append(M4_LSM_merge_M4_time_span_ns)
+ .append("\n");
+ stringBuilder.append("M4_LSM_FP_ns").append(",").append(M4_LSM_FP_ns).append("\n");
+ stringBuilder.append("M4_LSM_LP_ns").append(",").append(M4_LSM_LP_ns).append("\n");
+ stringBuilder.append("M4_LSM_BP_ns").append(",").append(M4_LSM_BP_ns).append("\n");
+ stringBuilder.append("M4_LSM_TP_ns").append(",").append(M4_LSM_TP_ns).append("\n");
+ stringBuilder
+ .append("DCP_A_GET_CHUNK_METADATAS_ns")
+ .append(",")
+ .append(DCP_A_GET_CHUNK_METADATAS_ns)
+ .append("\n");
+ stringBuilder
+ .append("DCP_B_READ_MEM_CHUNK_ns")
.append(",")
- .append(DCP_SeriesScanOperator_hasNext_ns)
+ .append(DCP_B_READ_MEM_CHUNK_ns)
.append("\n");
- stringBuilder.append("A_ns").append(",").append(DCP_A_GET_CHUNK_METADATAS_ns).append("\n");
- stringBuilder.append("B_ns").append(",").append(DCP_B_READ_MEM_CHUNK_ns).append("\n");
stringBuilder
- .append("C_ns")
+ .append("DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_ns")
.append(",")
.append(DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_ns)
.append("\n");
stringBuilder
- .append("D_ns")
+ .append("DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_ns")
.append(",")
.append(DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_ns)
.append("\n");
+ stringBuilder
+ .append("SEARCH_ARRAY_a_verifBPTP_ns")
+ .append(",")
+ .append(SEARCH_ARRAY_a_verifBPTP_ns)
+ .append("\n");
+ stringBuilder
+ .append("SEARCH_ARRAY_b_genFP_ns")
+ .append(",")
+ .append(SEARCH_ARRAY_b_genFP_ns)
+ .append("\n");
+ stringBuilder
+ .append("SEARCH_ARRAY_b_genLP_ns")
+ .append(",")
+ .append(SEARCH_ARRAY_b_genLP_ns)
+ .append("\n");
+ stringBuilder
+ .append("SEARCH_ARRAY_c_genBPTP_ns")
+ .append(",")
+ .append(SEARCH_ARRAY_c_genBPTP_ns)
+ .append("\n");
stringBuilder
.append("Server_Query_Execute_cnt")
@@ -230,22 +383,59 @@ public class IOMonitor2 {
.append(DCP_Server_Query_Fetch_count)
.append("\n");
stringBuilder
- .append("SeriesScanOperator_hasNext_cnt")
+ .append("M4_LSM_init_loadAllChunkMetadatas_cnt")
+ .append(",")
+ .append(M4_LSM_init_loadAllChunkMetadatas_count)
+ .append("\n");
+ stringBuilder
+ .append("M4_LSM_merge_M4_time_span_cnt")
.append(",")
- .append(DCP_SeriesScanOperator_hasNext_count)
+ .append(M4_LSM_merge_M4_time_span_count)
.append("\n");
- stringBuilder.append("A_cnt").append(",").append(DCP_A_GET_CHUNK_METADATAS_count).append("\n");
- stringBuilder.append("B_cnt").append(",").append(DCP_B_READ_MEM_CHUNK_count).append("\n");
+ stringBuilder.append("M4_LSM_FP_cnt").append(",").append(M4_LSM_FP_count).append("\n");
+ stringBuilder.append("M4_LSM_LP_cnt").append(",").append(M4_LSM_LP_count).append("\n");
+ stringBuilder.append("M4_LSM_BP_cnt").append(",").append(M4_LSM_BP_count).append("\n");
+ stringBuilder.append("M4_LSM_TP_cnt").append(",").append(M4_LSM_TP_count).append("\n");
stringBuilder
- .append("C_cnt")
+ .append("DCP_A_GET_CHUNK_METADATAS_cnt")
+ .append(",")
+ .append(DCP_A_GET_CHUNK_METADATAS_count)
+ .append("\n");
+ stringBuilder
+ .append("DCP_B_READ_MEM_CHUNK_cnt")
+ .append(",")
+ .append(DCP_B_READ_MEM_CHUNK_count)
+ .append("\n");
+ stringBuilder
+ .append("DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_cnt")
.append(",")
.append(DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_count)
.append("\n");
stringBuilder
- .append("D_cnt")
+ .append("DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_cnt")
.append(",")
.append(DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_count)
.append("\n");
+ stringBuilder
+ .append("SEARCH_ARRAY_a_verifBPTP_cnt")
+ .append(",")
+ .append(SEARCH_ARRAY_a_verifBPTP_count)
+ .append("\n");
+ stringBuilder
+ .append("SEARCH_ARRAY_b_genFP_cnt")
+ .append(",")
+ .append(SEARCH_ARRAY_b_genFP_count)
+ .append("\n");
+ stringBuilder
+ .append("SEARCH_ARRAY_b_genLP_cnt")
+ .append(",")
+ .append(SEARCH_ARRAY_b_genLP_count)
+ .append("\n");
+ stringBuilder
+ .append("SEARCH_ARRAY_c_genBPTP_cnt")
+ .append(",")
+ .append(SEARCH_ARRAY_c_genBPTP_count)
+ .append("\n");
stringBuilder
.append("DCP_D_traversedPointNum")
.append(",")
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
index 420a32735ac..cc667ab3844 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
@@ -168,6 +168,7 @@ public class PageReader implements IPageReader {
}
public void updateBPTP(ChunkSuit4CPV chunkSuit4CPV) {
+ long start = System.nanoTime();
deleteCursor = 0; // TODO DEBUG
Statistics statistics = null;
switch (dataType) {
@@ -187,6 +188,7 @@ public class PageReader implements IPageReader {
int count = 0; // update here, not in statistics
for (int pos = chunkSuit4CPV.startPos; pos <= chunkSuit4CPV.endPos; pos++) {
IOMonitor.incPointsTravered();
+ IOMonitor2.DCP_D_traversedPointNum++;
long timestamp = timeBuffer.getLong(pos * 8);
switch (dataType) {
case INT64:
@@ -229,6 +231,7 @@ public class PageReader implements IPageReader {
} else {
chunkSuit4CPV.statistics.setCount(0); // otherwise count won't be zero
}
+ IOMonitor2.addMeasure(Operation.SEARCH_ARRAY_c_genBPTP, System.nanoTime() - start);
}
/** @return the returned BatchData may be empty, but never be null */
[iotdb] 33/46: save first and last points
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit da7fa9210a0b3847a47d589fbfec453ebcf505ab
Author: Lei Rui <10...@qq.com>
AuthorDate: Tue Jul 4 18:42:19 2023 +0800
save first and last points
---
.../file/metadata/statistics/Statistics.java | 98 +++++++++++++++++-----
.../file/metadata/statistics/ValueIndex.java | 37 +++++---
2 files changed, 101 insertions(+), 34 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java
index 18e50e7be27..c712d1967f0 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java
@@ -130,17 +130,17 @@ public abstract class Statistics<T> {
byteLen += ReadWriteForEncodingUtils.writeUnsignedVarInt(count, outputStream);
byteLen += ReadWriteIOUtils.write(startTime, outputStream);
byteLen += ReadWriteIOUtils.write(endTime, outputStream);
+ // value statistics of different data type
+ byteLen += serializeStats(outputStream);
// serialize stepRegress
byteLen += serializeStepRegress(outputStream);
- // TODO serialize value index
+ // serialize value index
byteLen += serializeValueIndex(outputStream);
- // value statistics of different data type
- byteLen += serializeStats(outputStream);
return byteLen;
}
int serializeValueIndex(OutputStream outputStream) throws IOException {
- valueIndex.learn(); // TODO ensure executed once and only once
+ valueIndex.learn(); // ensure executed once and only once
int byteLen = 0;
byteLen += ReadWriteIOUtils.write(valueIndex.idxOut.size(), outputStream);
outputStream.write(
@@ -162,7 +162,7 @@ public abstract class Statistics<T> {
* last segment keys are not serialized here, because they are minTime and endTime respectively.
*/
int serializeStepRegress(OutputStream outputStream) throws IOException {
- stepRegress.learn(); // TODO ensure executed once and only once
+ stepRegress.learn(); // ensure executed once and only once
int byteLen = 0;
byteLen += ReadWriteIOUtils.write(stepRegress.getSlope(), outputStream); // K
DoubleArrayList segmentKeys = stepRegress.getSegmentKeys();
@@ -574,35 +574,91 @@ public abstract class Statistics<T> {
statistics.setCount(ReadWriteForEncodingUtils.readUnsignedVarInt(buffer));
statistics.setStartTime(ReadWriteIOUtils.readLong(buffer));
statistics.setEndTime(ReadWriteIOUtils.readLong(buffer));
- statistics.deserializeStepRegress(buffer);
- statistics.deserializeValueIndex(buffer); // TODO
statistics.deserialize(buffer);
+ statistics.deserializeStepRegress(buffer);
+ statistics.deserializeValueIndex(buffer);
statistics.isEmpty = false;
return statistics;
}
void deserializeValueIndex(ByteBuffer buffer) throws IOException {
+ // add the first point
+ valueIndex.modelPointIdx_list.add(1);
+ switch (getType()) {
+ case INT32:
+ int intV = (int) getFirstValue();
+ valueIndex.modelPointVal_list.add((double) intV);
+ break;
+ case INT64:
+ long longV = (long) getFirstValue();
+ valueIndex.modelPointVal_list.add((double) longV);
+ break;
+ case FLOAT:
+ float floatV = (float) getFirstValue();
+ valueIndex.modelPointVal_list.add((double) floatV);
+ break;
+ case DOUBLE:
+ double doubleV = (double) getFirstValue();
+ valueIndex.modelPointVal_list.add(doubleV);
+ break;
+ default:
+ throw new IOException("unsupported");
+ }
+
int idxSize = ReadWriteIOUtils.readInt(buffer);
- ByteBuffer idxBuffer = buffer.slice();
- idxBuffer.limit(idxSize);
+ if (idxSize > 0) {
+ ByteBuffer idxBuffer = buffer.slice();
+ idxBuffer.limit(idxSize);
+ Decoder idxDecoder = new IntDeltaDecoder();
+ while (idxDecoder.hasNext(idxBuffer)) {
+ int idx = idxDecoder.readInt(idxBuffer);
+ valueIndex.modelPointIdx_list.add(idx);
+ }
+ }
+
buffer.position(buffer.position() + idxSize);
- Decoder idxDecoder = new IntDeltaDecoder();
- while (idxDecoder.hasNext(idxBuffer)) {
- int idx = idxDecoder.readInt(idxBuffer);
- valueIndex.modelPointIdx_list.add(idx);
+ int valueSize = ReadWriteIOUtils.readInt(buffer);
+ if (valueSize > 0) {
+ ByteBuffer valueBuffer = buffer.slice();
+ valueBuffer.limit(valueSize);
+ Decoder valueDecoder = new DoublePrecisionDecoderV2();
+ while (valueDecoder.hasNext(valueBuffer)) {
+ double value = valueDecoder.readDouble(valueBuffer);
+ valueIndex.modelPointVal_list.add(value);
+ }
}
- int valueSize = ReadWriteIOUtils.readInt(buffer); // valueOut.size()
- ByteBuffer valueBuffer = buffer.slice();
- valueBuffer.limit(valueSize);
- buffer.position(buffer.position() + valueSize);
- Decoder valueDecoder = new DoublePrecisionDecoderV2();
- while (valueDecoder.hasNext(valueBuffer)) {
- double value = valueDecoder.readDouble(valueBuffer);
- valueIndex.modelPointVal_list.add(value);
+ // add the last point except the first point
+ if (count >= 2) { // otherwise only one point no need to store again
+ valueIndex.modelPointIdx_list.add(count);
+ switch (getType()) {
+ case INT32:
+ int intV = (int) getLastValue();
+ valueIndex.modelPointVal_list.add((double) intV);
+ break;
+ case INT64:
+ long longV = (long) getLastValue();
+ valueIndex.modelPointVal_list.add((double) longV);
+ break;
+ case FLOAT:
+ float floatV = (float) getLastValue();
+ valueIndex.modelPointVal_list.add((double) floatV);
+ break;
+ case DOUBLE:
+ double doubleV = (double) getLastValue();
+ valueIndex.modelPointVal_list.add(doubleV);
+ break;
+ default:
+ throw new IOException("unsupported");
+ }
}
+ // error bound
+ buffer.position(buffer.position() + valueSize);
valueIndex.errorBound = ReadWriteIOUtils.readDouble(buffer);
+
+ // System.out.println(valueIndex.modelPointIdx_list);
+ // System.out.println(valueIndex.modelPointVal_list);
}
void deserializeStepRegress(ByteBuffer byteBuffer) {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/ValueIndex.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/ValueIndex.java
index 24deea0a871..1ad935e6ef8 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/ValueIndex.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/ValueIndex.java
@@ -17,11 +17,6 @@ public class ValueIndex {
public DoublePrecisionEncoderV2 valueEncoder = new DoublePrecisionEncoderV2();
public PublicBAOS idxOut = new PublicBAOS();
public PublicBAOS valueOut = new PublicBAOS();
- public int lastIdx = 0;
- public int lastIntValue = 0;
- public long lastLongValue = 0;
- public float lastFloatValue = 0;
- public double lastDoubleValue = 0;
public IntArrayList modelPointIdx_list = new IntArrayList();
public DoubleArrayList modelPointVal_list = new DoubleArrayList();
@@ -82,19 +77,35 @@ public class ValueIndex {
isLearned = true;
initForLearn(); // set self-adapting CompDeviation for sdtEncoder
int pos = 0;
+ boolean hasDataToFlush = false;
for (double v : values.toArray()) {
pos++; // starting from 1
if (sdtEncoder.encodeDouble(pos, v)) {
- idxEncoder.encode((int) sdtEncoder.getTime(), idxOut);
- valueEncoder.encode(sdtEncoder.getDoubleValue(), valueOut);
+ if (pos > 1) {
+ // the first point value is stored as FirstValue in statistics, so here no need store the
+ // first point
+ // the last point won't be checked by the if SDT encode logic
+ idxEncoder.encode((int) sdtEncoder.getTime(), idxOut);
+ valueEncoder.encode(sdtEncoder.getDoubleValue(), valueOut);
+ if (!hasDataToFlush) {
+ hasDataToFlush = true;
+ }
+ }
}
}
- // add the last point except the first point
- if (values.size() >= 2) { // means there is last point except the first point
- idxEncoder.encode(pos, idxOut);
- valueEncoder.encode(values.getLast(), valueOut);
+ // // add the last point except the first point
+ // if (values.size() >= 2) { // means there is last point except the first point
+ // idxEncoder.encode(pos, idxOut);
+ // valueEncoder.encode(values.getLast(), valueOut);
+ // }
+
+ if (hasDataToFlush) {
+ // otherwise no need flush, because GorillaV2 encoding will output NaN even if
+ // hasDataToFlush=false
+ idxEncoder.flush(idxOut); // necessary
+ valueEncoder.flush(valueOut); // necessary
}
- idxEncoder.flush(idxOut);
- valueEncoder.flush(valueOut);
+
+ values = null; // raw values are not needed any more
}
}
[iotdb] 08/46: add m4_chunk_metadata log debug for visualization
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit bf15b534d8556f6ce423542ebc45e8aad73b4aff
Author: Lei Rui <10...@qq.com>
AuthorDate: Tue Jun 13 10:41:43 2023 +0800
add m4_chunk_metadata log debug for visualization
---
server/src/assembly/resources/conf/logback.xml | 20 +++++
.../dataset/groupby/LocalGroupByExecutor4CPV.java | 97 ++++++++++++++++++++++
2 files changed, 117 insertions(+)
diff --git a/server/src/assembly/resources/conf/logback.xml b/server/src/assembly/resources/conf/logback.xml
index f346e514bd1..9c640808a9c 100644
--- a/server/src/assembly/resources/conf/logback.xml
+++ b/server/src/assembly/resources/conf/logback.xml
@@ -231,6 +231,23 @@
<level>INFO</level>
</filter>
</appender>
+ <appender class="ch.qos.logback.core.rolling.RollingFileAppender" name="M4_CHUNK_METADATA">
+ <file>${IOTDB_HOME}/logs/log_m4_chunk_metadata.log</file>
+ <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+ <fileNamePattern>${IOTDB_HOME}/logs/log-m4-chunk-metadata-%d{yyyyMMdd}.%i.log.gz</fileNamePattern>
+ <maxFileSize>100MB</maxFileSize>
+ <maxHistory>168</maxHistory>
+ <totalSizeCap>512MB</totalSizeCap>
+ </rollingPolicy>
+ <append>true</append>
+ <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+ <pattern>%d [%t] %-5p %C{25}:%L - %m %n</pattern>
+ <charset>utf-8</charset>
+ </encoder>
+ <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+ <level>DEBUG</level>
+ </filter>
+ </appender>
<appender class="ch.qos.logback.core.rolling.RollingFileAppender" name="QUERY_FREQUENCY">
<file>${IOTDB_HOME}/logs/log_query_frequency.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
@@ -275,6 +292,9 @@
<logger level="info" name="SLOW_SQL">
<appender-ref ref="SLOW_SQL"/>
</logger>
+ <logger level="info" name="M4_CHUNK_METADATA">
+ <appender-ref ref="M4_CHUNK_METADATA"/>
+ </logger>
<logger level="info" name="QUERY_FREQUENCY">
<appender-ref ref="QUERY_FREQUENCY"/>
</logger>
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java
index 3d3f6c4b767..3cb9b812452 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java
@@ -32,12 +32,20 @@ import org.apache.iotdb.db.query.reader.universal.PriorityMergeReader.MergeReade
import org.apache.iotdb.db.utils.FileLoaderUtils;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.statistics.DoubleStatistics;
+import org.apache.iotdb.tsfile.file.metadata.statistics.FloatStatistics;
+import org.apache.iotdb.tsfile.file.metadata.statistics.IntegerStatistics;
+import org.apache.iotdb.tsfile.file.metadata.statistics.LongStatistics;
+import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.common.ChunkSuit4CPV;
import org.apache.iotdb.tsfile.read.common.TimeRange;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.read.reader.page.PageReader;
import org.apache.iotdb.tsfile.utils.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.Comparator;
@@ -56,6 +64,8 @@ import java.util.Set;
// This is the CPVGroupByExecutor in M4-LSM paper.
public class LocalGroupByExecutor4CPV implements GroupByExecutor {
+ private static final Logger M4_CHUNK_METADATA = LoggerFactory.getLogger("M4_CHUNK_METADATA");
+
// Aggregate result buffer of this path
private final List<AggregateResult> results = new ArrayList<>();
// private final TimeRange timeRange;
@@ -117,6 +127,93 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
.compareTo(o2.getChunkMetadata().getStartTime());
}
});
+
+ if (M4_CHUNK_METADATA.isDebugEnabled()) {
+ for (ChunkSuit4CPV chunkSuit4CPV : futureChunkList) {
+ Statistics statistics = chunkSuit4CPV.getChunkMetadata().getStatistics();
+ long FP_t = statistics.getStartTime();
+ long LP_t = statistics.getEndTime();
+ long BP_t = statistics.getBottomTimestamp();
+ long TP_t = statistics.getTopTimestamp();
+ switch (statistics.getType()) {
+ case INT32:
+ int FP_v_int = ((IntegerStatistics) statistics).getFirstValue();
+ int LP_v_int = ((IntegerStatistics) statistics).getLastValue();
+ int BP_v_int = ((IntegerStatistics) statistics).getMinValue();
+ int TP_v_int = ((IntegerStatistics) statistics).getMaxValue();
+ M4_CHUNK_METADATA.debug(
+ "M4_CHUNK_METADATA,{},{},{},{},{},{},{},{},{},{}",
+ FP_t,
+ LP_t,
+ BP_t,
+ TP_t,
+ FP_v_int,
+ LP_v_int,
+ BP_v_int,
+ TP_v_int,
+ chunkSuit4CPV.getChunkMetadata().getVersion(),
+ chunkSuit4CPV.getChunkMetadata().getOffsetOfChunkHeader());
+ break;
+ case INT64:
+ long FP_v_long = ((LongStatistics) statistics).getFirstValue();
+ long LP_v_long = ((LongStatistics) statistics).getLastValue();
+ long BP_v_long = ((LongStatistics) statistics).getMinValue();
+ long TP_v_long = ((LongStatistics) statistics).getMaxValue();
+ M4_CHUNK_METADATA.debug(
+ "M4_CHUNK_METADATA,{},{},{},{},{},{},{},{},{},{}",
+ FP_t,
+ LP_t,
+ BP_t,
+ TP_t,
+ FP_v_long,
+ LP_v_long,
+ BP_v_long,
+ TP_v_long,
+ chunkSuit4CPV.getChunkMetadata().getVersion(),
+ chunkSuit4CPV.getChunkMetadata().getOffsetOfChunkHeader());
+ break;
+ case FLOAT:
+ float FP_v_float = ((FloatStatistics) statistics).getFirstValue();
+ float LP_v_float = ((FloatStatistics) statistics).getLastValue();
+ float BP_v_float = ((FloatStatistics) statistics).getMinValue();
+ float TP_v_float = ((FloatStatistics) statistics).getMaxValue();
+ M4_CHUNK_METADATA.debug(
+ "M4_CHUNK_METADATA,{},{},{},{},{},{},{},{},{},{}",
+ FP_t,
+ LP_t,
+ BP_t,
+ TP_t,
+ FP_v_float,
+ LP_v_float,
+ BP_v_float,
+ TP_v_float,
+ chunkSuit4CPV.getChunkMetadata().getVersion(),
+ chunkSuit4CPV.getChunkMetadata().getOffsetOfChunkHeader());
+ break;
+ case DOUBLE:
+ double FP_v_double = ((DoubleStatistics) statistics).getFirstValue();
+ double LP_v_double = ((DoubleStatistics) statistics).getLastValue();
+ double BP_v_double = ((DoubleStatistics) statistics).getMinValue();
+ double TP_v_double = ((DoubleStatistics) statistics).getMaxValue();
+ M4_CHUNK_METADATA.debug(
+ "M4_CHUNK_METADATA,{},{},{},{},{},{},{},{},{},{}",
+ FP_t,
+ LP_t,
+ BP_t,
+ TP_t,
+ FP_v_double,
+ LP_v_double,
+ BP_v_double,
+ TP_v_double,
+ chunkSuit4CPV.getChunkMetadata().getVersion(),
+ chunkSuit4CPV.getChunkMetadata().getOffsetOfChunkHeader());
+ break;
+ default:
+ throw new QueryProcessException("unsupported data type!");
+ }
+ }
+ }
+
} catch (IOException e) {
throw new QueryProcessException(e.getMessage());
}
[iotdb] 24/46: update dataSetType
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 8fd0e7e20815fda383675005ce97d1af339024ae
Author: Lei Rui <10...@qq.com>
AuthorDate: Sun Jun 18 17:08:52 2023 +0800
update dataSetType
---
.../groupby/GroupByWithoutValueFilterDataSet.java | 22 ++++++++++++++++------
.../iotdb/tsfile/read/common/IOMonitor2.java | 10 ++++++----
2 files changed, 22 insertions(+), 10 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
index 05ae0fc6de2..0b8d58f145e 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
@@ -31,6 +31,7 @@ import org.apache.iotdb.db.query.aggregation.AggregateResult;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.factory.AggregateResultFactory;
import org.apache.iotdb.db.query.filter.TsFileFilter;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.IOMonitor2;
import org.apache.iotdb.tsfile.read.common.IOMonitor2.DataSetType;
@@ -200,14 +201,23 @@ public class GroupByWithoutValueFilterDataSet extends GroupByEngineDataSet {
boolean ascending)
throws StorageEngineException, QueryProcessException {
if (CONFIG.isEnableCPV()) {
- // System.out.println("====DEBUG====: use LocalGroupByExecutor4CPV for CPV");
- IOMonitor2.dataSetType =
- DataSetType.GroupByWithoutValueFilterDataSet_LocalGroupByExecutor4CPV;
+ if (TSFileDescriptor.getInstance().getConfig().isUseChunkIndex()) {
+ IOMonitor2.dataSetType =
+ DataSetType.GroupByWithoutValueFilterDataSet_LocalGroupByExecutor4CPV_UseIndex;
+ } else {
+ IOMonitor2.dataSetType =
+ DataSetType.GroupByWithoutValueFilterDataSet_LocalGroupByExecutor4CPV_NotUseIndex;
+ }
return new LocalGroupByExecutor4CPV(
path, allSensors, dataType, context, timeFilter, fileFilter, ascending);
- } else {
- // System.out.println("====DEBUG====: use LocalGroupByExecutor for MOC");
- IOMonitor2.dataSetType = DataSetType.GroupByWithoutValueFilterDataSet_LocalGroupByExecutor;
+ } else { // enableCPV=false
+ if (TSFileDescriptor.getInstance().getConfig().isUseStatistics()) {
+ IOMonitor2.dataSetType =
+ DataSetType.GroupByWithoutValueFilterDataSet_LocalGroupByExecutor_UseStatistics;
+ } else {
+ IOMonitor2.dataSetType =
+ DataSetType.GroupByWithoutValueFilterDataSet_LocalGroupByExecutor_NotUseStatistics;
+ }
return new LocalGroupByExecutor(
path, allSensors, dataType, context, timeFilter, fileFilter, ascending);
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
index d9b436b8c5e..62e79e40fac 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
@@ -24,11 +24,13 @@ public class IOMonitor2 {
public enum DataSetType { // dataSet, executor, reader, file
NONE,
RawQueryDataSetWithoutValueFilter,
- UDTFAlignByTimeDataSet_M4_POINT,
- UDTFAlignByTimeDataSet_M4_TIMEWINDOW,
- GroupByWithoutValueFilterDataSet_LocalGroupByExecutor4CPV,
+ UDTFAlignByTimeDataSet_M4_POINT, // mac
+ UDTFAlignByTimeDataSet_M4_TIMEWINDOW, // mac_tw
+ GroupByWithoutValueFilterDataSet_LocalGroupByExecutor4CPV_UseIndex, // cpv
+ GroupByWithoutValueFilterDataSet_LocalGroupByExecutor4CPV_NotUseIndex, // cpv_noIndex
- GroupByWithoutValueFilterDataSet_LocalGroupByExecutor
+ GroupByWithoutValueFilterDataSet_LocalGroupByExecutor_UseStatistics, // moc
+ GroupByWithoutValueFilterDataSet_LocalGroupByExecutor_NotUseStatistics // mac_groupBy
}
public enum Operation {
[iotdb] 09/46: add M4_QUERY_PARAM for log debug for visualization
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit aeaafe1b7e43ad687f985a104d3bd027987e7817
Author: Lei Rui <10...@qq.com>
AuthorDate: Tue Jun 13 13:45:41 2023 +0800
add M4_QUERY_PARAM for log debug for visualization
---
.../iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java | 8 ++++++++
.../java/org/apache/iotdb/tsfile/read/filter/GroupByFilter.java | 4 ++++
2 files changed, 12 insertions(+)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java
index 3cb9b812452..b4c59643688 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java
@@ -39,6 +39,7 @@ import org.apache.iotdb.tsfile.file.metadata.statistics.LongStatistics;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.common.ChunkSuit4CPV;
import org.apache.iotdb.tsfile.read.common.TimeRange;
+import org.apache.iotdb.tsfile.read.filter.GroupByFilter;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.read.reader.page.PageReader;
import org.apache.iotdb.tsfile.utils.Pair;
@@ -129,6 +130,13 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor {
});
if (M4_CHUNK_METADATA.isDebugEnabled()) {
+ if (timeFilter instanceof GroupByFilter) {
+ M4_CHUNK_METADATA.debug(
+ "M4_QUERY_PARAM,{},{},{}",
+ ((GroupByFilter) timeFilter).getStartTime(),
+ ((GroupByFilter) timeFilter).getEndTime(),
+ ((GroupByFilter) timeFilter).getInterval());
+ }
for (ChunkSuit4CPV chunkSuit4CPV : futureChunkList) {
Statistics statistics = chunkSuit4CPV.getChunkMetadata().getStatistics();
long FP_t = statistics.getStartTime();
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/GroupByFilter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/GroupByFilter.java
index d53b450e91c..d44e3cf104e 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/GroupByFilter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/GroupByFilter.java
@@ -145,4 +145,8 @@ public class GroupByFilter implements Filter, Serializable {
public long getEndTime() {
return endTime;
}
+
+ public long getInterval() {
+ return interval;
+ }
}
[iotdb] 16/46: deprecate old IOMonitor
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit b9d14edd76b54fafb2b16b8458d5ca78f8e0b538
Author: Lei Rui <10...@qq.com>
AuthorDate: Wed Jun 14 22:52:45 2023 +0800
deprecate old IOMonitor
---
.../apache/iotdb/db/engine/cache/ChunkCache.java | 3 +-
.../org/apache/iotdb/db/service/TSServiceImpl.java | 14 +-
.../org/apache/iotdb/db/utils/FileLoaderUtils.java | 16 +-
.../iotdb/tsfile/read/common/ChunkSuit4CPV.java | 26 +-
.../apache/iotdb/tsfile/read/common/IOMonitor.java | 445 +++++++++++----------
.../iotdb/tsfile/read/reader/page/PageReader.java | 13 +-
6 files changed, 256 insertions(+), 261 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
index 204a99e76ef..9204c10b9ec 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
@@ -26,7 +26,6 @@ import org.apache.iotdb.db.utils.TestOnly;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Chunk;
-import org.apache.iotdb.tsfile.read.common.IOMonitor;
import org.apache.iotdb.tsfile.read.common.IOMonitor2;
import org.apache.iotdb.tsfile.read.common.IOMonitor2.Operation;
import org.apache.iotdb.tsfile.utils.RamUsageEstimator;
@@ -84,7 +83,7 @@ public class ChunkCache {
Chunk ret = reader.readMemChunk(chunkMetadata);
IOMonitor2.addMeasure(
Operation.DCP_B_READ_MEM_CHUNK, System.nanoTime() - start);
- IOMonitor.incReadMemChunkTime(System.nanoTime() - start);
+ // IOMonitor.incReadMemChunkTime(System.nanoTime() - start);
return ret;
} catch (IOException e) {
logger.error("Something wrong happened in reading {}", chunkMetadata, e);
diff --git a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
index 4c396572e80..b8338773126 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
@@ -127,7 +127,6 @@ import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
-import org.apache.iotdb.tsfile.read.common.IOMonitor;
import org.apache.iotdb.tsfile.read.common.IOMonitor2;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet;
@@ -670,8 +669,9 @@ public class TSServiceImpl implements TSIService.Iface {
public TSExecuteFinishResp executeFinish() throws TException {
TSExecuteFinishResp ret = new TSExecuteFinishResp();
ret.setStatus(RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS));
- ret.setExecutionInfo(IOMonitor.print() + "\n" + IOMonitor2.print());
- IOMonitor.finish();
+ // ret.setExecutionInfo(IOMonitor.print() + "\n" + IOMonitor2.print());
+ // IOMonitor.finish();
+ ret.setExecutionInfo(IOMonitor2.print()); // reset is called at the end of IOMonitor2.print()
return ret;
}
@@ -724,8 +724,8 @@ public class TSServiceImpl implements TSIService.Iface {
TException, AuthException {
// start record execution time
- IOMonitor.setSQL(statement);
- long start = System.nanoTime();
+ // IOMonitor.setSQL(statement);
+ // long start = System.nanoTime();
queryCount.incrementAndGet();
AUDIT_LOGGER.debug(
"Session {} execute Query: {}", sessionManager.getCurrSessionId(), statement);
@@ -836,8 +836,8 @@ public class TSServiceImpl implements TSIService.Iface {
queryTimeManager.unRegisterQuery(queryId);
}
- IOMonitor.incTotalTime(System.nanoTime() - start);
- IOMonitor.reset();
+ // IOMonitor.incTotalTime(System.nanoTime() - start);
+ // IOMonitor.reset();
return resp;
} catch (Exception e) {
releaseQueryResourceNoExceptions(queryId);
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
index 59c2201251b..8993f6797f3 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
@@ -33,7 +33,6 @@ import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.file.metadata.TsFileMetadata;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Chunk;
-import org.apache.iotdb.tsfile.read.common.IOMonitor;
import org.apache.iotdb.tsfile.read.common.IOMonitor2;
import org.apache.iotdb.tsfile.read.common.IOMonitor2.Operation;
import org.apache.iotdb.tsfile.read.common.Path;
@@ -144,8 +143,7 @@ public class FileLoaderUtils {
return null;
}
}
- long duration = System.nanoTime() - start;
- IOMonitor.incMeta(duration);
+ // IOMonitor.incMeta(duration);
IOMonitor2.addMeasure(Operation.DCP_A_GET_CHUNK_METADATAS, System.nanoTime() - start);
return timeSeriesMetadata;
}
@@ -168,7 +166,7 @@ public class FileLoaderUtils {
*/
public static List<IPageReader> loadPageReaderList(ChunkMetadata chunkMetaData, Filter timeFilter)
throws IOException {
- long start = System.nanoTime();
+ // long start = System.nanoTime();
if (chunkMetaData == null) {
throw new IOException("Can't init null chunkMeta");
}
@@ -185,8 +183,8 @@ public class FileLoaderUtils {
new ChunkReader(chunk, timeFilter); // decompress page data, split time&value buffers
chunkReader.hasNextSatisfiedPage();
}
- long duration = System.nanoTime() - start;
- IOMonitor.incDataIOTime(duration);
+ // long duration = System.nanoTime() - start;
+ // IOMonitor.incDataIOTime(duration);
return chunkReader.loadPageReaderList();
} catch (IOException e) {
logger.error(
@@ -203,7 +201,7 @@ public class FileLoaderUtils {
*/
public static PageReader loadPageReaderList4CPV(ChunkMetadata chunkMetaData, Filter timeFilter)
throws IOException {
- long start = System.nanoTime();
+ // long start = System.nanoTime();
if (chunkMetaData == null) {
throw new IOException("Can't init null chunkMeta");
}
@@ -220,8 +218,8 @@ public class FileLoaderUtils {
new ChunkReader(chunk, timeFilter); // decompress page data, split time&value buffers
chunkReader.hasNextSatisfiedPage();
}
- long duration = System.nanoTime() - start;
- IOMonitor.incDataIOTime(duration);
+ // long duration = System.nanoTime() - start;
+ // IOMonitor.incDataIOTime(duration);
List<IPageReader> pageReaderList = chunkReader.loadPageReaderList();
if (pageReaderList.size() > 1) {
// TODO ATTENTION: YOU HAVE TO ENSURE THAT THERE IS ONLY ONE PAGE IN A CHUNK,
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
index d1396da497b..453fe2c6e2b 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
@@ -258,18 +258,18 @@ public class ChunkSuit4CPV {
if (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
while (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
estimatedPos++;
- IOMonitor.incPointsTravered();
+ // IOMonitor.incPointsTravered();
IOMonitor2.DCP_D_traversedPointNum++;
}
} else if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
while (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
estimatedPos--;
- IOMonitor.incPointsTravered();
+ // IOMonitor.incPointsTravered();
IOMonitor2.DCP_D_traversedPointNum++;
}
if (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
estimatedPos++;
- IOMonitor.incPointsTravered();
+ // IOMonitor.incPointsTravered();
IOMonitor2.DCP_D_traversedPointNum++;
} // else equal
} // else equal
@@ -312,7 +312,7 @@ public class ChunkSuit4CPV {
pageReader.valueBuffer.position(pageReader.timeBufferLength);
while (pageReader.timeBuffer.remaining() > 0) {
estimatedPos++;
- IOMonitor.incPointsTravered();
+ // IOMonitor.incPointsTravered();
IOMonitor2.DCP_D_traversedPointNum++;
long t = pageReader.timeBuffer.getLong();
if (t >= targetTimestamp) {
@@ -376,18 +376,18 @@ public class ChunkSuit4CPV {
if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
while (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
estimatedPos--;
- IOMonitor.incPointsTravered();
+ // IOMonitor.incPointsTravered();
IOMonitor2.DCP_D_traversedPointNum++;
}
} else if (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
while (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
estimatedPos++;
- IOMonitor.incPointsTravered();
+ // IOMonitor.incPointsTravered();
IOMonitor2.DCP_D_traversedPointNum++;
}
if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
estimatedPos--;
- IOMonitor.incPointsTravered();
+ // IOMonitor.incPointsTravered();
IOMonitor2.DCP_D_traversedPointNum++;
} // else equal
} // else equal
@@ -430,7 +430,7 @@ public class ChunkSuit4CPV {
pageReader.valueBuffer.position(pageReader.timeBufferLength);
while (pageReader.timeBuffer.remaining() > 0) {
estimatedPos++;
- IOMonitor.incPointsTravered();
+ // IOMonitor.incPointsTravered();
IOMonitor2.DCP_D_traversedPointNum++;
long t = pageReader.timeBuffer.getLong();
if (t >= targetTimestamp) {
@@ -438,7 +438,7 @@ public class ChunkSuit4CPV {
}
}
if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
- IOMonitor.incPointsTravered();
+ // IOMonitor.incPointsTravered();
IOMonitor2.DCP_D_traversedPointNum++;
estimatedPos--;
} // else equals no need to minus 1
@@ -499,18 +499,18 @@ public class ChunkSuit4CPV {
if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
while (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
estimatedPos--;
- IOMonitor.incPointsTravered();
+ // IOMonitor.incPointsTravered();
IOMonitor2.DCP_D_traversedPointNum++;
}
} else if (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
while (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
estimatedPos++;
- IOMonitor.incPointsTravered();
+ // IOMonitor.incPointsTravered();
IOMonitor2.DCP_D_traversedPointNum++;
}
if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
estimatedPos--;
- IOMonitor.incPointsTravered();
+ // IOMonitor.incPointsTravered();
IOMonitor2.DCP_D_traversedPointNum++;
} // else equal
} // else equal
@@ -529,7 +529,7 @@ public class ChunkSuit4CPV {
while (pageReader.timeBuffer.remaining() > 0) {
if (!flag) {
estimatedPos++;
- IOMonitor.incPointsTravered();
+ // IOMonitor.incPointsTravered();
IOMonitor2.DCP_D_traversedPointNum++;
}
long t = pageReader.timeBuffer.getLong();
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor.java
index 3bfed740893..2e025641b07 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor.java
@@ -1,222 +1,223 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.tsfile.read.common;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/** @author Yuyuan Kang */
-public class IOMonitor {
-
- public static long metaIOTime;
- public static int metaIONum;
- public static long dataIOTime;
- public static int dataIONum;
- public static long readMemChunkTime;
- public static int readMemChunkNum;
-
- public static long totalTime;
- public static String sql;
-
- public static long pointsTraversed;
-
- public static List<Long> metaIOTimes = new ArrayList<>();
- public static List<Long> dataIOTimes = new ArrayList<>();
- public static List<Long> readMemChunkTimes = new ArrayList<>();
- public static List<Integer> metaIONums = new ArrayList<>();
- public static List<Integer> dataIONums = new ArrayList<>();
- public static List<Integer> readMemChunkNums = new ArrayList<>();
-
- public static List<String> sqls = new ArrayList<>();
- public static List<Long> totalTimes = new ArrayList<>();
-
- public static List<Long> pointsTraversedList = new ArrayList<>();
-
- public static boolean isSet = false;
-
- public static void incReadMemChunkTime(long v) {
- readMemChunkTime += v;
- readMemChunkNum++;
- }
-
- public static void resetReadMemChunkTime() {
- readMemChunkNums.add(readMemChunkNum);
- readMemChunkTimes.add(readMemChunkTime);
- readMemChunkTime = 0;
- readMemChunkNum = 0;
- }
-
- public static void resetPointsTraversed() {
- pointsTraversedList.add(pointsTraversed);
- pointsTraversed = 0;
- }
-
- public static void incMeta(long v) {
- metaIOTime += v;
- metaIONum++;
- }
-
- public static void incPointsTravered() {
- pointsTraversed++;
- }
-
- private static void resetMeta() {
- metaIOTimes.add(metaIOTime);
- metaIONums.add(metaIONum);
- metaIOTime = 0;
- metaIONum = 0;
- }
-
- public static void incDataIOTime(long v) {
- dataIOTime += v;
- dataIONum++;
- }
-
- public static void incTotalTime(long val) {
- totalTime += val;
- }
-
- private static void resetDataIOTime() {
- dataIOTimes.add(dataIOTime);
- dataIONums.add(dataIONum);
- dataIOTime = 0;
- dataIONum = 0;
- }
-
- public static void resetTotalTime() {
- totalTimes.add(totalTime);
- totalTime = 0;
- }
-
- public static void setSQL(String v) {
- if (!isSet) {
- clear();
- isSet = true;
- sql = v;
- } else {
- reset();
- sql = v;
- }
- }
-
- public static void reset() {
- resetMeta();
- resetDataIOTime();
- resetTotalTime();
- resetReadMemChunkTime();
- sqls.add(sql);
- sql = null;
- resetPointsTraversed();
- }
-
- // private static double getAvg(List<Long> vals) {
- // return (double) getSum(vals) / vals.size();
- // }
-
- private static long getSumLong(List<Long> vals) {
- long sum = 0;
- for (Object v : vals) {
- sum += (long) v;
- }
- return sum;
- }
-
- private static long getSumInteger(List<Integer> vals) {
- Integer sum = 0;
- for (Integer v : vals) {
- sum += v;
- }
- return sum.longValue();
- }
-
- public static void clear() {
- isSet = false;
- metaIOTime = 0L;
- dataIOTime = 0L;
- totalTime = 0L;
- readMemChunkTime = 0L;
- dataIONum = 0;
- metaIONum = 0;
- readMemChunkNum = 0;
- sql = null;
-
- metaIOTimes.clear();
- metaIONums.clear();
- dataIONums.clear();
- dataIOTimes.clear();
- sqls.clear();
- totalTimes.clear();
-
- pointsTraversedList.clear();
- pointsTraversed = 0;
- }
-
- public static void finish() {
- clear();
- isSet = false;
- }
-
- public static String print() {
- if (sql != null) {
- reset();
- }
- String ret = "";
- for (int i = 0; i < sqls.size(); i++) {
- ret +=
- sqls.get(i)
- + "\t meta IO: \t"
- + metaIOTimes.get(i)
- + "\t meta num: \t"
- + metaIONums.get(i)
- + "\t data IO: \t"
- + dataIOTimes.get(i)
- + "\t data num: \t"
- + dataIONums.get(i)
- + "\t readMemChunk IO: \t"
- + readMemChunkTimes.get(i)
- + "\t readMemChunk num: \t"
- + readMemChunkNums.get(i)
- + "\t total: \t"
- + totalTimes.get(i)
- + "\t pointsTraversed: \t"
- + pointsTraversedList.get(i)
- + "\n";
- }
- ret +=
- "sum meta IO: \t"
- + getSumLong(metaIOTimes)
- + "\t sum meta nums: \t"
- + getSumInteger(metaIONums)
- + "\t sum data IO: \t"
- + getSumLong(dataIOTimes)
- + "\t sum data num: \t"
- + getSumInteger(dataIONums)
- + "\t sum readMemChunkTime: \t"
- + getSumLong(readMemChunkTimes)
- + "\t sum readMemChunkNum: \t"
- + getSumInteger(readMemChunkNums)
- + "\t avg total time: \t"
- + getSumLong(totalTimes)
- + "\t points traversed: \t"
- + getSumLong(pointsTraversedList)
- + "\t isSet: \t"
- + isSet;
- return ret;
- }
-}
+/// *
+// * Licensed to the Apache Software Foundation (ASF) under one
+// * or more contributor license agreements. See the NOTICE file
+// * distributed with this work for additional information
+// * regarding copyright ownership. The ASF licenses this file
+// * to you under the Apache License, Version 2.0 (the
+// * "License"); you may not use this file except in compliance
+// * with the License. You may obtain a copy of the License at
+// *
+// * http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing,
+// * software distributed under the License is distributed on an
+// * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// * KIND, either express or implied. See the License for the
+// * specific language governing permissions and limitations
+// * under the License.
+// */
+// package org.apache.iotdb.tsfile.read.common;
+//
+// import java.util.ArrayList;
+// import java.util.List;
+//
+// @Deprecated
+/// ** @author Yuyuan Kang */
+// public class IOMonitor {
+//
+// public static long metaIOTime;
+// public static int metaIONum;
+// public static long dataIOTime;
+// public static int dataIONum;
+// public static long readMemChunkTime;
+// public static int readMemChunkNum;
+//
+// public static long totalTime;
+// public static String sql;
+//
+// public static long pointsTraversed;
+//
+// public static List<Long> metaIOTimes = new ArrayList<>();
+// public static List<Long> dataIOTimes = new ArrayList<>();
+// public static List<Long> readMemChunkTimes = new ArrayList<>();
+// public static List<Integer> metaIONums = new ArrayList<>();
+// public static List<Integer> dataIONums = new ArrayList<>();
+// public static List<Integer> readMemChunkNums = new ArrayList<>();
+//
+// public static List<String> sqls = new ArrayList<>();
+// public static List<Long> totalTimes = new ArrayList<>();
+//
+// public static List<Long> pointsTraversedList = new ArrayList<>();
+//
+// public static boolean isSet = false;
+//
+// public static void incReadMemChunkTime(long v) {
+// readMemChunkTime += v;
+// readMemChunkNum++;
+// }
+//
+// public static void resetReadMemChunkTime() {
+// readMemChunkNums.add(readMemChunkNum);
+// readMemChunkTimes.add(readMemChunkTime);
+// readMemChunkTime = 0;
+// readMemChunkNum = 0;
+// }
+//
+// public static void resetPointsTraversed() {
+// pointsTraversedList.add(pointsTraversed);
+// pointsTraversed = 0;
+// }
+//
+// public static void incMeta(long v) {
+// metaIOTime += v;
+// metaIONum++;
+// }
+//
+// public static void incPointsTravered() {
+// pointsTraversed++;
+// }
+//
+// private static void resetMeta() {
+// metaIOTimes.add(metaIOTime);
+// metaIONums.add(metaIONum);
+// metaIOTime = 0;
+// metaIONum = 0;
+// }
+//
+// public static void incDataIOTime(long v) {
+// dataIOTime += v;
+// dataIONum++;
+// }
+//
+// public static void incTotalTime(long val) {
+// totalTime += val;
+// }
+//
+// private static void resetDataIOTime() {
+// dataIOTimes.add(dataIOTime);
+// dataIONums.add(dataIONum);
+// dataIOTime = 0;
+// dataIONum = 0;
+// }
+//
+// public static void resetTotalTime() {
+// totalTimes.add(totalTime);
+// totalTime = 0;
+// }
+//
+// public static void setSQL(String v) {
+// if (!isSet) {
+// clear();
+// isSet = true;
+// sql = v;
+// } else {
+// reset();
+// sql = v;
+// }
+// }
+//
+// public static void reset() {
+// resetMeta();
+// resetDataIOTime();
+// resetTotalTime();
+// resetReadMemChunkTime();
+// sqls.add(sql);
+// sql = null;
+// resetPointsTraversed();
+// }
+//
+// // private static double getAvg(List<Long> vals) {
+// // return (double) getSum(vals) / vals.size();
+// // }
+//
+// private static long getSumLong(List<Long> vals) {
+// long sum = 0;
+// for (Object v : vals) {
+// sum += (long) v;
+// }
+// return sum;
+// }
+//
+// private static long getSumInteger(List<Integer> vals) {
+// Integer sum = 0;
+// for (Integer v : vals) {
+// sum += v;
+// }
+// return sum.longValue();
+// }
+//
+// public static void clear() {
+// isSet = false;
+// metaIOTime = 0L;
+// dataIOTime = 0L;
+// totalTime = 0L;
+// readMemChunkTime = 0L;
+// dataIONum = 0;
+// metaIONum = 0;
+// readMemChunkNum = 0;
+// sql = null;
+//
+// metaIOTimes.clear();
+// metaIONums.clear();
+// dataIONums.clear();
+// dataIOTimes.clear();
+// sqls.clear();
+// totalTimes.clear();
+//
+// pointsTraversedList.clear();
+// pointsTraversed = 0;
+// }
+//
+// public static void finish() {
+// clear();
+// isSet = false;
+// }
+//
+// public static String print() {
+// if (sql != null) {
+// reset();
+// }
+// String ret = "";
+// for (int i = 0; i < sqls.size(); i++) {
+// ret +=
+// sqls.get(i)
+// + "\t meta IO: \t"
+// + metaIOTimes.get(i)
+// + "\t meta num: \t"
+// + metaIONums.get(i)
+// + "\t data IO: \t"
+// + dataIOTimes.get(i)
+// + "\t data num: \t"
+// + dataIONums.get(i)
+// + "\t readMemChunk IO: \t"
+// + readMemChunkTimes.get(i)
+// + "\t readMemChunk num: \t"
+// + readMemChunkNums.get(i)
+// + "\t total: \t"
+// + totalTimes.get(i)
+// + "\t pointsTraversed: \t"
+// + pointsTraversedList.get(i)
+// + "\n";
+// }
+// ret +=
+// "sum meta IO: \t"
+// + getSumLong(metaIOTimes)
+// + "\t sum meta nums: \t"
+// + getSumInteger(metaIONums)
+// + "\t sum data IO: \t"
+// + getSumLong(dataIOTimes)
+// + "\t sum data num: \t"
+// + getSumInteger(dataIONums)
+// + "\t sum readMemChunkTime: \t"
+// + getSumLong(readMemChunkTimes)
+// + "\t sum readMemChunkNum: \t"
+// + getSumInteger(readMemChunkNums)
+// + "\t avg total time: \t"
+// + getSumLong(totalTimes)
+// + "\t points traversed: \t"
+// + getSumLong(pointsTraversedList)
+// + "\t isSet: \t"
+// + isSet;
+// return ret;
+// }
+// }
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
index cc667ab3844..a75b368c00d 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
@@ -30,7 +30,6 @@ import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.common.BatchData;
import org.apache.iotdb.tsfile.read.common.BatchDataFactory;
import org.apache.iotdb.tsfile.read.common.ChunkSuit4CPV;
-import org.apache.iotdb.tsfile.read.common.IOMonitor;
import org.apache.iotdb.tsfile.read.common.IOMonitor2;
import org.apache.iotdb.tsfile.read.common.IOMonitor2.Operation;
import org.apache.iotdb.tsfile.read.common.TimeRange;
@@ -122,13 +121,11 @@ public class PageReader implements IPageReader {
Map<Integer, List<ChunkSuit4CPV>> splitChunkList,
ChunkMetadata chunkMetadata)
throws IOException { // note: [startTime,endTime), [curStartTime,curEndTime)
- // int curIdx = (int) Math.floor((curStartTime - startTime) * 1.0 / interval); // global
- // index
+ // endTime is excluded so -1
int numberOfSpans =
(int)
Math.floor(
- (Math.min(chunkMetadata.getEndTime(), endTime - 1) // endTime is excluded so -1
- - curStartTime)
+ (Math.min(chunkMetadata.getEndTime(), endTime - 1) - curStartTime)
* 1.0
/ interval)
+ 1;
@@ -148,7 +145,7 @@ public class PageReader implements IPageReader {
LP_pos = chunkSuit4CPV.updateLPwithTheClosetPointEqualOrBefore(rightEndExcluded - 1);
}
if (FP_pos != -1 && LP_pos != -1 && FP_pos > LP_pos) {
- // means the chunk has no point in this span, do nothing
+ // means the chunk has no point in this span, do nothing.
continue;
} else { // add this chunkSuit4CPV into currentChunkList or splitChunkList
if (n == 0) {
@@ -187,7 +184,7 @@ public class PageReader implements IPageReader {
// [startPos,endPos] definitely for curStartTime interval, thanks to split4CPV
int count = 0; // update here, not in statistics
for (int pos = chunkSuit4CPV.startPos; pos <= chunkSuit4CPV.endPos; pos++) {
- IOMonitor.incPointsTravered();
+ // IOMonitor.incPointsTravered();
IOMonitor2.DCP_D_traversedPointNum++;
long timestamp = timeBuffer.getLong(pos * 8);
switch (dataType) {
@@ -242,7 +239,7 @@ public class PageReader implements IPageReader {
BatchData pageData = BatchDataFactory.createBatchData(dataType, ascending, false);
while (timeDecoder.hasNext(timeBuffer)) {
- IOMonitor.incPointsTravered();
+ // IOMonitor.incPointsTravered();
IOMonitor2.DCP_D_traversedPointNum++;
long timestamp = timeDecoder.readLong(timeBuffer);
switch (dataType) {
[iotdb] 20/46: detail dataSetType info
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit f3aefc3a74e157094916e264a4db13063c7b60ce
Author: Lei Rui <10...@qq.com>
AuthorDate: Fri Jun 16 16:45:49 2023 +0800
detail dataSetType info
---
.../src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java | 1 -
.../db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java | 5 +++++
.../main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java | 4 +++-
3 files changed, 8 insertions(+), 2 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
index a37e1ddf3a2..ffff5ac268d 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
@@ -533,7 +533,6 @@ public class PlanExecutor implements IPlanExecutor {
GroupByTimeFillPlan groupByFillPlan = (GroupByTimeFillPlan) queryPlan;
queryDataSet = queryRouter.groupByFill(groupByFillPlan, context);
} else if (queryPlan instanceof GroupByTimePlan) {
- IOMonitor2.dataSetType = DataSetType.GroupByWithoutValueFilterDataSet;
GroupByTimePlan groupByTimePlan = (GroupByTimePlan) queryPlan;
queryDataSet = queryRouter.groupBy(groupByTimePlan, context);
} else if (queryPlan instanceof QueryIndexPlan) {
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
index 3d270cfa96d..05ae0fc6de2 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
@@ -32,6 +32,8 @@ import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.factory.AggregateResultFactory;
import org.apache.iotdb.db.query.filter.TsFileFilter;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.common.IOMonitor2;
+import org.apache.iotdb.tsfile.read.common.IOMonitor2.DataSetType;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.common.RowRecord;
import org.apache.iotdb.tsfile.read.expression.IExpression;
@@ -199,10 +201,13 @@ public class GroupByWithoutValueFilterDataSet extends GroupByEngineDataSet {
throws StorageEngineException, QueryProcessException {
if (CONFIG.isEnableCPV()) {
// System.out.println("====DEBUG====: use LocalGroupByExecutor4CPV for CPV");
+ IOMonitor2.dataSetType =
+ DataSetType.GroupByWithoutValueFilterDataSet_LocalGroupByExecutor4CPV;
return new LocalGroupByExecutor4CPV(
path, allSensors, dataType, context, timeFilter, fileFilter, ascending);
} else {
// System.out.println("====DEBUG====: use LocalGroupByExecutor for MOC");
+ IOMonitor2.dataSetType = DataSetType.GroupByWithoutValueFilterDataSet_LocalGroupByExecutor;
return new LocalGroupByExecutor(
path, allSensors, dataType, context, timeFilter, fileFilter, ascending);
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
index 903aa402654..7541741b1f4 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
@@ -25,7 +25,9 @@ public class IOMonitor2 {
NONE,
RawQueryDataSetWithoutValueFilter,
UDTFAlignByTimeDataSet,
- GroupByWithoutValueFilterDataSet
+ GroupByWithoutValueFilterDataSet_LocalGroupByExecutor4CPV,
+
+ GroupByWithoutValueFilterDataSet_LocalGroupByExecutor
}
public enum Operation {
[iotdb] 28/46: add tests in SDTEncoderTest
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit eb4a6ac5cf082cd1161962de4cde84fdc4e6ce25
Author: Lei Rui <10...@qq.com>
AuthorDate: Sun Jul 2 23:09:54 2023 +0800
add tests in SDTEncoderTest
---
.../iotdb/tsfile/encoding/SDTEncoderTest.java | 240 ++++++++++++++++++++-
1 file changed, 237 insertions(+), 3 deletions(-)
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SDTEncoderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SDTEncoderTest.java
index 901bc60661c..c703ed24d04 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SDTEncoderTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SDTEncoderTest.java
@@ -19,12 +19,17 @@
package org.apache.iotdb.tsfile.encoding;
-import org.apache.iotdb.tsfile.encoding.encoder.SDTEncoder;
+import static org.junit.Assert.assertEquals;
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import org.apache.iotdb.tsfile.encoding.encoder.SDTEncoder;
+import org.apache.iotdb.tsfile.utils.Pair;
import org.junit.Test;
-import static org.junit.Assert.assertEquals;
-
public class SDTEncoderTest {
@Test
@@ -61,6 +66,235 @@ public class SDTEncoderTest {
assertEquals(14, count);
}
+ @Test
+ public void mytest1() throws Exception {
+ String csvData = "D:\\full-game\\BallSpeed.csv";
+ int start = 200000;
+ int range = 200;
+ int end = start + range;
+ SDTEncoder encoder = new SDTEncoder();
+ double e = 80000.0 / 2; // std/2
+ encoder.setCompDeviation(e / 2);
+ long count = 0;
+ String line;
+ List<Long> timestampList = new ArrayList<>();
+ List<Long> valueList = new ArrayList<>();
+ List<Long> selectTimestamps = new ArrayList<>();
+ List<Long> selectValues = new ArrayList<>();
+ long idx = 0;
+ try (BufferedReader reader = new BufferedReader(new FileReader(csvData))) {
+ while ((line = reader.readLine()) != null) {
+ count++;
+ if (count >= start && count < end) {
+ idx++;
+// long time = Long.parseLong(line.split(",")[0]);
+ long time = idx;
+ long value = Long.parseLong(line.split(",")[1]);
+ timestampList.add(time);
+ valueList.add(value);
+ if (encoder.encodeLong(time, value)) {
+ selectTimestamps.add(encoder.getTime());
+ selectValues.add(encoder.getLongValue());
+ }
+ if (count == end - 1) { // last point
+ selectTimestamps.add(time);
+ selectValues.add(value);
+ }
+ } else if (count >= end) {
+ break;
+ }
+ }
+ }
+
+ System.out.println("close all;clear all;clc;");
+ System.out.println("e=" + e);
+ System.out.println("t=" + timestampList + ";");
+ System.out.println("v=" + valueList + ";");
+ System.out.println("at=" + selectTimestamps + ";");
+ System.out.println("av=" + selectValues + ";");
+
+ System.out.println("LB=av-e;\n"
+ + "UB=av+e;\n"
+ + "plot(t,v)\n"
+ + "hold on,plot(t,v,'+')\n"
+ + "hold on,plot(at,av,'r')\n"
+ + "%hold on,plot(at,av,'r+')\n"
+ + "hold on,plot(at,UB,'g')\n"
+ + "%hold on,plot(at,UB,'g+')\n"
+ + "hold on,plot(at,LB,'g')\n"
+ + "%hold on,plot(at,LB,'g+')\n"
+ + "\n"
+ + "% [val,idx]=max(av-e)\n"
+ + "% yline(val)\n"
+ + "\n"
+ + "[sortedX, sortedInds] = sort(LB,'descend');\n"
+ + "% yline(sortedX(1))\n"
+ + "% yline(sortedX(2))\n"
+ + "\n"
+ + "% 依次计算UB小于Nth-max-LB的点数\n"
+ + "% case 1: 两端都小于threshold，则该段内的点的UB都小于threshold，都可以被剪枝\n"
+ + "% case 2: 两端都大于等于threshold，则该段内的点的UB都大于等于threshold，则都不可以被剪枝\n"
+ + "% case 3: 一端小于一端大于等于threshold，则该段内有部分点的UB小于threshold可以被剪枝\n"
+ + "% (t1,v1), (t2,v2), t1<t2\n"
+ + "% y=(t-t1)*(v2-v1)/(t2-t1)+v1\n"
+ + "% y<thresold ===> (t-t1)*(v2-v1)/(t2-t1)+v1<threshold\n"
+ + "% 如果v2>=threshold>v1: 则t<(threshold-v1)*(t2-t1)/(v2-v1)+t1，于是[t1,(threshold-v1)*(t2-t1)/(v2-v1)+t1)内的点可剪枝\n"
+ + "% 如果v2<threshold<=v1: 则t>(threshold-v1)*(t2-t1)/(v2-v1)+t1，于是((threshold-v1)*(t2-t1)/(v2-v1)+t1,t2]内的点可剪枝\n"
+ + "rank=5\n"
+ + "threshold=sortedX(rank)\n"
+ + "hold on, yline(threshold);\n"
+ + "hold on, plot(at(sortedInds(rank)),threshold,'o')\n"
+ + "\n"
+ + "prune_t=[]; % point\n"
+ + "prune_v=[]; % point\n"
+ + "prune_interval=[]; % interval\n"
+ + "interval_start=-1;\n"
+ + "for i=2:1:length(av)\n"
+ + "\tt1=at(i-1);\n"
+ + "\tt2=at(i);\n"
+ + "\tv1=UB(i-1);\n"
+ + "\tv2=UB(i);\n"
+ + "\ta=[];\n"
+ + "\tif v1<threshold && v2<threshold\n"
+ + "\t\tif i<length(av)\n"
+ + "\t\t\ta=t1:1:t2-1;\n"
+ + "\t\telse \n"
+ + "\t\t\ta=t1:1:t2;\n"
+ + "\t\tend\n"
+ + "\t\tif interval_start<0\n"
+ + "\t\t\tinterval_start=t1;\n"
+ + " end\n"
+ + " interval_end=t2; % continuous\n"
+ + "\telseif v1<threshold && v2>=threshold\n"
+ + "\t\ta=t1:1:floor((threshold-v1)*(t2-t1)/(v2-v1)+t1); % no need -1 here\n"
+ + "\t\tif interval_start<0\n"
+ + "\t\t\tinterval_start=t1;\n"
+ + " end\n"
+ + " prune_interval=[prune_interval;[interval_start,floor((threshold-v1)*(t2-t1)/(v2-v1)+t1)]];\n"
+ + "\t\tinterval_start=-1; % discontinuous\n"
+ + "\telseif v1>=threshold && v2<threshold\n"
+ + "\t\tif i<length(av)\n"
+ + "\t\t\ta=ceil((threshold-v1)*(t2-t1)/(v2-v1)+t1):1:t2-1;\n"
+ + "\t\telse\n"
+ + "\t\t\ta=ceil((threshold-v1)*(t2-t1)/(v2-v1)+t1):1:t2;\n"
+ + "\t\tend\n"
+ + "\t\tinterval_start=ceil((threshold-v1)*(t2-t1)/(v2-v1)+t1);\n"
+ + "\t\tinterval_end=t2;\n"
+ + "\tend\n"
+ + "\tprune_t=[prune_t,a];\n"
+ + "\tprune_v=[prune_v,(a-t1)*(v2-v1)/(t2-t1)+v1];\n"
+ + "end\n"
+ + "if interval_start>0\n"
+ + "\tprune_interval=[prune_interval;[interval_start,interval_end]];\n"
+ + "end\n"
+ + "\n"
+ + "hold on,plot(prune_t,prune_v,'b.')\n"
+ + "disp(length(prune_t)/length(t))\n"
+ + "\n"
+ + "%rectangle('Position', [at(1) av(1) at(2) av(2)], 'Facec',[0.5 1 0.5])\n"
+ + "prune_interval\n"
+ + "\n"
+ + "%for i=1:1:length(prune_interval)\n"
+ + "%\txline(prune_interval(i,1));\n"
+ + "%\txline(prune_interval(i,2));\n"
+ + "%end\n"
+ + "\n"
+ + "% https://se.mathworks.com/help/matlab/ref/xregion.html\n"
+ + "% xregion\n"
+ + "\n"
+ + "for i=1:1:length(prune_interval)\n"
+ + "\tx=prune_interval(i,1):1:prune_interval(i,2);\n"
+ + "\ty=[];\n"
+ + "\tfor j=1:1:length(x)\n"
+ + "\t\tidx=x(j);\n"
+ + "\t\tfor n=2:1:length(at)\n"
+ + "\t\t\tif idx<=at(n)\n"
+ + "\t\t\t\tt1=at(n-1);\n"
+ + "\t\t\t\tt2=at(n);\n"
+ + "\t\t\t\tv1=av(n-1);\n"
+ + "\t\t\t\tv2=av(n);\n"
+ + "\t\t\t\ty=[y,(idx-t1)*(v2-v1)/(t2-t1)+v1];\n"
+ + "\t\t\t\tbreak;\n"
+ + "\t\t\tend\n"
+ + "\t\tend\n"
+ + "\tend\n"
+ + "\ty1=y-e;\n"
+ + "\ty2=y+e;\n"
+ + "\tL1=[x,fliplr(x)];\n"
+ + "\tL2=[y1,fliplr(y2)];\n"
+ + "\tfill(L1,L2,'b','FaceAlpha',0.1);\n"
+ + "end");
+ }
+
+ @Test
+ public void mytest2() throws Exception {
+ String csvData = "D:\\full-game\\BallSpeed.csv";
+ int start = 200000;
+ int range = 200;
+ int end = start + range;
+ SDTEncoder encoder = new SDTEncoder();
+ double e = 80000.0 / 2; // std/2
+ encoder.setCompDeviation(e / 2);
+ long count = 0;
+ String line;
+ List<Integer> timestampList = new ArrayList<>(); // idx
+ List<Long> valueList = new ArrayList<>();
+ List<Integer> selectTimestamps = new ArrayList<>();
+ List<Long> selectValues = new ArrayList<>();
+ int idx = 0;
+ try (BufferedReader reader = new BufferedReader(new FileReader(csvData))) {
+ while ((line = reader.readLine()) != null) {
+ count++;
+ if (count >= start && count < end) {
+ idx++;
+// long time = Long.parseLong(line.split(",")[0]);
+ int time = idx;
+ long value = Long.parseLong(line.split(",")[1]);
+ timestampList.add(time);
+ valueList.add(value);
+ if (encoder.encodeLong(time, value)) {
+ selectTimestamps.add((int) encoder.getTime());
+ selectValues.add(encoder.getLongValue());
+ }
+ if (count == end - 1) { // last point
+ selectTimestamps.add(time);
+ selectValues.add(value);
+ }
+ } else if (count >= end) {
+ break;
+ }
+ }
+ }
+
+ System.out.println("close all;clear all;clc;");
+ System.out.println("e=" + e);
+ System.out.println("t=" + timestampList + ";");
+ System.out.println("v=" + valueList + ";");
+ System.out.println("at=" + selectTimestamps + ";");
+ System.out.println("av=" + selectValues + ";");
+
+ // 计算maxLB
+ long maxVal = Collections.max(selectValues);
+ int maxIdx = selectTimestamps.get(selectValues.indexOf(maxVal));
+ System.out.println(maxIdx);
+ System.out.println(maxVal);
+ double maxLB = maxVal - e;
+
+ // 计算UB<maxLB的剪枝段
+ List<Pair<Integer, Integer>> prune_intervals = new ArrayList<>();
+ int interval_start=-1;
+ for (int i = 1; i < selectTimestamps.size(); i++) {
+ int t1=selectTimestamps.get(i-1);
+ int t2=selectTimestamps.get(i);
+ long v1=selectValues.get(i-1);
+ long v2=selectValues.get(i);
+
+ }
+
+ // 对剪枝段取余后搜索遍历计算TP
+
+ }
+
@Test
public void testLongSingleValue() {
SDTEncoder encoder = new SDTEncoder();
[iotdb] 46/46: update readme
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 359389a85e84ddf81117719ff9827ce7a920a2ac
Author: Lei Rui <10...@qq.com>
AuthorDate: Mon Jul 10 01:09:29 2023 +0800
update readme
---
README.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/README.md b/README.md
index 35a0dea4f4b..4462ef32d10 100644
--- a/README.md
+++ b/README.md
@@ -24,5 +24,5 @@
- M4-UDF: M4-UDF is implemented in [server/src/main/java/org/apache/iotdb/db/query/udf/builtin/UDTFM4MAC.java](server/src/main/java/org/apache/iotdb/db/query/udf/builtin/UDTFM4MAC.java). The document of the M4 function is available on the product [website](https://iotdb.apache.org/UserGuide/Master/Operators-Functions/Sample.html#m4-function) of Apache IoTDB.
- M4-LSM: The candidate generation and verification framework is implemented in [server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java](server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java). The step regression index is implemented in [tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/StepRegress.java](tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/StepRegress.java).
- Some integration tests for correctness are in [server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest1.java](server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest1.java).
-- The codes, data and scripts for experiments are in [another GitHub repository](https://github.com/LeiRui/M4-visualization-exp.git) for reproducibility.
+- The codes, data and scripts for experiments are in [another GitHub repository](https://anonymous.4open.science/r/M4-visualization-exp-D659) for reproducibility.
- For the README of [Apache IoTDB](https://iotdb.apache.org/) itself, please see [README_IOTDB.md](README_IOTDB.md). To build this repository, run `mvn clean package -DskipTests -pl -distribution`.
[iotdb] 05/46: modify python-iotdb-connector add execute_finish
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit b634e42dd1d02963329fb9ec3923f3235dd74acc
Author: Lei Rui <10...@qq.com>
AuthorDate: Mon Jun 12 14:32:35 2023 +0800
modify python-iotdb-connector add execute_finish
---
client-py/iotdb/Session.py | 7 +++++++
client-py/iotdb/utils/IoTDBConstants.py | 3 +++
2 files changed, 10 insertions(+)
diff --git a/client-py/iotdb/Session.py b/client-py/iotdb/Session.py
index 5a44312a5f9..7e861ffb247 100644
--- a/client-py/iotdb/Session.py
+++ b/client-py/iotdb/Session.py
@@ -168,6 +168,13 @@ class Session(object):
return Session.verify_success(status)
+ def execute_finish(self):
+ resp = self.__client.executeFinish()
+ logger.debug(
+ "execute_finish message: {}".format(resp.executionInfo)
+ )
+ return resp.executionInfo
+
def delete_storage_group(self, storage_group):
"""
delete one storage group.
diff --git a/client-py/iotdb/utils/IoTDBConstants.py b/client-py/iotdb/utils/IoTDBConstants.py
index eb2d89c5317..9a1a1564668 100644
--- a/client-py/iotdb/utils/IoTDBConstants.py
+++ b/client-py/iotdb/utils/IoTDBConstants.py
@@ -33,6 +33,9 @@ class TSDataType(Enum):
def __eq__(self, other) -> bool:
return self.value == other.value
+ def __hash__(self):
+ return self.value
+
@unique
class TSEncoding(Enum):
[iotdb] 41/46: debu
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 960357c01edac2beeacfdef10cc3c2c3a659a03d
Author: Lei Rui <10...@qq.com>
AuthorDate: Fri Jul 7 14:07:40 2023 +0800
debu
---
.../org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
index 866ae9e513c..57250041c2e 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
@@ -374,6 +374,17 @@ public class ChunkSuit4CPV {
// search from estimatePos in the timeBuffer to find the closet timestamp equal to or smaller
// than the given timestamp
+ // TODO debug
+ try {
+ long tmp = pageReader.timeBuffer.getLong(estimatedPos * 8);
+ } catch (IndexOutOfBoundsException e) {
+ System.out.println("targetTimestamp=" + targetTimestamp);
+ System.out.println("estimatedPos=" + estimatedPos);
+ System.out.println("count=" + chunkMetadata.getStatistics().getCount());
+ System.out.println("stepregress segmentKeys=" + stepRegress.getSegmentKeys());
+ System.out.println("stepregress slope=" + stepRegress.getSlope());
+ }
+
if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
while (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
estimatedPos--;
[iotdb] 43/46: fix
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 8d2f71561a0d572ecdc2240e9565a71d058978f8
Author: Lei Rui <10...@qq.com>
AuthorDate: Fri Jul 7 14:51:34 2023 +0800
fix
---
.../main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java | 2 ++
1 file changed, 2 insertions(+)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
index daa37817ca8..b5791f0ed4e 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
@@ -386,6 +386,8 @@ public class ChunkSuit4CPV {
M4_CHUNK_METADATA.info("estimatedPos=" + estimatedPos);
M4_CHUNK_METADATA.info("count=" + chunkMetadata.getStatistics().getCount());
M4_CHUNK_METADATA.info("stepregress segmentKeys=" + stepRegress.getSegmentKeys());
+ M4_CHUNK_METADATA.info(
+ "stepregress segmentIntercepts=" + stepRegress.getSegmentIntercepts());
M4_CHUNK_METADATA.info("stepregress slope=" + stepRegress.getSlope());
throw e;
}
[iotdb] 03/46: add IOMonitor2 and add metrics for raw data query
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 052fd37301e8eebfd938916817f661d08fa8bbd7
Author: Lei Rui <10...@qq.com>
AuthorDate: Mon Jun 12 11:57:12 2023 +0800
add IOMonitor2 and add metrics for raw data query
---
.../apache/iotdb/db/engine/cache/ChunkCache.java | 6 +
.../org/apache/iotdb/db/service/TSServiceImpl.java | 42 ++--
.../org/apache/iotdb/db/utils/FileLoaderUtils.java | 3 +
.../iotdb/tsfile/read/common/IOMonitor2.java | 227 +++++++++++++++++++++
.../tsfile/read/reader/chunk/ChunkReader.java | 5 +
.../iotdb/tsfile/read/reader/page/PageReader.java | 7 +-
6 files changed, 272 insertions(+), 18 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
index 0e96e2a4ccd..204a99e76ef 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
@@ -27,6 +27,8 @@ import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Chunk;
import org.apache.iotdb.tsfile.read.common.IOMonitor;
+import org.apache.iotdb.tsfile.read.common.IOMonitor2;
+import org.apache.iotdb.tsfile.read.common.IOMonitor2.Operation;
import org.apache.iotdb.tsfile.utils.RamUsageEstimator;
import com.github.benmanes.caffeine.cache.Caffeine;
@@ -80,6 +82,8 @@ public class ChunkCache {
.get(chunkMetadata.getFilePath(), chunkMetadata.isClosed());
long start = System.nanoTime();
Chunk ret = reader.readMemChunk(chunkMetadata);
+ IOMonitor2.addMeasure(
+ Operation.DCP_B_READ_MEM_CHUNK, System.nanoTime() - start);
IOMonitor.incReadMemChunkTime(System.nanoTime() - start);
return ret;
} catch (IOException e) {
@@ -102,7 +106,9 @@ public class ChunkCache {
TsFileSequenceReader reader =
FileReaderManager.getInstance()
.get(chunkMetaData.getFilePath(), chunkMetaData.isClosed());
+ long start = System.nanoTime();
Chunk chunk = reader.readMemChunk(chunkMetaData);
+ IOMonitor2.addMeasure(Operation.DCP_B_READ_MEM_CHUNK, System.nanoTime() - start);
return new Chunk(
chunk.getHeader(),
chunk.getData().duplicate(),
diff --git a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
index 00c01bb37da..4c396572e80 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
@@ -128,6 +128,7 @@ import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.read.common.IOMonitor;
+import org.apache.iotdb.tsfile.read.common.IOMonitor2;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet;
@@ -628,6 +629,7 @@ public class TSServiceImpl implements TSIService.Iface {
@Override
public TSExecuteStatementResp executeQueryStatement(TSExecuteStatementReq req) {
+ long start = System.nanoTime();
try {
if (!checkLogin(req.getSessionId())) {
return RpcUtils.getTSExecuteStatementResp(TSStatusCode.NOT_LOGIN_ERROR);
@@ -638,17 +640,21 @@ public class TSServiceImpl implements TSIService.Iface {
processor.parseSQLToPhysicalPlan(
statement, sessionManager.getZoneId(req.sessionId), req.fetchSize);
// System.out.println("====DEBUG====: fetchSize=" + req.fetchSize);
- return physicalPlan.isQuery()
- ? internalExecuteQueryStatement(
- statement,
- req.statementId,
- physicalPlan,
- req.fetchSize,
- req.timeout,
- sessionManager.getUsername(req.getSessionId()),
- req.isEnableRedirectQuery())
- : RpcUtils.getTSExecuteStatementResp(
- TSStatusCode.EXECUTE_STATEMENT_ERROR, "Statement is not a query statement.");
+ TSExecuteStatementResp resp =
+ physicalPlan.isQuery()
+ ? internalExecuteQueryStatement(
+ statement,
+ req.statementId,
+ physicalPlan,
+ req.fetchSize,
+ req.timeout,
+ sessionManager.getUsername(req.getSessionId()),
+ req.isEnableRedirectQuery())
+ : RpcUtils.getTSExecuteStatementResp(
+ TSStatusCode.EXECUTE_STATEMENT_ERROR, "Statement is not a query statement.");
+ IOMonitor2.addMeasure(
+ IOMonitor2.Operation.DCP_Server_Query_Execute, System.nanoTime() - start);
+ return resp;
} catch (InterruptedException e) {
LOGGER.error(INFO_INTERRUPT_ERROR, req, e);
Thread.currentThread().interrupt();
@@ -664,7 +670,7 @@ public class TSServiceImpl implements TSIService.Iface {
public TSExecuteFinishResp executeFinish() throws TException {
TSExecuteFinishResp ret = new TSExecuteFinishResp();
ret.setStatus(RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS));
- ret.setExecutionInfo(IOMonitor.print());
+ ret.setExecutionInfo(IOMonitor.print() + "\n" + IOMonitor2.print());
IOMonitor.finish();
return ret;
}
@@ -719,8 +725,8 @@ public class TSServiceImpl implements TSIService.Iface {
// start record execution time
IOMonitor.setSQL(statement);
- queryCount.incrementAndGet();
long start = System.nanoTime();
+ queryCount.incrementAndGet();
AUDIT_LOGGER.debug(
"Session {} execute Query: {}", sessionManager.getCurrSessionId(), statement);
long startTime = System.currentTimeMillis();
@@ -1017,6 +1023,7 @@ public class TSServiceImpl implements TSIService.Iface {
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
@Override
public TSFetchResultsResp fetchResults(TSFetchResultsReq req) {
+ long start = System.nanoTime();
try {
if (!checkLogin(req.getSessionId())) {
return RpcUtils.getTSFetchResultsResp(TSStatusCode.NOT_LOGIN_ERROR);
@@ -1032,6 +1039,7 @@ public class TSServiceImpl implements TSIService.Iface {
req.queryId, System.currentTimeMillis(), req.statement, req.timeout);
QueryDataSet queryDataSet = sessionManager.getDataset(req.queryId);
+ TSFetchResultsResp resp;
if (req.isAlign) {
TSQueryDataSet result =
fillRpcReturnData(
@@ -1040,13 +1048,12 @@ public class TSServiceImpl implements TSIService.Iface {
if (!hasResultSet) {
releaseQueryResourceNoExceptions(req.queryId);
}
- TSFetchResultsResp resp = RpcUtils.getTSFetchResultsResp(TSStatusCode.SUCCESS_STATUS);
+ resp = RpcUtils.getTSFetchResultsResp(TSStatusCode.SUCCESS_STATUS);
resp.setHasResultSet(hasResultSet);
resp.setQueryDataSet(result);
resp.setIsAlign(true);
queryTimeManager.unRegisterQuery(req.queryId);
- return resp;
} else {
TSQueryNonAlignDataSet nonAlignResult =
fillRpcNonAlignReturnData(
@@ -1061,14 +1068,15 @@ public class TSServiceImpl implements TSIService.Iface {
if (!hasResultSet) {
sessionManager.removeDataset(req.queryId);
}
- TSFetchResultsResp resp = RpcUtils.getTSFetchResultsResp(TSStatusCode.SUCCESS_STATUS);
+ resp = RpcUtils.getTSFetchResultsResp(TSStatusCode.SUCCESS_STATUS);
resp.setHasResultSet(hasResultSet);
resp.setNonAlignQueryDataSet(nonAlignResult);
resp.setIsAlign(false);
queryTimeManager.unRegisterQuery(req.queryId);
- return resp;
}
+ IOMonitor2.addMeasure(IOMonitor2.Operation.DCP_Server_Query_Fetch, System.nanoTime() - start);
+ return resp;
} catch (InterruptedException e) {
LOGGER.error(INFO_INTERRUPT_ERROR, req, e);
Thread.currentThread().interrupt();
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
index fd7b6f4cb1d..59c2201251b 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
@@ -34,6 +34,8 @@ import org.apache.iotdb.tsfile.file.metadata.TsFileMetadata;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Chunk;
import org.apache.iotdb.tsfile.read.common.IOMonitor;
+import org.apache.iotdb.tsfile.read.common.IOMonitor2;
+import org.apache.iotdb.tsfile.read.common.IOMonitor2.Operation;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.controller.IChunkLoader;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
@@ -144,6 +146,7 @@ public class FileLoaderUtils {
}
long duration = System.nanoTime() - start;
IOMonitor.incMeta(duration);
+ IOMonitor2.addMeasure(Operation.DCP_A_GET_CHUNK_METADATAS, System.nanoTime() - start);
return timeSeriesMetadata;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
new file mode 100644
index 00000000000..1682eaa6494
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
@@ -0,0 +1,227 @@
+package org.apache.iotdb.tsfile.read.common;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class IOMonitor2 {
+
+ public enum Operation {
+ DCP_Server_Query_Execute("DCP_Server_Query_Execute"),
+ DCP_Server_Query_Fetch("DCP_Server_Query_Fetch"),
+ DCP_SeriesScanOperator_hasNext("DCP_SeriesScanOperator_hasNext"),
+ DCP_A_GET_CHUNK_METADATAS("DCP_A_GET_CHUNK_METADATAS"),
+ DCP_B_READ_MEM_CHUNK("DCP_B_READ_MEM_CHUNK"),
+ DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA(
+ "DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA"),
+ DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS("DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS");
+
+ public String getName() {
+ return name;
+ }
+
+ String name;
+
+ Operation(String name) {
+ this.name = name;
+ }
+ }
+
+ /**
+ * [Raw data query metrics] - server端execute&fetch的次数和耗时 -
+ * SeriesScanOperator.hasNext的次数和耗时（问题：root.hasNext执行次数等于decode
+ * page次数，但是SeriesScanOperator.hasNext次数大于那个次数？破案了：是root.isFinished->SeriesScanOperator.isFinished里面调用了hasNext，和root.hasNext里调用SeriesScanOperator.hasNext的次数加起来就对了。）
+ * - A：从磁盘解读chunk元数据的次数和耗时 - B：从磁盘加载chunk数据的次数和耗时 - C：解压缩所有page构造pageReaderList的次数和耗时 -
+ * D：解码page并遍历点的次数和耗时 （遍历点数不用统计，因为一定是全量点）
+ */
+ public static int DCP_Server_Query_Execute_count = 0;
+
+ public static long DCP_Server_Query_Execute_ns = 0;
+
+ public static int DCP_Server_Query_Fetch_count = 0;
+ public static long DCP_Server_Query_Fetch_ns = 0;
+
+ public static int DCP_SeriesScanOperator_hasNext_count = 0;
+ public static long DCP_SeriesScanOperator_hasNext_ns = 0;
+
+ public static int DCP_A_GET_CHUNK_METADATAS_count = 0;
+ public static long DCP_A_GET_CHUNK_METADATAS_ns = 0;
+
+ public static int DCP_B_READ_MEM_CHUNK_count = 0;
+ public static long DCP_B_READ_MEM_CHUNK_ns = 0;
+
+ public static int DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_count = 0;
+ public static long DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_ns = 0;
+
+ public static int DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_count = 0;
+ public static long DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_ns = 0;
+
+ public static long DCP_D_traversedPointNum = 0;
+
+ private static final Logger DEBUG_LOGGER = LoggerFactory.getLogger("IOMonitor2");
+
+ private static void reset() {
+ DCP_Server_Query_Execute_count = 0;
+ DCP_Server_Query_Execute_ns = 0;
+
+ DCP_Server_Query_Fetch_count = 0;
+ DCP_Server_Query_Fetch_ns = 0;
+
+ DCP_SeriesScanOperator_hasNext_count = 0;
+ DCP_SeriesScanOperator_hasNext_ns = 0;
+
+ DCP_A_GET_CHUNK_METADATAS_count = 0;
+ DCP_A_GET_CHUNK_METADATAS_ns = 0;
+
+ DCP_B_READ_MEM_CHUNK_count = 0;
+ DCP_B_READ_MEM_CHUNK_ns = 0;
+
+ DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_count = 0;
+ DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_ns = 0;
+
+ DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_count = 0;
+ DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_ns = 0;
+
+ DCP_D_traversedPointNum = 0;
+ }
+
+ public static void addMeasure(Operation operation, long elapsedTimeInNanosecond) {
+ // TODO tmp for debug
+ // DEBUG_LOGGER.info(operation.getName() + ": " + elapsedTimeInNanosecond + " ns");
+ switch (operation) {
+ case DCP_Server_Query_Execute:
+ DCP_Server_Query_Execute_count++;
+ DCP_Server_Query_Execute_ns += elapsedTimeInNanosecond;
+ break;
+ case DCP_Server_Query_Fetch:
+ DCP_Server_Query_Fetch_count++;
+ DCP_Server_Query_Fetch_ns += elapsedTimeInNanosecond;
+ break;
+ case DCP_SeriesScanOperator_hasNext:
+ DCP_SeriesScanOperator_hasNext_count++;
+ DCP_SeriesScanOperator_hasNext_ns += elapsedTimeInNanosecond;
+ break;
+ case DCP_A_GET_CHUNK_METADATAS:
+ DCP_A_GET_CHUNK_METADATAS_count++;
+ DCP_A_GET_CHUNK_METADATAS_ns += elapsedTimeInNanosecond;
+ break;
+ case DCP_B_READ_MEM_CHUNK:
+ DCP_B_READ_MEM_CHUNK_count++;
+ DCP_B_READ_MEM_CHUNK_ns += elapsedTimeInNanosecond;
+ break;
+ case DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA:
+ DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_count++;
+ DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_ns += elapsedTimeInNanosecond;
+ break;
+ case DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS:
+ DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_count++;
+ DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_ns += elapsedTimeInNanosecond;
+ break;
+ default:
+ System.out.println("not supported operation type"); // this will not happen
+ break;
+ }
+ }
+
+ public static void addMeasure(Operation operation, long elapsedTimeInNanosecond, int count) {
+ switch (operation) {
+ case DCP_Server_Query_Execute:
+ DCP_Server_Query_Execute_count += count;
+ DCP_Server_Query_Execute_ns += elapsedTimeInNanosecond;
+ break;
+ case DCP_Server_Query_Fetch:
+ DCP_Server_Query_Fetch_count += count;
+ DCP_Server_Query_Fetch_ns += elapsedTimeInNanosecond;
+ break;
+ case DCP_SeriesScanOperator_hasNext:
+ DCP_SeriesScanOperator_hasNext_count += count;
+ DCP_SeriesScanOperator_hasNext_ns += elapsedTimeInNanosecond;
+ break;
+ case DCP_A_GET_CHUNK_METADATAS:
+ DCP_A_GET_CHUNK_METADATAS_count += count;
+ DCP_A_GET_CHUNK_METADATAS_ns += elapsedTimeInNanosecond;
+ break;
+ case DCP_B_READ_MEM_CHUNK:
+ DCP_B_READ_MEM_CHUNK_count += count;
+ DCP_B_READ_MEM_CHUNK_ns += elapsedTimeInNanosecond;
+ break;
+ case DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA:
+ DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_count += count;
+ DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_ns += elapsedTimeInNanosecond;
+ break;
+ case DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS:
+ DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_count += count;
+ DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_ns += elapsedTimeInNanosecond;
+ break;
+ default:
+ System.out.println("not supported operation type"); // this will not happen
+ break;
+ }
+ }
+
+ public static String print() {
+ StringBuilder stringBuilder = new StringBuilder();
+ stringBuilder
+ .append("Server_Query_Execute_ns")
+ .append(",")
+ .append(DCP_Server_Query_Execute_ns)
+ .append("\n");
+ stringBuilder
+ .append("Server_Query_Fetch_ns")
+ .append(",")
+ .append(DCP_Server_Query_Fetch_ns)
+ .append("\n");
+ stringBuilder
+ .append("SeriesScanOperator_hasNext_ns")
+ .append(",")
+ .append(DCP_SeriesScanOperator_hasNext_ns)
+ .append("\n");
+ stringBuilder.append("A_ns").append(",").append(DCP_A_GET_CHUNK_METADATAS_ns).append("\n");
+ stringBuilder.append("B_ns").append(",").append(DCP_B_READ_MEM_CHUNK_ns).append("\n");
+ stringBuilder
+ .append("C_ns")
+ .append(",")
+ .append(DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_ns)
+ .append("\n");
+ stringBuilder
+ .append("D_ns")
+ .append(",")
+ .append(DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_ns)
+ .append("\n");
+
+ stringBuilder
+ .append("Server_Query_Execute_cnt")
+ .append(",")
+ .append(DCP_Server_Query_Execute_count)
+ .append("\n");
+ stringBuilder
+ .append("Server_Query_Fetch_cnt")
+ .append(",")
+ .append(DCP_Server_Query_Fetch_count)
+ .append("\n");
+ stringBuilder
+ .append("SeriesScanOperator_hasNext_cnt")
+ .append(",")
+ .append(DCP_SeriesScanOperator_hasNext_count)
+ .append("\n");
+ stringBuilder.append("A_cnt").append(",").append(DCP_A_GET_CHUNK_METADATAS_count).append("\n");
+ stringBuilder.append("B_cnt").append(",").append(DCP_B_READ_MEM_CHUNK_count).append("\n");
+ stringBuilder
+ .append("C_cnt")
+ .append(",")
+ .append(DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_count)
+ .append("\n");
+ stringBuilder
+ .append("D_cnt")
+ .append(",")
+ .append(DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_count)
+ .append("\n");
+ stringBuilder
+ .append("DCP_D_traversedPointNum")
+ .append(",")
+ .append(DCP_D_traversedPointNum)
+ .append("\n");
+
+ reset(); // whenever print() is called, reset the metrics, to clean warm up information.
+ return stringBuilder.toString();
+ }
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReader.java
index c794f87f78d..7e5011e1b37 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReader.java
@@ -31,6 +31,8 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.common.BatchData;
import org.apache.iotdb.tsfile.read.common.Chunk;
+import org.apache.iotdb.tsfile.read.common.IOMonitor2;
+import org.apache.iotdb.tsfile.read.common.IOMonitor2.Operation;
import org.apache.iotdb.tsfile.read.common.TimeRange;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.read.reader.IChunkReader;
@@ -81,6 +83,7 @@ public class ChunkReader implements IChunkReader {
}
private void initAllPageReaders(Statistics chunkStatistic) throws IOException {
+ long start = System.nanoTime();
// construct next satisfied page header
while (chunkDataBuffer.remaining() > 0) {
// deserialize a PageHeader from chunkDataBuffer
@@ -97,6 +100,8 @@ public class ChunkReader implements IChunkReader {
skipBytesInStreamByLength(pageHeader.getCompressedSize());
}
}
+ IOMonitor2.addMeasure(
+ Operation.DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA, System.nanoTime() - start);
}
/** judge if has next page whose page header satisfies the filter. */
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
index 68c7d4e6dd6..420a32735ac 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
@@ -31,6 +31,8 @@ import org.apache.iotdb.tsfile.read.common.BatchData;
import org.apache.iotdb.tsfile.read.common.BatchDataFactory;
import org.apache.iotdb.tsfile.read.common.ChunkSuit4CPV;
import org.apache.iotdb.tsfile.read.common.IOMonitor;
+import org.apache.iotdb.tsfile.read.common.IOMonitor2;
+import org.apache.iotdb.tsfile.read.common.IOMonitor2.Operation;
import org.apache.iotdb.tsfile.read.common.TimeRange;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.read.filter.operator.AndFilter;
@@ -233,11 +235,12 @@ public class PageReader implements IPageReader {
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
@Override
public BatchData getAllSatisfiedPageData(boolean ascending) throws IOException {
-
+ long start = System.nanoTime();
BatchData pageData = BatchDataFactory.createBatchData(dataType, ascending, false);
while (timeDecoder.hasNext(timeBuffer)) {
IOMonitor.incPointsTravered();
+ IOMonitor2.DCP_D_traversedPointNum++;
long timestamp = timeDecoder.readLong(timeBuffer);
switch (dataType) {
case BOOLEAN:
@@ -280,6 +283,8 @@ public class PageReader implements IPageReader {
throw new UnSupportedDataTypeException(String.valueOf(dataType));
}
}
+ IOMonitor2.addMeasure(
+ Operation.DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS, System.nanoTime() - start);
return pageData.flip();
}
[iotdb] 30/46: add test
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 19cc917b5b8627b20ad98c4eefb786cafdc68985
Author: Lei Rui <10...@qq.com>
AuthorDate: Mon Jul 3 23:44:12 2023 +0800
add test
---
.../iotdb/tsfile/read/common/IOMonitor2.java | 16 ++
.../iotdb/tsfile/encoding/SDTEncoderTest.java | 254 +++++++++++++++------
2 files changed, 199 insertions(+), 71 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
index 62e79e40fac..cd39d9af1e0 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
@@ -247,6 +247,22 @@ public class IOMonitor2 {
M4_LSM_TP_SEARCH_ARRAY_c_genBPTP_cnt = 0;
}
+ public static class ValuePoint implements Comparable<ValuePoint> {
+ public final int index;
+ public final long value;
+
+ public ValuePoint(int index, long value) {
+ this.index = index;
+ this.value = value;
+ }
+
+ @Override
+ public int compareTo(ValuePoint other) {
+ // multiplied to -1 as the author need descending sort order
+ return -1 * Long.valueOf(this.value).compareTo(other.value);
+ }
+ }
+
public static void addMeasure(Operation operation, long elapsedTimeInNanosecond) {
switch (operation) {
case DCP_Server_Query_Execute:
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SDTEncoderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SDTEncoderTest.java
index 9f62a16a794..c33ee296425 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SDTEncoderTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SDTEncoderTest.java
@@ -19,15 +19,18 @@
package org.apache.iotdb.tsfile.encoding;
-import static org.junit.Assert.assertEquals;
+import org.apache.iotdb.tsfile.encoding.encoder.SDTEncoder;
+import org.apache.iotdb.tsfile.read.common.IOMonitor2.ValuePoint;
+
+import org.junit.Test;
import java.io.BufferedReader;
import java.io.FileReader;
import java.util.ArrayList;
-import java.util.Collections;
+import java.util.Comparator;
import java.util.List;
-import org.apache.iotdb.tsfile.encoding.encoder.SDTEncoder;
-import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
public class SDTEncoderTest {
@@ -69,10 +72,10 @@ public class SDTEncoderTest {
public void mytest1() throws Exception {
String csvData = "D:\\full-game\\BallSpeed.csv";
int start = 200000;
- int range = 200;
+ int range = 100000;
int end = start + range;
SDTEncoder encoder = new SDTEncoder();
- double e = 80000.0 / 2; // std/2
+ double e = 100000; // std/2
encoder.setCompDeviation(e / 2);
long count = 0;
String line;
@@ -86,8 +89,7 @@ public class SDTEncoderTest {
count++;
if (count >= start && count < end) {
idx++;
-// long time = Long.parseLong(line.split(",")[0]);
- long time = idx;
+ long time = idx; // not real timestamp, but relative position id
long value = Long.parseLong(line.split(",")[1]);
timestampList.add(time);
valueList.add(value);
@@ -112,59 +114,128 @@ public class SDTEncoderTest {
System.out.println("at=" + selectTimestamps + ";");
System.out.println("av=" + selectValues + ";");
- System.out.println("LB=av-e;\n" + "UB=av+e;\n" + "plot(t,v)\n" + "hold on,plot(t,v,'+')\n"
- + "hold on,plot(at,av,'r')\n" + "%hold on,plot(at,av,'r+')\n" + "hold on,plot(at,UB,'g')\n"
- + "%hold on,plot(at,UB,'g+')\n" + "hold on,plot(at,LB,'g')\n"
- + "%hold on,plot(at,LB,'g+')\n" + "\n" + "% [val,idx]=max(av-e)\n" + "% yline(val)\n" + "\n"
- + "[sortedX, sortedInds] = sort(LB,'descend');\n" + "% yline(sortedX(1))\n"
- + "% yline(sortedX(2))\n" + "\n" + "% 依次计算UB小于Nth-max-LB的点数\n"
- + "% case 1: 两端都小于threshold，则该段内的点的UB都小于threshold，都可以被剪枝\n"
- + "% case 2: 两端都大于等于threshold，则该段内的点的UB都大于等于threshold，则都不可以被剪枝\n"
- + "% case 3: 一端小于一端大于等于threshold，则该段内有部分点的UB小于threshold可以被剪枝\n"
- + "% (t1,v1), (t2,v2), t1<t2\n" + "% y=(t-t1)*(v2-v1)/(t2-t1)+v1\n"
- + "% y<thresold ===> (t-t1)*(v2-v1)/(t2-t1)+v1<threshold\n"
- + "% 如果v2>=threshold>v1: 则t<(threshold-v1)*(t2-t1)/(v2-v1)+t1，于是[t1,(threshold-v1)*(t2-t1)/(v2-v1)+t1)内的点可剪枝\n"
- + "% 如果v2<threshold<=v1: 则t>(threshold-v1)*(t2-t1)/(v2-v1)+t1，于是((threshold-v1)*(t2-t1)/(v2-v1)+t1,t2]内的点可剪枝\n"
- + "rank=1\n" + "threshold=sortedX(rank)\n" + "hold on, yline(threshold);\n"
- + "hold on, plot(at(sortedInds(rank)),threshold,'o')\n" + "\n" + "prune_t=[]; % point\n"
- + "prune_v=[]; % point\n" + "prune_interval=[]; % interval\n" + "interval_start=-1;\n"
- + "for i=2:1:length(av)\n" + "\tt1=at(i-1);\n" + "\tt2=at(i);\n" + "\tv1=UB(i-1);\n"
- + "\tv2=UB(i);\n" + "\ta=[];\n" + "\tif v1<threshold && v2<threshold\n"
- + "\t\tif i<length(av)\n" + "\t\t\ta=t1:1:t2-1;\n" + "\t\telse \n" + "\t\t\ta=t1:1:t2;\n"
- + "\t\tend\n" + "\t\tif interval_start<0\n" + "\t\t\tinterval_start=t1;\n" + " end\n"
- + " interval_end=t2; % continuous\n" + "\telseif v1<threshold && v2>=threshold\n"
- + "\t\ta=t1:1:floor((threshold-v1)*(t2-t1)/(v2-v1)+t1); % no need -1 here\n"
- + "\t\tif interval_start<0\n" + "\t\t\tinterval_start=t1;\n" + " end\n"
- + " prune_interval=[prune_interval;[interval_start,floor((threshold-v1)*(t2-t1)/(v2-v1)+t1)]];\n"
- + "\t\tinterval_start=-1; % discontinuous\n" + "\telseif v1>=threshold && v2<threshold\n"
- + "\t\tif i<length(av)\n" + "\t\t\ta=ceil((threshold-v1)*(t2-t1)/(v2-v1)+t1):1:t2-1;\n"
- + "\t\telse\n" + "\t\t\ta=ceil((threshold-v1)*(t2-t1)/(v2-v1)+t1):1:t2;\n" + "\t\tend\n"
- + "\t\tinterval_start=ceil((threshold-v1)*(t2-t1)/(v2-v1)+t1);\n" + "\t\tinterval_end=t2;\n"
- + "\tend\n" + "\tprune_t=[prune_t,a];\n"
- + "\tprune_v=[prune_v,(a-t1)*(v2-v1)/(t2-t1)+v1];\n" + "end\n" + "if interval_start>0\n"
- + "\tprune_interval=[prune_interval;[interval_start,interval_end]];\n" + "end\n" + "\n"
- + "hold on,plot(prune_t,prune_v,'b.')\n" + "disp(length(prune_t)/length(t))\n" + "\n"
- + "%rectangle('Position', [at(1) av(1) at(2) av(2)], 'Facec',[0.5 1 0.5])\n"
- + "prune_interval\n" + "\n" + "%for i=1:1:length(prune_interval)\n"
- + "%\txline(prune_interval(i,1));\n" + "%\txline(prune_interval(i,2));\n" + "%end\n" + "\n"
- + "% https://se.mathworks.com/help/matlab/ref/xregion.html\n" + "% xregion\n" + "\n"
- + "for i=1:1:length(prune_interval)\n" + "\tx=prune_interval(i,1):1:prune_interval(i,2);\n"
- + "\ty=[];\n" + "\tfor j=1:1:length(x)\n" + "\t\tidx=x(j);\n" + "\t\tfor n=2:1:length(at)\n"
- + "\t\t\tif idx<=at(n)\n" + "\t\t\t\tt1=at(n-1);\n" + "\t\t\t\tt2=at(n);\n"
- + "\t\t\t\tv1=av(n-1);\n" + "\t\t\t\tv2=av(n);\n"
- + "\t\t\t\ty=[y,(idx-t1)*(v2-v1)/(t2-t1)+v1];\n" + "\t\t\t\tbreak;\n" + "\t\t\tend\n"
- + "\t\tend\n" + "\tend\n" + "\ty1=y-e;\n" + "\ty2=y+e;\n" + "\tL1=[x,fliplr(x)];\n"
- + "\tL2=[y1,fliplr(y2)];\n" + "\tfill(L1,L2,'b','FaceAlpha',0.1);\n" + "end");
+ System.out.println(
+ "LB=av-e;\n"
+ + "UB=av+e;\n"
+ + "plot(t,v)\n"
+ + "hold on,plot(t,v,'+')\n"
+ + "hold on,plot(at,av,'r')\n"
+ + "%hold on,plot(at,av,'r+')\n"
+ + "hold on,plot(at,UB,'g')\n"
+ + "%hold on,plot(at,UB,'g+')\n"
+ + "hold on,plot(at,LB,'g')\n"
+ + "%hold on,plot(at,LB,'g+')\n"
+ + "\n"
+ + "% [val,idx]=max(av-e)\n"
+ + "% yline(val)\n"
+ + "\n"
+ + "[sortedX, sortedInds] = sort(LB,'descend');\n"
+ + "% yline(sortedX(1))\n"
+ + "% yline(sortedX(2))\n"
+ + "\n"
+ + "% 依次计算UB小于Nth-max-LB的点数\n"
+ + "% case 1: 两端都小于threshold，则该段内的点的UB都小于threshold，都可以被剪枝\n"
+ + "% case 2: 两端都大于等于threshold，则该段内的点的UB都大于等于threshold，则都不可以被剪枝\n"
+ + "% case 3: 一端小于一端大于等于threshold，则该段内有部分点的UB小于threshold可以被剪枝\n"
+ + "% (t1,v1), (t2,v2), t1<t2\n"
+ + "% y=(t-t1)*(v2-v1)/(t2-t1)+v1\n"
+ + "% y<thresold ===> (t-t1)*(v2-v1)/(t2-t1)+v1<threshold\n"
+ + "% 如果v2>=threshold>v1: 则t<(threshold-v1)*(t2-t1)/(v2-v1)+t1，于是[t1,(threshold-v1)*(t2-t1)/(v2-v1)+t1)内的点可剪枝\n"
+ + "% 如果v2<threshold<=v1: 则t>(threshold-v1)*(t2-t1)/(v2-v1)+t1，于是((threshold-v1)*(t2-t1)/(v2-v1)+t1,t2]内的点可剪枝\n"
+ + "rank=1\n"
+ + "threshold=sortedX(rank)\n"
+ + "hold on, yline(threshold);\n"
+ + "hold on, plot(at(sortedInds(rank)),threshold,'o')\n"
+ + "\n"
+ + "prune_t=[]; % point\n"
+ + "prune_v=[]; % point\n"
+ + "prune_interval=[]; % interval\n"
+ + "interval_start=-1;\n"
+ + "for i=2:1:length(av)\n"
+ + "\tt1=at(i-1);\n"
+ + "\tt2=at(i);\n"
+ + "\tv1=UB(i-1);\n"
+ + "\tv2=UB(i);\n"
+ + "\ta=[];\n"
+ + "\tif v1<threshold && v2<threshold\n"
+ + "\t\tif i<length(av)\n"
+ + "\t\t\ta=t1:1:t2-1;\n"
+ + "\t\telse \n"
+ + "\t\t\ta=t1:1:t2;\n"
+ + "\t\tend\n"
+ + "\t\tif interval_start<0\n"
+ + "\t\t\tinterval_start=t1;\n"
+ + " end\n"
+ + " interval_end=t2; % continuous\n"
+ + "\telseif v1<threshold && v2>=threshold\n"
+ + "\t\ta=t1:1:floor((threshold-v1)*(t2-t1)/(v2-v1)+t1); % no need -1 here\n"
+ + "\t\tif interval_start<0\n"
+ + "\t\t\tinterval_start=t1;\n"
+ + " end\n"
+ + " prune_interval=[prune_interval;[interval_start,floor((threshold-v1)*(t2-t1)/(v2-v1)+t1)]];\n"
+ + "\t\tinterval_start=-1; % discontinuous\n"
+ + "\telseif v1>=threshold && v2<threshold\n"
+ + "\t\tif i<length(av)\n"
+ + "\t\t\ta=ceil((threshold-v1)*(t2-t1)/(v2-v1)+t1):1:t2-1;\n"
+ + "\t\telse\n"
+ + "\t\t\ta=ceil((threshold-v1)*(t2-t1)/(v2-v1)+t1):1:t2;\n"
+ + "\t\tend\n"
+ + "\t\tinterval_start=ceil((threshold-v1)*(t2-t1)/(v2-v1)+t1);\n"
+ + "\t\tinterval_end=t2;\n"
+ + "\tend\n"
+ + "\tprune_t=[prune_t,a];\n"
+ + "\tprune_v=[prune_v,(a-t1)*(v2-v1)/(t2-t1)+v1];\n"
+ + "end\n"
+ + "if interval_start>0\n"
+ + "\tprune_interval=[prune_interval;[interval_start,interval_end]];\n"
+ + "end\n"
+ + "\n"
+ + "hold on,plot(prune_t,prune_v,'b.')\n"
+ + "disp(length(prune_t)/length(t))\n"
+ + "\n"
+ + "%rectangle('Position', [at(1) av(1) at(2) av(2)], 'Facec',[0.5 1 0.5])\n"
+ + "prune_interval\n"
+ + "\n"
+ + "%for i=1:1:length(prune_interval)\n"
+ + "%\txline(prune_interval(i,1));\n"
+ + "%\txline(prune_interval(i,2));\n"
+ + "%end\n"
+ + "\n"
+ + "% https://se.mathworks.com/help/matlab/ref/xregion.html\n"
+ + "% xregion\n"
+ + "\n"
+ + "for i=1:1:length(prune_interval)\n"
+ + "\tx=prune_interval(i,1):1:prune_interval(i,2);\n"
+ + "\ty=[];\n"
+ + "\tfor j=1:1:length(x)\n"
+ + "\t\tidx=x(j);\n"
+ + "\t\tfor n=2:1:length(at)\n"
+ + "\t\t\tif idx<=at(n)\n"
+ + "\t\t\t\tt1=at(n-1);\n"
+ + "\t\t\t\tt2=at(n);\n"
+ + "\t\t\t\tv1=av(n-1);\n"
+ + "\t\t\t\tv2=av(n);\n"
+ + "\t\t\t\ty=[y,(idx-t1)*(v2-v1)/(t2-t1)+v1];\n"
+ + "\t\t\t\tbreak;\n"
+ + "\t\t\tend\n"
+ + "\t\tend\n"
+ + "\tend\n"
+ + "\ty1=y-e;\n"
+ + "\ty2=y+e;\n"
+ + "\tL1=[x,fliplr(x)];\n"
+ + "\tL2=[y1,fliplr(y2)];\n"
+ + "\tfill(L1,L2,'b','FaceAlpha',0.1);\n"
+ + "end");
}
@Test
public void mytest2() throws Exception {
String csvData = "D:\\full-game\\BallSpeed.csv";
int start = 200000;
- int range = 200;
+ int range = 100000;
int end = start + range;
SDTEncoder encoder = new SDTEncoder();
- double e = 80000.0 / 2; // std/2
+ double e = 100000; // std/2
encoder.setCompDeviation(e / 2);
long count = 0;
String line;
@@ -172,17 +243,18 @@ public class SDTEncoderTest {
List<Long> valueList = new ArrayList<>();
List<Integer> selectTimestamps = new ArrayList<>();
List<Long> selectValues = new ArrayList<>();
+ List<ValuePoint> points = new ArrayList<>();
int idx = 0;
try (BufferedReader reader = new BufferedReader(new FileReader(csvData))) {
while ((line = reader.readLine()) != null) {
count++;
if (count >= start && count < end) {
idx++;
-// long time = Long.parseLong(line.split(",")[0]);
- int time = idx;
+ int time = idx; // not real timestamp, but relative position id
long value = Long.parseLong(line.split(",")[1]);
timestampList.add(time);
valueList.add(value);
+ points.add(new ValuePoint(time, value));
if (encoder.encodeLong(time, value)) {
selectTimestamps.add((int) encoder.getTime());
selectValues.add(encoder.getLongValue());
@@ -197,18 +269,31 @@ public class SDTEncoderTest {
}
}
- System.out.println("close all;clear all;clc;");
- System.out.println("e=" + e);
- System.out.println("t=" + timestampList + ";");
- System.out.println("v=" + valueList + ";");
- System.out.println("at=" + selectTimestamps + ";");
- System.out.println("av=" + selectValues + ";");
+ // System.out.println("e=" + e);
+ // System.out.println("t=" + timestampList + ";");
+ // System.out.println("v=" + valueList + ";");
+ // System.out.println("at=" + selectTimestamps + ";");
+ // System.out.println("av=" + selectValues + ";");
+ System.out.println("v.size=" + valueList.size());
+ System.out.println("av.size=" + selectTimestamps.size());
+
+ System.out.println("start test------");
+
+ // ValuePoint[] myArray = (ValuePoint[]) points.toArray();
+ // Arrays.sort(points.toArray());
+ points.sort(
+ new Comparator<ValuePoint>() {
+ public int compare(ValuePoint o1, ValuePoint o2) {
+ return ((Comparable) (o1.value)).compareTo(o2.value);
+ }
+ });
+
+ long startTime = System.nanoTime();
// 计算maxLB
- long maxVal = Collections.max(selectValues);
-// int maxIdx = selectTimestamps.get(selectValues.indexOf(maxVal));
+ long maxVal = points.get(points.size() - 1).value;
double threshold = maxVal - e; // maxLB
- System.out.println("threshold(maxLB)=" + threshold);
+ // System.out.println("threshold(maxLB)=" + threshold);
// 计算UB<maxLB的剪枝段
List<Integer> prune_intervals_start = new ArrayList<>();
@@ -241,13 +326,13 @@ public class SDTEncoderTest {
prune_intervals_start.add(interval_start);
prune_intervals_end.add(interval_end);
}
- System.out.println("UB<maxLB prune intervals (included ends):");
- System.out.println(prune_intervals_start);
- System.out.println(prune_intervals_end);
+ // System.out.println("UB<maxLB prune intervals (included ends, starting from 1):");
+ // System.out.println(prune_intervals_start);
+ // System.out.println(prune_intervals_end);
// 对剪枝段取余后搜索遍历计算TP
int startIdx = 1;
- int endIdx = 200;
+ int endIdx = range;
startIdx = startIdx - 1; // as excluded treated
endIdx = endIdx + 1; // as excluded treated
prune_intervals_start.add(endIdx); // turn into search_intervals_end (excluded)
@@ -257,16 +342,43 @@ public class SDTEncoderTest {
for (int i = 0; i < prune_intervals_start.size(); i++) {
int search_interval_start = prune_intervals_end.get(i) + 1; // included
int search_interval_end = prune_intervals_start.get(i) - 1; // included
- System.out.println(search_interval_start + "," + search_interval_end);
- for (int j = search_interval_start; j <= search_interval_end; j++) {
- long v = valueList.get(j);
+ // System.out.println(search_interval_start + "," + search_interval_end);
+ for (int j = search_interval_start; j <= search_interval_end; j++) { // starting from 1
+ long v = valueList.get(j - 1); // java list starting from 0
if (v > candidateTPvalue) {
candidateTPvalue = v;
candidateTPidx = j;
}
}
}
+ long elapsedTime = System.nanoTime() - startTime;
+ System.out.println("search with value index: " + elapsedTime / 1000000.0 + " m s");
System.out.println("TP=(" + candidateTPidx + "," + candidateTPvalue + ")");
+ System.out.println("search interval number=" + prune_intervals_end.size());
+ int traversedPoints = 0;
+ for (int i = 0; i < prune_intervals_start.size(); i++) {
+ int search_interval_start = prune_intervals_end.get(i) + 1; // included
+ int search_interval_end = prune_intervals_start.get(i) - 1; // included
+ if (search_interval_end >= search_interval_start) {
+ traversedPoints += (search_interval_end - search_interval_start + 1);
+ }
+ }
+ System.out.println("number of traversed points: " + traversedPoints);
+
+ System.out.println("start test------");
+ startTime = System.nanoTime();
+ long candidateTPvalue_raw = -1;
+ int candidateTPidx_raw = -1;
+ for (int j = 1; j <= valueList.size(); j++) { // starting from 1
+ long v = valueList.get(j - 1); // java list starting from 0
+ if (v > candidateTPvalue_raw) {
+ candidateTPvalue_raw = v;
+ candidateTPidx_raw = j;
+ }
+ }
+ elapsedTime = System.nanoTime() - startTime;
+ System.out.println("search without value index: " + elapsedTime / 1000000.0 + " m s");
+ System.out.println("TP=(" + candidateTPidx_raw + "," + candidateTPvalue_raw + ")");
}
@Test
[iotdb] 37/46: add metric
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 89c9cb7cb11ad386af8810136feaaf0723387c2a
Author: Lei Rui <10...@qq.com>
AuthorDate: Thu Jul 6 15:13:08 2023 +0800
add metric
---
.../iotdb/tsfile/read/common/ChunkSuit4CPV.java | 26 ++++++++++----------
.../iotdb/tsfile/read/common/IOMonitor2.java | 28 ++++++++++++++++------
.../iotdb/tsfile/read/reader/page/PageReader.java | 23 +++++++++---------
3 files changed, 45 insertions(+), 32 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
index 18d61065299..e0de93e5079 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
@@ -262,18 +262,18 @@ public class ChunkSuit4CPV {
while (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
estimatedPos++;
// IOMonitor.incPointsTravered();
- IOMonitor2.DCP_D_traversedPointNum++;
+ IOMonitor2.DCP_D_timeIndex_traversedPointNum++;
}
} else if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
while (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
estimatedPos--;
// IOMonitor.incPointsTravered();
- IOMonitor2.DCP_D_traversedPointNum++;
+ IOMonitor2.DCP_D_timeIndex_traversedPointNum++;
}
if (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
estimatedPos++;
// IOMonitor.incPointsTravered();
- IOMonitor2.DCP_D_traversedPointNum++;
+ IOMonitor2.DCP_D_timeIndex_traversedPointNum++;
} // else equal
} // else equal
this.startPos = estimatedPos; // note this
@@ -316,7 +316,7 @@ public class ChunkSuit4CPV {
while (pageReader.timeBuffer.remaining() > 0) {
estimatedPos++;
// IOMonitor.incPointsTravered();
- IOMonitor2.DCP_D_traversedPointNum++;
+ IOMonitor2.DCP_D_timeIndex_traversedPointNum++;
long t = pageReader.timeBuffer.getLong();
if (t >= targetTimestamp) {
break;
@@ -379,18 +379,18 @@ public class ChunkSuit4CPV {
while (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
estimatedPos--;
// IOMonitor.incPointsTravered();
- IOMonitor2.DCP_D_traversedPointNum++;
+ IOMonitor2.DCP_D_timeIndex_traversedPointNum++;
}
} else if (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
while (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
estimatedPos++;
// IOMonitor.incPointsTravered();
- IOMonitor2.DCP_D_traversedPointNum++;
+ IOMonitor2.DCP_D_timeIndex_traversedPointNum++;
}
if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
estimatedPos--;
// IOMonitor.incPointsTravered();
- IOMonitor2.DCP_D_traversedPointNum++;
+ IOMonitor2.DCP_D_timeIndex_traversedPointNum++;
} // else equal
} // else equal
this.endPos = estimatedPos; // note this
@@ -433,7 +433,7 @@ public class ChunkSuit4CPV {
while (pageReader.timeBuffer.remaining() > 0) {
estimatedPos++;
// IOMonitor.incPointsTravered();
- IOMonitor2.DCP_D_traversedPointNum++;
+ IOMonitor2.DCP_D_timeIndex_traversedPointNum++;
long t = pageReader.timeBuffer.getLong();
if (t >= targetTimestamp) {
break;
@@ -441,7 +441,7 @@ public class ChunkSuit4CPV {
}
if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
// IOMonitor.incPointsTravered();
- IOMonitor2.DCP_D_traversedPointNum++;
+ IOMonitor2.DCP_D_timeIndex_traversedPointNum++;
estimatedPos--;
} // else equals no need to minus 1
@@ -502,18 +502,18 @@ public class ChunkSuit4CPV {
while (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
estimatedPos--;
// IOMonitor.incPointsTravered();
- IOMonitor2.DCP_D_traversedPointNum++;
+ IOMonitor2.DCP_D_timeIndex_traversedPointNum++;
}
} else if (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
while (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
estimatedPos++;
// IOMonitor.incPointsTravered();
- IOMonitor2.DCP_D_traversedPointNum++;
+ IOMonitor2.DCP_D_timeIndex_traversedPointNum++;
}
if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
estimatedPos--;
// IOMonitor.incPointsTravered();
- IOMonitor2.DCP_D_traversedPointNum++;
+ IOMonitor2.DCP_D_timeIndex_traversedPointNum++;
} // else equal
} // else equal
@@ -532,7 +532,7 @@ public class ChunkSuit4CPV {
if (!flag) {
estimatedPos++;
// IOMonitor.incPointsTravered();
- IOMonitor2.DCP_D_traversedPointNum++;
+ IOMonitor2.DCP_D_timeIndex_traversedPointNum++;
}
long t = pageReader.timeBuffer.getLong();
if (t >= targetTimestamp) {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
index 44f5084c5da..e2bececf2c6 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
@@ -98,11 +98,8 @@ public class IOMonitor2 {
public static int DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_count = 0; // level 4
public static long DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_ns = 0; // level 4
-
- public static long DCP_D_traversedPointNum = 0; // level 4
public static int DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_count = 0; // level 4
public static long DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_ns = 0; // level 4
-
public static int SEARCH_ARRAY_a_verifBPTP_count = 0; // level 4
public static long SEARCH_ARRAY_a_verifBPTP_ns = 0; // level 4
@@ -115,6 +112,10 @@ public class IOMonitor2 {
public static int SEARCH_ARRAY_c_genBPTP_count = 0; // level 4
public static long SEARCH_ARRAY_c_genBPTP_ns = 0; // level 4
+ public static long DCP_D_getAllSatisfiedPageData_traversedPointNum = 0; // level 5
+ public static long DCP_D_timeIndex_traversedPointNum = 0; // level 5
+ public static long DCP_D_valueIndex_traversedPointNum = 0; // level 5
+
public static Operation M4_LSM_status =
null; // for counting the number of calling BCD by each step of M4-LSM
public static int M4_LSM_merge_M4_time_span_B_READ_MEM_CHUNK_cnt = 0; // map from level 3 to 4
@@ -195,8 +196,6 @@ public class IOMonitor2 {
DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_count = 0;
DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_ns = 0;
- DCP_D_traversedPointNum = 0;
-
DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_count = 0;
DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_ns = 0;
@@ -209,6 +208,11 @@ public class IOMonitor2 {
SEARCH_ARRAY_c_genBPTP_count = 0;
SEARCH_ARRAY_c_genBPTP_ns = 0;
+ /** level 5 */
+ DCP_D_getAllSatisfiedPageData_traversedPointNum = 0;
+ DCP_D_timeIndex_traversedPointNum = 0;
+ DCP_D_valueIndex_traversedPointNum = 0;
+
/** map from level 3 to level 4 */
M4_LSM_status = null;
M4_LSM_merge_M4_time_span_B_READ_MEM_CHUNK_cnt =
@@ -593,9 +597,19 @@ public class IOMonitor2 {
.append(SEARCH_ARRAY_c_genBPTP_count)
.append("\n");
stringBuilder
- .append("[4-cnt]DCP_D_traversedPointNum")
+ .append("[5-cnt]DCP_D_getAllSatisfiedPageData_traversedPointNum")
+ .append(",")
+ .append(DCP_D_getAllSatisfiedPageData_traversedPointNum)
+ .append("\n");
+ stringBuilder
+ .append("[5-cnt]DCP_D_timeIndex_traversedPointNum")
+ .append(",")
+ .append(DCP_D_timeIndex_traversedPointNum)
+ .append("\n");
+ stringBuilder
+ .append("[5-cnt]DCP_D_valueIndex_traversedPointNum")
.append(",")
- .append(DCP_D_traversedPointNum)
+ .append(DCP_D_valueIndex_traversedPointNum)
.append("\n");
stringBuilder
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
index 27dac52c0b4..6c63f57282b 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
@@ -179,7 +179,7 @@ public class PageReader implements IPageReader {
double foundValue = 0;
// iterate SDT points from value big to small to find the first point not deleted
for (int n = valueIndex.sortedModelPoints.size() - 1; n >= 0; n--) { // NOTE from big to small
- IOMonitor2.DCP_D_traversedPointNum++; // TODO
+ IOMonitor2.DCP_D_valueIndex_traversedPointNum++; // TODO
ValuePoint valuePoint = valueIndex.sortedModelPoints.get(n);
int idx = valuePoint.index; // index starting from 1
int pos = idx - 1; // pos starting from 0
@@ -232,7 +232,7 @@ public class PageReader implements IPageReader {
while (localCursor < valueIndex.modelPointIdx_list.size()
&& (idx1 = valueIndex.modelPointIdx_list.get(localCursor - 1)) - 1
<= chunkSuit4CPV.endPos) {
- IOMonitor2.DCP_D_traversedPointNum++; // TODO
+ IOMonitor2.DCP_D_valueIndex_traversedPointNum++; // TODO
idx2 = valueIndex.modelPointIdx_list.get(localCursor);
double v1_UB = valueIndex.modelPointVal_list.get(localCursor - 1) + valueIndex.errorBound;
double v2_UB = valueIndex.modelPointVal_list.get(localCursor) + valueIndex.errorBound;
@@ -310,7 +310,7 @@ public class PageReader implements IPageReader {
for (int j = search_interval_start;
j <= search_interval_end;
j++) { // idx starting from 1
- IOMonitor2.DCP_D_traversedPointNum++; // TODO
+ IOMonitor2.DCP_D_valueIndex_traversedPointNum++; // TODO
double v = valueBuffer.getDouble(timeBufferLength + (j - 1) * 8); // pos starting from 0
long t = timeBuffer.getLong((j - 1) * 8); // pos starting from 0
if (v > candidateTPvalue && !isDeleted_NoHistoryDeleteCursor(t)) {
@@ -327,7 +327,7 @@ public class PageReader implements IPageReader {
int search_interval_start = prune_intervals_end.get(i) + 1; // included
int search_interval_end = prune_intervals_start.get(i) - 1; // included
for (int j = search_interval_start; j <= search_interval_end; j++) { // starting from 1
- IOMonitor2.DCP_D_traversedPointNum++; // TODO
+ IOMonitor2.DCP_D_valueIndex_traversedPointNum++; // TODO
long v = valueBuffer.getLong(timeBufferLength + (j - 1) * 8);
long t = timeBuffer.getLong((j - 1) * 8);
if (v > candidateTPvalue && !isDeleted_NoHistoryDeleteCursor(t)) {
@@ -358,7 +358,7 @@ public class PageReader implements IPageReader {
double foundValue = 0;
// iterate SDT points from value small to big to find the first point not deleted
for (int n = 0; n < valueIndex.sortedModelPoints.size(); n++) { // NOTE from small to big
- IOMonitor2.DCP_D_traversedPointNum++; // TODO
+ IOMonitor2.DCP_D_valueIndex_traversedPointNum++; // TODO
ValuePoint valuePoint = valueIndex.sortedModelPoints.get(n);
int idx = valuePoint.index; // index starting from 1
int pos = idx - 1; // pos starting from 0
@@ -390,7 +390,7 @@ public class PageReader implements IPageReader {
// time span
while ((idx2 = valueIndex.modelPointIdx_list.get(chunkSuit4CPV.modelPointsCursor)) - 1
< chunkSuit4CPV.startPos) {
- IOMonitor2.DCP_D_traversedPointNum++; // TODO
+ IOMonitor2.DCP_D_valueIndex_traversedPointNum++; // TODO
// -1 because idx starting from 1 while pos starting from 0
chunkSuit4CPV.modelPointsCursor++;
// pointing to the right end of the first model segment that passes the left endpoint of the
@@ -412,7 +412,7 @@ public class PageReader implements IPageReader {
while (localCursor < valueIndex.modelPointIdx_list.size()
&& (idx1 = valueIndex.modelPointIdx_list.get(localCursor - 1)) - 1
<= chunkSuit4CPV.endPos) {
- IOMonitor2.DCP_D_traversedPointNum++; // TODO
+ IOMonitor2.DCP_D_valueIndex_traversedPointNum++; // TODO
idx2 = valueIndex.modelPointIdx_list.get(localCursor);
double v1_LB = valueIndex.modelPointVal_list.get(localCursor - 1) - valueIndex.errorBound;
double v2_LB = valueIndex.modelPointVal_list.get(localCursor) - valueIndex.errorBound;
@@ -490,7 +490,7 @@ public class PageReader implements IPageReader {
for (int j = search_interval_start;
j <= search_interval_end;
j++) { // idx starting from 1
- IOMonitor2.DCP_D_traversedPointNum++; // TODO
+ IOMonitor2.DCP_D_valueIndex_traversedPointNum++; // TODO
double v = valueBuffer.getDouble(timeBufferLength + (j - 1) * 8); // pos starting from 0
long t = timeBuffer.getLong((j - 1) * 8); // pos starting from 0
if (v < candidateBPvalue && !isDeleted_NoHistoryDeleteCursor(t)) {
@@ -507,7 +507,7 @@ public class PageReader implements IPageReader {
int search_interval_start = prune_intervals_end.get(i) + 1; // included
int search_interval_end = prune_intervals_start.get(i) - 1; // included
for (int j = search_interval_start; j <= search_interval_end; j++) { // starting from 1
- IOMonitor2.DCP_D_traversedPointNum++; // TODO
+ IOMonitor2.DCP_D_valueIndex_traversedPointNum++; // TODO
long v = valueBuffer.getLong(timeBufferLength + (j - 1) * 8);
long t = timeBuffer.getLong((j - 1) * 8);
if (v < candidateBPvalue && !isDeleted_NoHistoryDeleteCursor(t)) {
@@ -547,7 +547,7 @@ public class PageReader implements IPageReader {
int count = 0; // update here, not in statistics
for (int pos = chunkSuit4CPV.startPos; pos <= chunkSuit4CPV.endPos; pos++) {
// IOMonitor.incPointsTravered();
- IOMonitor2.DCP_D_traversedPointNum++;
+ IOMonitor2.DCP_D_valueIndex_traversedPointNum++;
long timestamp = timeBuffer.getLong(pos * 8);
switch (dataType) {
case INT64:
@@ -603,8 +603,7 @@ public class PageReader implements IPageReader {
BatchData pageData = BatchDataFactory.createBatchData(dataType, ascending, false);
while (timeDecoder.hasNext(timeBuffer)) {
- // IOMonitor.incPointsTravered();
- IOMonitor2.DCP_D_traversedPointNum++;
+ IOMonitor2.DCP_D_getAllSatisfiedPageData_traversedPointNum++;
long timestamp = timeDecoder.readLong(timeBuffer);
switch (dataType) {
case BOOLEAN:
[iotdb] 01/46: fix sketch script
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5cc69fee9a0fb7fb09db0089ca1de060da32cba0
Author: Lei Rui <10...@qq.com>
AuthorDate: Sun Jun 11 14:33:45 2023 +0800
fix sketch script
---
.../src/assembly/resources/tools/tsfileToolSet/print-tsfile-sketch.sh | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/server/src/assembly/resources/tools/tsfileToolSet/print-tsfile-sketch.sh b/server/src/assembly/resources/tools/tsfileToolSet/print-tsfile-sketch.sh
index 7d10136b4b0..1a55ed95e7d 100755
--- a/server/src/assembly/resources/tools/tsfileToolSet/print-tsfile-sketch.sh
+++ b/server/src/assembly/resources/tools/tsfileToolSet/print-tsfile-sketch.sh
@@ -26,6 +26,8 @@ if [ -z "${IOTDB_HOME}" ]; then
export IOTDB_HOME="$(cd "`dirname "$0"`"/../..; pwd)"
fi
+IOTDB_CONF=${IOTDB_HOME}/conf
+
if [ -n "$JAVA_HOME" ]; then
for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do
if [ -x "$java" ]; then
@@ -44,5 +46,5 @@ done
MAIN_CLASS=org.apache.iotdb.db.tools.TsFileSketchTool
-"$JAVA" -cp "$CLASSPATH" "$MAIN_CLASS" "$@"
+"$JAVA" "-DTSFILE_CONF=${IOTDB_CONF}" -cp "$CLASSPATH" "$MAIN_CLASS" "$@"
exit $?
\ No newline at end of file
[iotdb] 10/46: finish debug
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5551260886623c0e6117fe8c98bfdd92e599fd45
Author: Lei Rui <10...@qq.com>
AuthorDate: Tue Jun 13 16:24:46 2023 +0800
finish debug
---
.../org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor.java | 1 +
1 file changed, 1 insertion(+)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor.java
index b60ac39ea00..1914c508193 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor.java
@@ -321,6 +321,7 @@ public class LocalGroupByExecutor implements GroupByExecutor {
// can use pageHeader
if (reader.canUseCurrentPageStatistics()
&& timeRange.contains(pageStatistics.getStartTime(), pageStatistics.getEndTime())) {
+ // here embodies the number of chunks that moc waste loading and decompressing
calcFromStatistics(pageStatistics);
reader.skipCurrentPage();
if (isEndCalc()) {
[iotdb] 17/46: modify print
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 4173b42565bb1085bc637a045af129601ab2fc50
Author: Lei Rui <10...@qq.com>
AuthorDate: Thu Jun 15 12:41:54 2023 +0800
modify print
---
.../iotdb/tsfile/read/common/IOMonitor2.java | 115 ++++++++++++---------
1 file changed, 67 insertions(+), 48 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
index ec0d78d777b..903aa402654 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
@@ -451,124 +451,143 @@ public class IOMonitor2 {
public static String print() {
StringBuilder stringBuilder = new StringBuilder();
- // [1] is client elapsed time, not measured by the server side
+ // [1-cnt] is client elapsed time, not measured by the server side
stringBuilder
- .append("[2]Server_Query_Execute_ns_cnt")
+ .append("[2-ns]Server_Query_Execute")
.append(",")
.append(DCP_Server_Query_Execute_ns)
- .append(",")
- .append(DCP_Server_Query_Execute_count)
.append("\n");
stringBuilder
- .append("[2]Server_Query_Fetch_ns_cnt")
+ .append("[2-ns]Server_Query_Fetch")
.append(",")
.append(DCP_Server_Query_Fetch_ns)
- .append(",")
- .append(DCP_Server_Query_Fetch_count)
.append("\n");
stringBuilder.append("[3]dataSetType").append(",").append(dataSetType).append("\n");
stringBuilder
- .append("[3]M4_LSM_init_loadAllChunkMetadatas_ns_cnt")
+ .append("[3-ns]M4_LSM_init_loadAllChunkMetadatas")
.append(",")
.append(M4_LSM_init_loadAllChunkMetadatas_ns)
- .append(",")
- .append(M4_LSM_init_loadAllChunkMetadatas_count)
.append("\n");
stringBuilder
- .append("[3]M4_LSM_merge_M4_time_span_ns_cnt")
+ .append("[3-ns]M4_LSM_merge_M4_time_span")
.append(",")
.append(M4_LSM_merge_M4_time_span_ns)
+ .append("\n");
+ stringBuilder.append("[3-ns]M4_LSM_FP").append(",").append(M4_LSM_FP_ns).append("\n");
+ stringBuilder.append("[3-ns]M4_LSM_LP").append(",").append(M4_LSM_LP_ns).append("\n");
+ stringBuilder.append("[3-ns]M4_LSM_BP").append(",").append(M4_LSM_BP_ns).append("\n");
+ stringBuilder.append("[3-ns]M4_LSM_TP").append(",").append(M4_LSM_TP_ns).append("\n");
+
+ stringBuilder
+ .append("[4-ns]DCP_A_GET_CHUNK_METADATAS")
.append(",")
- .append(M4_LSM_merge_M4_time_span_count)
+ .append(DCP_A_GET_CHUNK_METADATAS_ns)
.append("\n");
stringBuilder
- .append("[3]M4_LSM_FP_ns_cnt")
+ .append("[4-ns]DCP_B_READ_MEM_CHUNK")
.append(",")
- .append(M4_LSM_FP_ns)
+ .append(DCP_B_READ_MEM_CHUNK_ns)
+ .append("\n");
+ stringBuilder
+ .append("[4-ns]DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA")
.append(",")
- .append(M4_LSM_FP_count)
+ .append(DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_ns)
.append("\n");
stringBuilder
- .append("[3]M4_LSM_LP_ns_cnt")
+ .append("[4-ns]DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS")
.append(",")
- .append(M4_LSM_LP_ns)
+ .append(DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_ns)
+ .append("\n");
+ stringBuilder
+ .append("[4-ns]SEARCH_ARRAY_a_verifBPTP")
.append(",")
- .append(M4_LSM_LP_count)
+ .append(SEARCH_ARRAY_a_verifBPTP_ns)
.append("\n");
stringBuilder
- .append("[3]M4_LSM_BP_ns_cnt")
+ .append("[4-ns]SEARCH_ARRAY_b_genFP")
.append(",")
- .append(M4_LSM_BP_ns)
+ .append(SEARCH_ARRAY_b_genFP_ns)
+ .append("\n");
+ stringBuilder
+ .append("[4-ns]SEARCH_ARRAY_b_genLP")
.append(",")
- .append(M4_LSM_BP_count)
+ .append(SEARCH_ARRAY_b_genLP_ns)
.append("\n");
stringBuilder
- .append("[3]M4_LSM_TP_ns_cnt")
+ .append("[4-ns]SEARCH_ARRAY_c_genBPTP")
.append(",")
- .append(M4_LSM_TP_ns)
+ .append(SEARCH_ARRAY_c_genBPTP_ns)
+ .append("\n");
+
+ stringBuilder
+ .append("[2-cnt]Server_Query_Execute")
.append(",")
- .append(M4_LSM_TP_count)
+ .append(DCP_Server_Query_Execute_count)
+ .append("\n");
+ stringBuilder
+ .append("[2-cnt]Server_Query_Fetch")
+ .append(",")
+ .append(DCP_Server_Query_Fetch_count)
.append("\n");
stringBuilder
- .append("[4]DCP_A_GET_CHUNK_METADATAS_ns_cnt")
+ .append("[3-cnt]M4_LSM_init_loadAllChunkMetadatas")
.append(",")
- .append(DCP_A_GET_CHUNK_METADATAS_ns)
+ .append(M4_LSM_init_loadAllChunkMetadatas_count)
+ .append("\n");
+ stringBuilder
+ .append("[3-cnt]M4_LSM_merge_M4_time_span")
.append(",")
- .append(DCP_A_GET_CHUNK_METADATAS_count)
+ .append(M4_LSM_merge_M4_time_span_count)
.append("\n");
+ stringBuilder.append("[3-cnt]M4_LSM_FP").append(",").append(M4_LSM_FP_count).append("\n");
+ stringBuilder.append("[3-cnt]M4_LSM_LP").append(",").append(M4_LSM_LP_count).append("\n");
+ stringBuilder.append("[3-cnt]M4_LSM_BP").append(",").append(M4_LSM_BP_count).append("\n");
+ stringBuilder.append("[3-cnt]M4_LSM_TP").append(",").append(M4_LSM_TP_count).append("\n");
+
stringBuilder
- .append("[4]DCP_B_READ_MEM_CHUNK_ns_cnt")
+ .append("[4-cnt]DCP_A_GET_CHUNK_METADATAS")
.append(",")
- .append(DCP_B_READ_MEM_CHUNK_ns)
+ .append(DCP_A_GET_CHUNK_METADATAS_count)
+ .append("\n");
+ stringBuilder
+ .append("[4-cnt]DCP_B_READ_MEM_CHUNK")
.append(",")
.append(DCP_B_READ_MEM_CHUNK_count)
.append("\n");
stringBuilder
- .append("[4]DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_ns_cnt")
- .append(",")
- .append(DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_ns)
+ .append("[4-cnt]DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA")
.append(",")
.append(DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_count)
.append("\n");
stringBuilder
- .append("[4]DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_ns_cnt")
- .append(",")
- .append(DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_ns)
+ .append("[4-cnt]DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS")
.append(",")
.append(DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_count)
.append("\n");
stringBuilder
- .append("[4]SEARCH_ARRAY_a_verifBPTP_ns_cnt")
- .append(",")
- .append(SEARCH_ARRAY_a_verifBPTP_ns)
+ .append("[4-cnt]SEARCH_ARRAY_a_verifBPTP")
.append(",")
.append(SEARCH_ARRAY_a_verifBPTP_count)
.append("\n");
stringBuilder
- .append("[4]SEARCH_ARRAY_b_genFP_ns_cnt")
- .append(",")
- .append(SEARCH_ARRAY_b_genFP_ns)
+ .append("[4-cnt]SEARCH_ARRAY_b_genFP")
.append(",")
.append(SEARCH_ARRAY_b_genFP_count)
.append("\n");
stringBuilder
- .append("[4]SEARCH_ARRAY_b_genLP_ns_cnt")
- .append(",")
- .append(SEARCH_ARRAY_b_genLP_ns)
+ .append("[4-cnt]SEARCH_ARRAY_b_genLP")
.append(",")
.append(SEARCH_ARRAY_b_genLP_count)
.append("\n");
stringBuilder
- .append("[4]SEARCH_ARRAY_c_genBPTP_ns_cnt")
- .append(",")
- .append(SEARCH_ARRAY_c_genBPTP_ns)
+ .append("[4-cnt]SEARCH_ARRAY_c_genBPTP")
.append(",")
.append(SEARCH_ARRAY_c_genBPTP_count)
.append("\n");
stringBuilder
- .append("[4]DCP_D_traversedPointNum")
+ .append("[4-cnt]DCP_D_traversedPointNum")
.append(",")
.append(DCP_D_traversedPointNum)
.append("\n");
[iotdb] 42/46: fix
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit d400a425893112790ebcbe1c3dd67ca4a2d063fc
Author: Lei Rui <10...@qq.com>
AuthorDate: Fri Jul 7 14:30:59 2023 +0800
fix
---
.../apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java | 17 +++++++++++------
1 file changed, 11 insertions(+), 6 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
index 57250041c2e..daa37817ca8 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
@@ -32,9 +32,13 @@ import org.apache.iotdb.tsfile.file.metadata.statistics.StepRegress;
import org.apache.iotdb.tsfile.read.common.IOMonitor2.Operation;
import org.apache.iotdb.tsfile.read.reader.page.PageReader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.io.IOException;
public class ChunkSuit4CPV {
+ private static final Logger M4_CHUNK_METADATA = LoggerFactory.getLogger("M4_CHUNK_METADATA");
private ChunkMetadata chunkMetadata; // fixed info, including version, dataType, stepRegress
public int modelPointsCursor =
@@ -377,12 +381,13 @@ public class ChunkSuit4CPV {
// TODO debug
try {
long tmp = pageReader.timeBuffer.getLong(estimatedPos * 8);
- } catch (IndexOutOfBoundsException e) {
- System.out.println("targetTimestamp=" + targetTimestamp);
- System.out.println("estimatedPos=" + estimatedPos);
- System.out.println("count=" + chunkMetadata.getStatistics().getCount());
- System.out.println("stepregress segmentKeys=" + stepRegress.getSegmentKeys());
- System.out.println("stepregress slope=" + stepRegress.getSlope());
+ } catch (Exception e) {
+ M4_CHUNK_METADATA.info("targetTimestamp=" + targetTimestamp);
+ M4_CHUNK_METADATA.info("estimatedPos=" + estimatedPos);
+ M4_CHUNK_METADATA.info("count=" + chunkMetadata.getStatistics().getCount());
+ M4_CHUNK_METADATA.info("stepregress segmentKeys=" + stepRegress.getSegmentKeys());
+ M4_CHUNK_METADATA.info("stepregress slope=" + stepRegress.getSlope());
+ throw e;
}
if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
[iotdb] 27/46: fix index out of bounds
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit dca90f4782eff0b0919072b71184943feb0324e5
Author: Lei Rui <10...@qq.com>
AuthorDate: Sun Jun 25 11:01:56 2023 +0800
fix index out of bounds
---
.../main/java/org/apache/iotdb/db/rescon/PrimitiveArrayManager.java | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/server/src/main/java/org/apache/iotdb/db/rescon/PrimitiveArrayManager.java b/server/src/main/java/org/apache/iotdb/db/rescon/PrimitiveArrayManager.java
index e595cfc6ead..4e87117cadd 100644
--- a/server/src/main/java/org/apache/iotdb/db/rescon/PrimitiveArrayManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/rescon/PrimitiveArrayManager.java
@@ -64,6 +64,10 @@ public class PrimitiveArrayManager {
/** TSDataType#serialize() -> count of allocation requests */
private static final AtomicLong[] ALLOCATION_REQUEST_COUNTS =
new AtomicLong[] {
+ new AtomicLong(0),
+ new AtomicLong(0),
+ new AtomicLong(0),
+ new AtomicLong(0),
new AtomicLong(0),
new AtomicLong(0),
new AtomicLong(0),
[iotdb] 23/46: add config use_Statistics
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 4dbc88e3f27524c4354d2f828ecb51dabdec66e1
Author: Lei Rui <10...@qq.com>
AuthorDate: Sun Jun 18 16:36:33 2023 +0800
add config use_Statistics
---
server/src/assembly/resources/conf/iotdb-engine.properties | 2 ++
.../main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java | 8 ++++++++
.../iotdb/db/query/reader/series/SeriesAggregateReader.java | 10 ++++++++++
.../java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java | 10 ++++++++++
.../org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java | 3 +++
5 files changed, 33 insertions(+)
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index 31cfbba9fb5..28ca06a1e62 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -606,6 +606,8 @@ enable_unseq_compaction=false
### Configurations for tsfile-format
####################
+use_Statistics=true
+
use_ChunkIndex=true
use_Mad=false
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index ba2b9667c79..04da72c848a 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -896,6 +896,14 @@ public class IoTDBDescriptor {
}
private void loadTsFileProps(Properties properties) {
+ TSFileDescriptor.getInstance()
+ .getConfig()
+ .setUseStatistics(
+ Boolean.parseBoolean(
+ properties.getProperty(
+ "use_Statistics",
+ Boolean.toString(
+ TSFileDescriptor.getInstance().getConfig().isUseStatistics()))));
TSFileDescriptor.getInstance()
.getConfig()
.setUseChunkIndex(
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesAggregateReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesAggregateReader.java
index 5f30838bf16..0e7b84f710a 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesAggregateReader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesAggregateReader.java
@@ -24,6 +24,7 @@ import org.apache.iotdb.db.metadata.PartialPath;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.filter.TsFileFilter;
import org.apache.iotdb.db.utils.TestOnly;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.common.BatchData;
@@ -96,6 +97,9 @@ public class SeriesAggregateReader implements IAggregateReader {
@Override
public boolean canUseCurrentFileStatistics() throws IOException {
+ if (!TSFileDescriptor.getInstance().getConfig().isUseStatistics()) {
+ return false;
+ }
Statistics fileStatistics = currentFileStatistics();
return !seriesReader.isFileOverlapped()
&& containedByTimeFilter(fileStatistics)
@@ -119,6 +123,9 @@ public class SeriesAggregateReader implements IAggregateReader {
@Override
public boolean canUseCurrentChunkStatistics() throws IOException {
+ if (!TSFileDescriptor.getInstance().getConfig().isUseStatistics()) {
+ return false;
+ }
Statistics chunkStatistics = currentChunkStatistics();
return !seriesReader.isChunkOverlapped()
&& containedByTimeFilter(chunkStatistics)
@@ -142,6 +149,9 @@ public class SeriesAggregateReader implements IAggregateReader {
@Override
public boolean canUseCurrentPageStatistics() throws IOException {
+ if (!TSFileDescriptor.getInstance().getConfig().isUseStatistics()) {
+ return false;
+ }
Statistics currentPageStatistics = currentPageStatistics();
if (currentPageStatistics == null) {
return false;
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
index b7ba5a57d53..65cc33c8c6e 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
@@ -27,6 +27,8 @@ import java.nio.charset.Charset;
/** TSFileConfig is a configure class. Every variables is public and has default value. */
public class TSFileConfig implements Serializable {
+ private boolean useStatistics = true;
+
private boolean useChunkIndex = true;
private boolean useMad = false;
@@ -151,6 +153,14 @@ public class TSFileConfig implements Serializable {
public TSFileConfig() {}
+ public boolean isUseStatistics() {
+ return useStatistics;
+ }
+
+ public void setUseStatistics(boolean useStatistics) {
+ this.useStatistics = useStatistics;
+ }
+
public boolean isUseChunkIndex() {
return useChunkIndex;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java
index 403b40b606e..c287ad85d57 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java
@@ -105,6 +105,9 @@ public class TSFileDescriptor {
Properties properties = new Properties();
try {
properties.load(inputStream);
+ conf.setUseStatistics(
+ Boolean.parseBoolean(
+ properties.getProperty("use_Statistics", Boolean.toString(conf.isUseStatistics()))));
conf.setUseChunkIndex(
Boolean.parseBoolean(
properties.getProperty("use_ChunkIndex", Boolean.toString(conf.isUseChunkIndex()))));
[iotdb] 15/46: add print
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 4bb3a52976ac84787a689d7fddbe297013447dc0
Author: Lei Rui <10...@qq.com>
AuthorDate: Wed Jun 14 20:55:23 2023 +0800
add print
---
.../iotdb/tsfile/read/common/IOMonitor2.java | 222 +++++++++++++++------
1 file changed, 157 insertions(+), 65 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
index 90bccfaf4ca..ec0d78d777b 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java
@@ -466,6 +466,7 @@ public class IOMonitor2 {
.append(",")
.append(DCP_Server_Query_Fetch_count)
.append("\n");
+
stringBuilder.append("[3]dataSetType").append(",").append(dataSetType).append("\n");
stringBuilder
.append("[3]M4_LSM_init_loadAllChunkMetadatas_ns_cnt")
@@ -509,6 +510,7 @@ public class IOMonitor2 {
.append(",")
.append(M4_LSM_TP_count)
.append("\n");
+
stringBuilder
.append("[4]DCP_A_GET_CHUNK_METADATAS_ns_cnt")
.append(",")
@@ -565,77 +567,167 @@ public class IOMonitor2 {
.append(",")
.append(SEARCH_ARRAY_c_genBPTP_count)
.append("\n");
-
- // stringBuilder
- // .append("[2]Server_Query_Execute_cnt")
- // .append(",")
- // .append(DCP_Server_Query_Execute_count)
- // .append("\n");
- // stringBuilder
- // .append("Server_Query_Fetch_cnt")
- // .append(",")
- // .append(DCP_Server_Query_Fetch_count)
- // .append("\n");
- // stringBuilder
- // .append("M4_LSM_init_loadAllChunkMetadatas_cnt")
- // .append(",")
- // .append(M4_LSM_init_loadAllChunkMetadatas_count)
- // .append("\n");
- // stringBuilder
- // .append("M4_LSM_merge_M4_time_span_cnt")
- // .append(",")
- // .append(M4_LSM_merge_M4_time_span_count)
- // .append("\n");
- // stringBuilder.append("M4_LSM_FP_cnt").append(",").append(M4_LSM_FP_count).append("\n");
- // stringBuilder.append("M4_LSM_LP_cnt").append(",").append(M4_LSM_LP_count).append("\n");
- // stringBuilder.append("M4_LSM_BP_cnt").append(",").append(M4_LSM_BP_count).append("\n");
- // stringBuilder.append("M4_LSM_TP_cnt").append(",").append(M4_LSM_TP_count).append("\n");
- // stringBuilder
- // .append("DCP_A_GET_CHUNK_METADATAS_cnt")
- // .append(",")
- // .append(DCP_A_GET_CHUNK_METADATAS_count)
- // .append("\n");
- // stringBuilder
- // .append("DCP_B_READ_MEM_CHUNK_cnt")
- // .append(",")
- // .append(DCP_B_READ_MEM_CHUNK_count)
- // .append("\n");
- // stringBuilder
- // .append("DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_cnt")
- // .append(",")
- // .append(DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_count)
- // .append("\n");
- // stringBuilder
- // .append("DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_cnt")
- // .append(",")
- // .append(DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_count)
- // .append("\n");
- // stringBuilder
- // .append("SEARCH_ARRAY_a_verifBPTP_cnt")
- // .append(",")
- // .append(SEARCH_ARRAY_a_verifBPTP_count)
- // .append("\n");
- // stringBuilder
- // .append("SEARCH_ARRAY_b_genFP_cnt")
- // .append(",")
- // .append(SEARCH_ARRAY_b_genFP_count)
- // .append("\n");
- // stringBuilder
- // .append("SEARCH_ARRAY_b_genLP_cnt")
- // .append(",")
- // .append(SEARCH_ARRAY_b_genLP_count)
- // .append("\n");
- // stringBuilder
- // .append("SEARCH_ARRAY_c_genBPTP_cnt")
- // .append(",")
- // .append(SEARCH_ARRAY_c_genBPTP_count)
- // .append("\n");
stringBuilder
.append("[4]DCP_D_traversedPointNum")
.append(",")
.append(DCP_D_traversedPointNum)
.append("\n");
+ stringBuilder
+ .append("[3-4]M4_LSM_merge_M4_time_span_B_READ_MEM_CHUNK_cnt")
+ .append(",")
+ .append(M4_LSM_merge_M4_time_span_B_READ_MEM_CHUNK_cnt)
+ .append("\n"); // 5*6
+ stringBuilder
+ .append("[3-4]M4_LSM_merge_M4_time_span_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_cnt")
+ .append(",")
+ .append(M4_LSM_merge_M4_time_span_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_cnt)
+ .append("\n");
+ stringBuilder
+ .append("[3-4]M4_LSM_merge_M4_time_span_SEARCH_ARRAY_a_verifBPTP_cnt")
+ .append(",")
+ .append(M4_LSM_merge_M4_time_span_SEARCH_ARRAY_a_verifBPTP_cnt)
+ .append("\n");
+ stringBuilder
+ .append("[3-4]M4_LSM_merge_M4_time_span_SEARCH_ARRAY_b_genFP_cnt")
+ .append(",")
+ .append(M4_LSM_merge_M4_time_span_SEARCH_ARRAY_b_genFP_cnt)
+ .append("\n");
+ stringBuilder
+ .append("[3-4]M4_LSM_merge_M4_time_span_SEARCH_ARRAY_b_genLP_cnt")
+ .append(",")
+ .append(M4_LSM_merge_M4_time_span_SEARCH_ARRAY_b_genLP_cnt)
+ .append("\n");
+ stringBuilder
+ .append("[3-4]M4_LSM_merge_M4_time_span_SEARCH_ARRAY_c_genBPTP_cnt")
+ .append(",")
+ .append(M4_LSM_merge_M4_time_span_SEARCH_ARRAY_c_genBPTP_cnt)
+ .append("\n");
+
+ stringBuilder
+ .append("[3-4]M4_LSM_FP_B_READ_MEM_CHUNK_cnt")
+ .append(",")
+ .append(M4_LSM_FP_B_READ_MEM_CHUNK_cnt)
+ .append("\n"); // 5*6
+ stringBuilder
+ .append("[3-4]M4_LSM_FP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_cnt")
+ .append(",")
+ .append(M4_LSM_FP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_cnt)
+ .append("\n");
+ stringBuilder
+ .append("[3-4]M4_LSM_FP_SEARCH_ARRAY_a_verifBPTP_cnt")
+ .append(",")
+ .append(M4_LSM_FP_SEARCH_ARRAY_a_verifBPTP_cnt)
+ .append("\n");
+ stringBuilder
+ .append("[3-4]M4_LSM_FP_SEARCH_ARRAY_b_genFP_cnt")
+ .append(",")
+ .append(M4_LSM_FP_SEARCH_ARRAY_b_genFP_cnt)
+ .append("\n");
+ stringBuilder
+ .append("[3-4]M4_LSM_FP_SEARCH_ARRAY_b_genLP_cnt")
+ .append(",")
+ .append(M4_LSM_FP_SEARCH_ARRAY_b_genLP_cnt)
+ .append("\n");
+ stringBuilder
+ .append("[3-4]M4_LSM_FP_SEARCH_ARRAY_c_genBPTP_cnt")
+ .append(",")
+ .append(M4_LSM_FP_SEARCH_ARRAY_c_genBPTP_cnt)
+ .append("\n");
+
+ stringBuilder
+ .append("[3-4]M4_LSM_LP_B_READ_MEM_CHUNK_cnt")
+ .append(",")
+ .append(M4_LSM_LP_B_READ_MEM_CHUNK_cnt)
+ .append("\n"); // 5*6
+ stringBuilder
+ .append("[3-4]M4_LSM_LP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_cnt")
+ .append(",")
+ .append(M4_LSM_LP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_cnt)
+ .append("\n");
+ stringBuilder
+ .append("[3-4]M4_LSM_LP_SEARCH_ARRAY_a_verifBPTP_cnt")
+ .append(",")
+ .append(M4_LSM_LP_SEARCH_ARRAY_a_verifBPTP_cnt)
+ .append("\n");
+ stringBuilder
+ .append("[3-4]M4_LSM_LP_SEARCH_ARRAY_b_genFP_cnt")
+ .append(",")
+ .append(M4_LSM_LP_SEARCH_ARRAY_b_genFP_cnt)
+ .append("\n");
+ stringBuilder
+ .append("[3-4]M4_LSM_LP_SEARCH_ARRAY_b_genLP_cnt")
+ .append(",")
+ .append(M4_LSM_LP_SEARCH_ARRAY_b_genLP_cnt)
+ .append("\n");
+ stringBuilder
+ .append("[3-4]M4_LSM_LP_SEARCH_ARRAY_c_genBPTP_cnt")
+ .append(",")
+ .append(M4_LSM_LP_SEARCH_ARRAY_c_genBPTP_cnt)
+ .append("\n");
+
+ stringBuilder
+ .append("[3-4]M4_LSM_BP_B_READ_MEM_CHUNK_cnt")
+ .append(",")
+ .append(M4_LSM_BP_B_READ_MEM_CHUNK_cnt)
+ .append("\n"); // 5*6
+ stringBuilder
+ .append("[3-4]M4_LSM_BP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_cnt")
+ .append(",")
+ .append(M4_LSM_BP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_cnt)
+ .append("\n");
+ stringBuilder
+ .append("[3-4]M4_LSM_BP_SEARCH_ARRAY_a_verifBPTP_cnt")
+ .append(",")
+ .append(M4_LSM_BP_SEARCH_ARRAY_a_verifBPTP_cnt)
+ .append("\n");
+ stringBuilder
+ .append("[3-4]M4_LSM_BP_SEARCH_ARRAY_b_genFP_cnt")
+ .append(",")
+ .append(M4_LSM_BP_SEARCH_ARRAY_b_genFP_cnt)
+ .append("\n");
+ stringBuilder
+ .append("[3-4]M4_LSM_BP_SEARCH_ARRAY_b_genLP_cnt")
+ .append(",")
+ .append(M4_LSM_BP_SEARCH_ARRAY_b_genLP_cnt)
+ .append("\n");
+ stringBuilder
+ .append("[3-4]M4_LSM_BP_SEARCH_ARRAY_c_genBPTP_cnt")
+ .append(",")
+ .append(M4_LSM_BP_SEARCH_ARRAY_c_genBPTP_cnt)
+ .append("\n");
+
+ stringBuilder
+ .append("[3-4]M4_LSM_TP_B_READ_MEM_CHUNK_cnt")
+ .append(",")
+ .append(M4_LSM_TP_B_READ_MEM_CHUNK_cnt)
+ .append("\n"); // 5*6
+ stringBuilder
+ .append("[3-4]M4_LSM_TP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_cnt")
+ .append(",")
+ .append(M4_LSM_TP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_cnt)
+ .append("\n");
+ stringBuilder
+ .append("[3-4]M4_LSM_TP_SEARCH_ARRAY_a_verifBPTP_cnt")
+ .append(",")
+ .append(M4_LSM_TP_SEARCH_ARRAY_a_verifBPTP_cnt)
+ .append("\n");
+ stringBuilder
+ .append("[3-4]M4_LSM_TP_SEARCH_ARRAY_b_genFP_cnt")
+ .append(",")
+ .append(M4_LSM_TP_SEARCH_ARRAY_b_genFP_cnt)
+ .append("\n");
+ stringBuilder
+ .append("[3-4]M4_LSM_TP_SEARCH_ARRAY_b_genLP_cnt")
+ .append(",")
+ .append(M4_LSM_TP_SEARCH_ARRAY_b_genLP_cnt)
+ .append("\n");
+ stringBuilder
+ .append("[3-4]M4_LSM_TP_SEARCH_ARRAY_c_genBPTP_cnt")
+ .append(",")
+ .append(M4_LSM_TP_SEARCH_ARRAY_c_genBPTP_cnt)
+ .append("\n");
+
reset(); // whenever print() is called, reset the metrics, to clean warm up information.
return stringBuilder.toString();
}
[iotdb] 31/46: add tests, confirm the best e
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 3aa5d8d7cc0789366c1534abfc213d7f0ff1c76b
Author: Lei Rui <10...@qq.com>
AuthorDate: Tue Jul 4 10:49:21 2023 +0800
add tests, confirm the best e
---
.../iotdb/tsfile/encoding/SDTEncoderTest.java | 320 ++++++++++++---------
1 file changed, 181 insertions(+), 139 deletions(-)
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SDTEncoderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SDTEncoderTest.java
index c33ee296425..0a21c23f605 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SDTEncoderTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SDTEncoderTest.java
@@ -27,6 +27,7 @@ import org.junit.Test;
import java.io.BufferedReader;
import java.io.FileReader;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
@@ -230,155 +231,196 @@ public class SDTEncoderTest {
@Test
public void mytest2() throws Exception {
+ // when e=2*std=160000 is the best
String csvData = "D:\\full-game\\BallSpeed.csv";
- int start = 200000;
- int range = 100000;
- int end = start + range;
- SDTEncoder encoder = new SDTEncoder();
- double e = 100000; // std/2
- encoder.setCompDeviation(e / 2);
- long count = 0;
- String line;
- List<Integer> timestampList = new ArrayList<>(); // idx
- List<Long> valueList = new ArrayList<>();
- List<Integer> selectTimestamps = new ArrayList<>();
- List<Long> selectValues = new ArrayList<>();
- List<ValuePoint> points = new ArrayList<>();
- int idx = 0;
- try (BufferedReader reader = new BufferedReader(new FileReader(csvData))) {
- while ((line = reader.readLine()) != null) {
- count++;
- if (count >= start && count < end) {
- idx++;
- int time = idx; // not real timestamp, but relative position id
- long value = Long.parseLong(line.split(",")[1]);
- timestampList.add(time);
- valueList.add(value);
- points.add(new ValuePoint(time, value));
- if (encoder.encodeLong(time, value)) {
- selectTimestamps.add((int) encoder.getTime());
- selectValues.add(encoder.getLongValue());
+ double[] eList = new double[] {500000, 400000, 300000, 200000, 160000, 100000, 50000, 10000};
+ int[] startList = new int[] {1, 200000, 300000, 400000, 500000, 600000, 700000, 800000, 900000};
+ List<Double> elapsedTime_withValueIndex_list = new ArrayList<>();
+ List<Double> elapsedTime_withoutValueIndex_list = new ArrayList<>();
+ List<Double> traversedComplexity_list = new ArrayList<>();
+ List<Double> std_list = new ArrayList<>();
+ for (double e : eList) {
+ double elapsedTime_withValueIndex = 0;
+ double elapsedTime_withoutValueIndex = 0;
+ double traversedComplexity = 0;
+ for (int start : startList) {
+ int range = 50000;
+ int end = start + range;
+ SDTEncoder encoder = new SDTEncoder();
+ encoder.setCompDeviation(e / 2);
+ long count = 0;
+ String line;
+ List<Integer> timestampList = new ArrayList<>(); // idx
+ List<Long> valueList = new ArrayList<>();
+ List<Integer> selectTimestamps = new ArrayList<>();
+ List<Long> selectValues = new ArrayList<>();
+ List<ValuePoint> points = new ArrayList<>();
+ int idx = 0;
+ try (BufferedReader reader = new BufferedReader(new FileReader(csvData))) {
+ long cnt = 0;
+ double sumX2 = 0.0;
+ double sumX1 = 0.0;
+ while ((line = reader.readLine()) != null) {
+ count++;
+ if (count >= start && count < end) {
+ idx++;
+ int time = idx; // not real timestamp, but relative position id
+ long value = Long.parseLong(line.split(",")[1]);
+
+ cnt++;
+ sumX1 += value;
+ sumX2 += value * value;
+
+ timestampList.add(time);
+ valueList.add(value);
+ points.add(new ValuePoint(time, value));
+ if (encoder.encodeLong(time, value)) {
+ selectTimestamps.add((int) encoder.getTime());
+ selectValues.add(encoder.getLongValue());
+ }
+ if (count == end - 1) { // last point
+ selectTimestamps.add(time);
+ selectValues.add(value);
+ }
+ } else if (count >= end) {
+ break;
+ }
}
- if (count == end - 1) { // last point
- selectTimestamps.add(time);
- selectValues.add(value);
- }
- } else if (count >= end) {
- break;
+ double std = Math.sqrt(sumX2 / cnt - Math.pow(sumX1 / cnt, 2));
+ std_list.add(Math.sqrt(Math.pow(std, 2) * cnt / (cnt - 1)));
}
- }
- }
- // System.out.println("e=" + e);
- // System.out.println("t=" + timestampList + ";");
- // System.out.println("v=" + valueList + ";");
- // System.out.println("at=" + selectTimestamps + ";");
- // System.out.println("av=" + selectValues + ";");
- System.out.println("v.size=" + valueList.size());
- System.out.println("av.size=" + selectTimestamps.size());
-
- System.out.println("start test------");
-
- // ValuePoint[] myArray = (ValuePoint[]) points.toArray();
- // Arrays.sort(points.toArray());
- points.sort(
- new Comparator<ValuePoint>() {
- public int compare(ValuePoint o1, ValuePoint o2) {
- return ((Comparable) (o1.value)).compareTo(o2.value);
+ // System.out.println("e=" + e);
+ // System.out.println("t=" + timestampList + ";");
+ // System.out.println("v=" + valueList + ";");
+ // System.out.println("at=" + selectTimestamps + ";");
+ // System.out.println("av=" + selectValues + ";");
+ // System.out.println("v.size=" + valueList.size());
+ // System.out.println("av.size=" + selectTimestamps.size());
+
+ // System.out.println("start test------");
+
+ // ValuePoint[] myArray = (ValuePoint[]) points.toArray();
+ // Arrays.sort(points.toArray());
+ points.sort(
+ new Comparator<ValuePoint>() {
+ public int compare(ValuePoint o1, ValuePoint o2) {
+ return ((Comparable) (o1.value)).compareTo(o2.value);
+ }
+ });
+
+ long startTime = System.nanoTime();
+
+ // 计算maxLB
+ traversedComplexity += selectValues.size();
+ long maxVal = points.get(points.size() - 2).value;
+ double threshold = maxVal - e; // maxLB
+ // System.out.println("threshold(maxLB)=" + threshold);
+
+ // 计算UB<maxLB的剪枝段
+ traversedComplexity += selectValues.size();
+ List<Integer> prune_intervals_start = new ArrayList<>();
+ List<Integer> prune_intervals_end = new ArrayList<>();
+ int interval_start = -1;
+ int interval_end = -1;
+ for (int i = 1; i < selectTimestamps.size(); i++) {
+ int t1 = selectTimestamps.get(i - 1);
+ int t2 = selectTimestamps.get(i);
+ double v1 = selectValues.get(i - 1) + e; // UB
+ double v2 = selectValues.get(i) + e; // UB
+ if (v1 < threshold && v2 < threshold) {
+ if (interval_start < 0) {
+ interval_start = t1;
+ }
+ interval_end = t2; // continuous
+ } else if (v1 < threshold && v2 >= threshold) {
+ if (interval_start < 0) {
+ interval_start = t1;
+ }
+ prune_intervals_start.add(interval_start);
+ prune_intervals_end.add(
+ (int) Math.floor((threshold - v1) * (t2 - t1) / (v2 - v1) + t1));
+ interval_start = -1; // discontinuous
+ } else if (v1 >= threshold && v2 < threshold) {
+ interval_start = (int) Math.ceil((threshold - v1) * (t2 - t1) / (v2 - v1) + t1);
+ interval_end = t2; // continuous
}
- });
-
- long startTime = System.nanoTime();
-
- // 计算maxLB
- long maxVal = points.get(points.size() - 1).value;
- double threshold = maxVal - e; // maxLB
- // System.out.println("threshold(maxLB)=" + threshold);
-
- // 计算UB<maxLB的剪枝段
- List<Integer> prune_intervals_start = new ArrayList<>();
- List<Integer> prune_intervals_end = new ArrayList<>();
- int interval_start = -1;
- int interval_end = -1;
- for (int i = 1; i < selectTimestamps.size(); i++) {
- int t1 = selectTimestamps.get(i - 1);
- int t2 = selectTimestamps.get(i);
- double v1 = selectValues.get(i - 1) + e; // UB
- double v2 = selectValues.get(i) + e; // UB
- if (v1 < threshold && v2 < threshold) {
- if (interval_start < 0) {
- interval_start = t1;
}
- interval_end = t2; // continuous
- } else if (v1 < threshold && v2 >= threshold) {
- if (interval_start < 0) {
- interval_start = t1;
+ if (interval_start > 0) {
+ prune_intervals_start.add(interval_start);
+ prune_intervals_end.add(interval_end);
}
- prune_intervals_start.add(interval_start);
- prune_intervals_end.add((int) Math.floor((threshold - v1) * (t2 - t1) / (v2 - v1) + t1));
- interval_start = -1; // discontinuous
- } else if (v1 >= threshold && v2 < threshold) {
- interval_start = (int) Math.ceil((threshold - v1) * (t2 - t1) / (v2 - v1) + t1);
- interval_end = t2; // continuous
- }
- }
- if (interval_start > 0) {
- prune_intervals_start.add(interval_start);
- prune_intervals_end.add(interval_end);
- }
- // System.out.println("UB<maxLB prune intervals (included ends, starting from 1):");
- // System.out.println(prune_intervals_start);
- // System.out.println(prune_intervals_end);
-
- // 对剪枝段取余后搜索遍历计算TP
- int startIdx = 1;
- int endIdx = range;
- startIdx = startIdx - 1; // as excluded treated
- endIdx = endIdx + 1; // as excluded treated
- prune_intervals_start.add(endIdx); // turn into search_intervals_end (excluded)
- prune_intervals_end.add(0, startIdx); // turn into search_intervals_start (excluded)
- long candidateTPvalue = -1;
- int candidateTPidx = -1;
- for (int i = 0; i < prune_intervals_start.size(); i++) {
- int search_interval_start = prune_intervals_end.get(i) + 1; // included
- int search_interval_end = prune_intervals_start.get(i) - 1; // included
- // System.out.println(search_interval_start + "," + search_interval_end);
- for (int j = search_interval_start; j <= search_interval_end; j++) { // starting from 1
- long v = valueList.get(j - 1); // java list starting from 0
- if (v > candidateTPvalue) {
- candidateTPvalue = v;
- candidateTPidx = j;
+ // System.out.println("UB<maxLB prune intervals (included ends, starting from 1):");
+ // System.out.println(prune_intervals_start);
+ // System.out.println(prune_intervals_end);
+
+ // 对剪枝段取余后搜索遍历计算TP
+ int startIdx = 1;
+ int endIdx = range;
+ startIdx = startIdx - 1; // as excluded treated
+ endIdx = endIdx + 1; // as excluded treated
+ prune_intervals_start.add(endIdx); // turn into search_intervals_end (excluded)
+ prune_intervals_end.add(0, startIdx); // turn into search_intervals_start (excluded)
+ long candidateTPvalue = -1;
+ int candidateTPidx = -1;
+ for (int i = 0; i < prune_intervals_start.size(); i++) {
+ int search_interval_start = prune_intervals_end.get(i) + 1; // included
+ int search_interval_end = prune_intervals_start.get(i) - 1; // included
+ // System.out.println(search_interval_start + "," + search_interval_end);
+ for (int j = search_interval_start; j <= search_interval_end; j++) { // starting from 1
+ long v = valueList.get(j - 1); // java list starting from 0
+ if (v > candidateTPvalue) {
+ candidateTPvalue = v;
+ candidateTPidx = j;
+ }
+ }
}
+ long elapsedTime = System.nanoTime() - startTime;
+ elapsedTime_withValueIndex += elapsedTime;
+ // System.out.println("search with value index: " + elapsedTime / 1000000.0 + " ms");
+ // System.out.println("TP=(" + candidateTPidx + "," + candidateTPvalue + ")");
+ // System.out.println("search interval number=" + prune_intervals_end.size());
+ int traversedPoints = 0;
+ for (int i = 0; i < prune_intervals_start.size(); i++) {
+ int search_interval_start = prune_intervals_end.get(i) + 1; // included
+ int search_interval_end = prune_intervals_start.get(i) - 1; // included
+ if (search_interval_end >= search_interval_start) {
+ traversedPoints += (search_interval_end - search_interval_start + 1);
+ }
+ }
+ traversedComplexity += traversedPoints;
+ // System.out.println("number of traversed points: " + traversedPoints);
+
+ // System.out.println("start test------");
+ startTime = System.nanoTime();
+ long candidateTPvalue_raw = -1;
+ int candidateTPidx_raw = -1;
+ for (int j = 1; j <= valueList.size(); j++) { // starting from 1
+ long v = valueList.get(j - 1); // java list starting from 0
+ if (v > candidateTPvalue_raw) {
+ candidateTPvalue_raw = v;
+ candidateTPidx_raw = j;
+ }
+ }
+ elapsedTime = System.nanoTime() - startTime;
+ elapsedTime_withoutValueIndex += elapsedTime;
+ // System.out.println("search without value index: " + elapsedTime / 1000000.0 + "
+ // ms");
+ // System.out.println("TP=(" + candidateTPidx_raw + "," + candidateTPvalue_raw +
+ // ")");
}
+ elapsedTime_withValueIndex_list.add(
+ elapsedTime_withValueIndex / startList.length / 1000000.0);
+ elapsedTime_withoutValueIndex_list.add(
+ elapsedTime_withoutValueIndex / startList.length / 1000000.0);
+ traversedComplexity_list.add(traversedComplexity / startList.length);
}
- long elapsedTime = System.nanoTime() - startTime;
- System.out.println("search with value index: " + elapsedTime / 1000000.0 + " m s");
- System.out.println("TP=(" + candidateTPidx + "," + candidateTPvalue + ")");
- System.out.println("search interval number=" + prune_intervals_end.size());
- int traversedPoints = 0;
- for (int i = 0; i < prune_intervals_start.size(); i++) {
- int search_interval_start = prune_intervals_end.get(i) + 1; // included
- int search_interval_end = prune_intervals_start.get(i) - 1; // included
- if (search_interval_end >= search_interval_start) {
- traversedPoints += (search_interval_end - search_interval_start + 1);
- }
- }
- System.out.println("number of traversed points: " + traversedPoints);
-
- System.out.println("start test------");
- startTime = System.nanoTime();
- long candidateTPvalue_raw = -1;
- int candidateTPidx_raw = -1;
- for (int j = 1; j <= valueList.size(); j++) { // starting from 1
- long v = valueList.get(j - 1); // java list starting from 0
- if (v > candidateTPvalue_raw) {
- candidateTPvalue_raw = v;
- candidateTPidx_raw = j;
- }
- }
- elapsedTime = System.nanoTime() - startTime;
- System.out.println("search without value index: " + elapsedTime / 1000000.0 + " m s");
- System.out.println("TP=(" + candidateTPidx_raw + "," + candidateTPvalue_raw + ")");
+ System.out.println("=================");
+ System.out.println("eList: " + Arrays.toString(eList));
+ System.out.println("time with index: " + elapsedTime_withValueIndex_list);
+ System.out.println("time without index: " + elapsedTime_withoutValueIndex_list);
+ System.out.println("traversed points: " + traversedComplexity_list);
+ System.out.println("std: " + std_list);
}
@Test
[iotdb] 19/46: fix python-iotdb-connector with MIN_MAX self-defined datatype
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5d1b38cf49ad89032459322223776b5b9617ea62
Author: Lei Rui <10...@qq.com>
AuthorDate: Fri Jun 16 15:56:57 2023 +0800
fix python-iotdb-connector with MIN_MAX self-defined datatype
---
client-py/iotdb/utils/Field.py | 24 ++++++++++++++++++++----
client-py/iotdb/utils/IoTDBConstants.py | 4 ++++
client-py/iotdb/utils/IoTDBRpcDataSet.py | 6 +++++-
client-py/iotdb/utils/SessionDataSet.py | 10 +++++++++-
4 files changed, 38 insertions(+), 6 deletions(-)
diff --git a/client-py/iotdb/utils/Field.py b/client-py/iotdb/utils/Field.py
index 0756b1c49d6..add1b2c5398 100644
--- a/client-py/iotdb/utils/Field.py
+++ b/client-py/iotdb/utils/Field.py
@@ -47,7 +47,11 @@ class Field(object):
output.set_float_value(field.get_float_value())
elif output.get_data_type() == TSDataType.DOUBLE:
output.set_double_value(field.get_double_value())
- elif output.get_data_type() == TSDataType.TEXT:
+ elif output.get_data_type() == TSDataType.TEXT \
+ or output.get_data_type() == TSDataType.MIN_MAX_DOUBLE \
+ or output.get_data_type() == TSDataType.MIN_MAX_FLOAT \
+ or output.get_data_type() == TSDataType.MIN_MAX_INT32 \
+ or output.get_data_type() == TSDataType.MIN_MAX_INT64:
output.set_binary_value(field.get_binary_value())
else:
raise Exception(
@@ -122,7 +126,11 @@ class Field(object):
return str(self.__float_value)
elif self.__data_type == TSDataType.DOUBLE:
return str(self.__double_value)
- elif self.__data_type == TSDataType.TEXT:
+ elif self.__data_type == TSDataType.TEXT \
+ or self.__data_type == TSDataType.MIN_MAX_DOUBLE \
+ or self.__data_type == TSDataType.MIN_MAX_FLOAT \
+ or self.__data_type == TSDataType.MIN_MAX_INT32 \
+ or self.__data_type == TSDataType.MIN_MAX_INT64:
return self.__binary_value.decode("utf-8")
else:
raise Exception("unsupported data type {}".format(self.__data_type))
@@ -146,7 +154,11 @@ class Field(object):
return self.get_float_value()
elif data_type == TSDataType.DOUBLE:
return self.get_double_value()
- elif data_type == TSDataType.TEXT:
+ elif data_type == TSDataType.TEXT \
+ or data_type == TSDataType.MIN_MAX_DOUBLE \
+ or data_type == TSDataType.MIN_MAX_FLOAT \
+ or data_type == TSDataType.MIN_MAX_INT32 \
+ or data_type == TSDataType.MIN_MAX_INT64:
return self.get_binary_value()
else:
raise Exception("unsupported data type {}".format(data_type))
@@ -170,7 +182,11 @@ class Field(object):
field.set_float_value(value)
elif data_type == TSDataType.DOUBLE:
field.set_double_value(value)
- elif data_type == TSDataType.TEXT:
+ elif data_type == TSDataType.TEXT \
+ or data_type == TSDataType.MIN_MAX_DOUBLE \
+ or data_type == TSDataType.MIN_MAX_FLOAT \
+ or data_type == TSDataType.MIN_MAX_INT32 \
+ or data_type == TSDataType.MIN_MAX_INT64:
field.set_binary_value(value)
else:
raise Exception("unsupported data type {}".format(data_type))
diff --git a/client-py/iotdb/utils/IoTDBConstants.py b/client-py/iotdb/utils/IoTDBConstants.py
index 9a1a1564668..849fc5a2865 100644
--- a/client-py/iotdb/utils/IoTDBConstants.py
+++ b/client-py/iotdb/utils/IoTDBConstants.py
@@ -27,6 +27,10 @@ class TSDataType(Enum):
FLOAT = 3
DOUBLE = 4
TEXT = 5
+ MIN_MAX_INT32 = 6
+ MIN_MAX_INT64 = 7
+ MIN_MAX_FLOAT = 8
+ MIN_MAX_DOUBLE = 9
# this method is implemented to avoid the issue reported by:
# https://bugs.python.org/issue30545
diff --git a/client-py/iotdb/utils/IoTDBRpcDataSet.py b/client-py/iotdb/utils/IoTDBRpcDataSet.py
index 6520a048d78..1f72c6510de 100644
--- a/client-py/iotdb/utils/IoTDBRpcDataSet.py
+++ b/client-py/iotdb/utils/IoTDBRpcDataSet.py
@@ -173,7 +173,11 @@ class IoTDBRpcDataSet(object):
elif data_type == TSDataType.DOUBLE:
self.__value[i] = value_buffer[:8]
self.__query_data_set.valueList[i] = value_buffer[8:]
- elif data_type == TSDataType.TEXT:
+ elif data_type == TSDataType.TEXT \
+ or data_type == TSDataType.MIN_MAX_DOUBLE \
+ or data_type == TSDataType.MIN_MAX_FLOAT \
+ or data_type == TSDataType.MIN_MAX_INT32 \
+ or data_type == TSDataType.MIN_MAX_INT64:
length = int.from_bytes(
value_buffer[:4], byteorder="big", signed=False
)
diff --git a/client-py/iotdb/utils/SessionDataSet.py b/client-py/iotdb/utils/SessionDataSet.py
index f8946cd3c07..1b3494d2b03 100644
--- a/client-py/iotdb/utils/SessionDataSet.py
+++ b/client-py/iotdb/utils/SessionDataSet.py
@@ -116,7 +116,11 @@ class SessionDataSet(object):
elif data_type == TSDataType.DOUBLE:
value = struct.unpack(">d", value_bytes)[0]
field.set_double_value(value)
- elif data_type == TSDataType.TEXT:
+ elif data_type == TSDataType.TEXT \
+ or data_type == TSDataType.MIN_MAX_DOUBLE \
+ or data_type == TSDataType.MIN_MAX_FLOAT \
+ or data_type == TSDataType.MIN_MAX_INT32 \
+ or data_type == TSDataType.MIN_MAX_INT64:
field.set_binary_value(value_bytes)
else:
raise RuntimeError("unsupported data type {}.".format(data_type))
@@ -172,6 +176,10 @@ def get_typed_point(field: Field, none_value=None):
TSDataType.INT32: lambda field: field.get_int_value(),
TSDataType.DOUBLE: lambda field: field.get_double_value(),
TSDataType.INT64: lambda field: field.get_long_value(),
+ TSDataType.MIN_MAX_DOUBLE: lambda field: field.get_string_value(),
+ TSDataType.MIN_MAX_FLOAT: lambda field: field.get_string_value(),
+ TSDataType.MIN_MAX_INT32: lambda field: field.get_string_value(),
+ TSDataType.MIN_MAX_INT64: lambda field: field.get_string_value(),
}
result_next_type: TSDataType = field.get_data_type()
[iotdb] 02/46: improve sketch tool
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 45a63a8139e29173e3fc243a13cbd98e37f26617
Author: Lei Rui <10...@qq.com>
AuthorDate: Sun Jun 11 15:55:53 2023 +0800
improve sketch tool
---
.../apache/iotdb/db/tools/TsFileSketchTool.java | 89 ++++++++++++++++++----
1 file changed, 76 insertions(+), 13 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java b/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java
index c416e5690a0..6cdb37b8ab5 100644
--- a/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java
+++ b/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java
@@ -22,6 +22,7 @@ package org.apache.iotdb.db.tools;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.file.MetaMarker;
import org.apache.iotdb.tsfile.file.header.ChunkGroupHeader;
+import org.apache.iotdb.tsfile.file.header.PageHeader;
import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetadata;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.MetadataIndexEntry;
@@ -37,6 +38,7 @@ import org.apache.iotdb.tsfile.utils.Pair;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
+import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -100,26 +102,87 @@ public class TsFileSketchTool {
// chunk begins
for (ChunkMetadata chunkMetadata : chunkGroupMetadata.getChunkMetadataList()) {
Chunk chunk = reader.readMemChunk(chunkMetadata);
+ // printlnBoth(
+ // pw,
+ // String.format("%20d", chunkMetadata.getOffsetOfChunkHeader())
+ // + "|\t[Chunk] of "
+ // + chunkMetadata.getMeasurementUid()
+ // + ", numOfPoints:"
+ // + chunkMetadata.getNumOfPoints()
+ // + ", time range:["
+ // + chunkMetadata.getStartTime()
+ // + ","
+ // + chunkMetadata.getEndTime()
+ // + "], tsDataType:"
+ // + chunkMetadata.getDataType()
+ // + ", \n"
+ // + String.format("%20s", "")
+ // + " \t"
+ // + chunkMetadata.getStatistics());
+ // printlnBoth(
+ // pw,
+ // String.format("%20s", "") + "|\t\t[marker] " +
+ // chunk.getHeader().getChunkType());
printlnBoth(
pw,
String.format("%20d", chunkMetadata.getOffsetOfChunkHeader())
+ "|\t[Chunk] of "
- + chunkMetadata.getMeasurementUid()
- + ", numOfPoints:"
- + chunkMetadata.getNumOfPoints()
- + ", time range:["
- + chunkMetadata.getStartTime()
- + ","
- + chunkMetadata.getEndTime()
- + "], tsDataType:"
- + chunkMetadata.getDataType()
- + ", \n"
- + String.format("%20s", "")
- + " \t"
+ + new Path(chunkGroupHeader.getDeviceID(), chunkMetadata.getMeasurementUid())
+ + ", "
+ chunkMetadata.getStatistics());
printlnBoth(
pw,
- String.format("%20s", "") + "|\t\t[marker] " + chunk.getHeader().getChunkType());
+ String.format("%20s", "")
+ + "|\t\t[chunk header] "
+ + "marker="
+ + chunk.getHeader().getChunkType()
+ + ", measurementID="
+ + chunk.getHeader().getMeasurementID()
+ + ", dataSize="
+ + chunk.getHeader().getDataSize()
+ + ", dataType="
+ + chunk.getHeader().getDataType()
+ + ", compressionType="
+ + chunk.getHeader().getCompressionType()
+ + ", encodingType="
+ + chunk.getHeader().getEncodingType());
+ PageHeader pageHeader;
+ if (((byte) (chunk.getHeader().getChunkType() & 0x3F))
+ == MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER) {
+ pageHeader =
+ PageHeader.deserializeFrom(chunk.getData(), chunkMetadata.getStatistics());
+ printlnBoth(
+ pw,
+ String.format("%20s", "")
+ + "|\t\t[page] "
+ + " UncompressedSize:"
+ + pageHeader.getUncompressedSize()
+ + ", CompressedSize:"
+ + pageHeader.getCompressedSize());
+ } else { // more than one page in this chunk
+ ByteBuffer chunkDataBuffer = chunk.getData();
+ int pageID = 0;
+ while (chunkDataBuffer.remaining() > 0) {
+ pageID++;
+ // deserialize a PageHeader from chunkDataBuffer
+ pageHeader =
+ PageHeader.deserializeFrom(chunkDataBuffer, chunk.getHeader().getDataType());
+ // skip the compressed bytes
+ chunkDataBuffer.position(
+ chunkDataBuffer.position() + pageHeader.getCompressedSize());
+ // print page info
+ printlnBoth(
+ pw,
+ String.format("%20s", "")
+ + String.format("|\t\t[page-%s] ", pageID)
+ + " UncompressedSize:"
+ + pageHeader.getUncompressedSize()
+ + ", CompressedSize:"
+ + pageHeader.getCompressedSize()
+ + ", "
+ + pageHeader.getStatistics());
+ }
+ }
nextChunkGroupHeaderPos =
chunkMetadata.getOffsetOfChunkHeader()
+ chunk.getHeader().getSerializedSize()
[iotdb] 12/46: fix
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 76c0ff69420d9ad130d663b9ea720452bc839e4c
Author: Lei Rui <10...@qq.com>
AuthorDate: Wed Jun 14 15:09:56 2023 +0800
fix
---
.../src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java | 1 -
1 file changed, 1 deletion(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
index 67520439d22..d1396da497b 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
@@ -422,7 +422,6 @@ public class ChunkSuit4CPV {
default:
throw new IOException("Unsupported data type!");
}
- return estimatedPos;
} else {
// to find the closet timestamp equal to or smaller
// than the given timestamp