You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by le...@apache.org on 2023/04/12 23:51:11 UTC
[iotdb] branch research/M4-visualization updated: for ablation
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/research/M4-visualization by this push:
new e44d20677e for ablation
e44d20677e is described below
commit e44d20677ee74b6bb23d97448bc5bc0836f193ae
Author: Lei Rui <10...@qq.com>
AuthorDate: Thu Apr 13 07:49:09 2023 +0800
for ablation
---
.../resources/conf/iotdb-engine.properties | 2 +
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 8 +
.../apache/iotdb/db/integration/m4/MyTest1.java | 5 +
.../apache/iotdb/db/integration/m4/MyTest2.java | 5 +
.../apache/iotdb/db/integration/m4/MyTest3.java | 5 +
.../apache/iotdb/db/integration/m4/MyTest4.java | 6 +
.../iotdb/tsfile/common/conf/TSFileConfig.java | 10 +
.../iotdb/tsfile/common/conf/TSFileDescriptor.java | 3 +
.../iotdb/tsfile/read/common/ChunkSuit4CPV.java | 359 ++++++++++++++-------
9 files changed, 284 insertions(+), 119 deletions(-)
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index eb330b8b66..78c74ae658 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -606,6 +606,8 @@ enable_unseq_compaction=false
### Configurations for tsfile-format
####################
+use_ChunkIndex=true
+
# group_size_in_byte=134217728
# The memory size for each series writer to pack page, default value is 64KB
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 2963fbaf77..5b30980b89 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -896,6 +896,14 @@ public class IoTDBDescriptor {
}
private void loadTsFileProps(Properties properties) {
+ TSFileDescriptor.getInstance()
+ .getConfig()
+ .setUseChunkIndex(
+ Boolean.parseBoolean(
+ properties.getProperty(
+ "use_ChunkIndex",
+ Boolean.toString(
+ TSFileDescriptor.getInstance().getConfig().isUseChunkIndex()))));
TSFileDescriptor.getInstance()
.getConfig()
.setGroupSizeInByte(
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest1.java b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest1.java
index 92dd34d3ef..60169de2c3 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest1.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest1.java
@@ -57,6 +57,7 @@ public class MyTest1 {
private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
private static boolean originalEnableCPV;
+ private static boolean originalUseChunkIndex;
private static CompactionStrategy originalCompactionStrategy;
@Before
@@ -68,6 +69,9 @@ public class MyTest1 {
originalEnableCPV = config.isEnableCPV();
config.setEnableCPV(true); // CPV
+ originalUseChunkIndex = TSFileDescriptor.getInstance().getConfig().isUseChunkIndex();
+ TSFileDescriptor.getInstance().getConfig().setUseChunkIndex(false);
+
EnvironmentUtils.envSetUp();
Class.forName(Config.JDBC_DRIVER_NAME);
config.setTimestampPrecision("ms");
@@ -78,6 +82,7 @@ public class MyTest1 {
EnvironmentUtils.cleanEnv();
config.setCompactionStrategy(originalCompactionStrategy);
config.setEnableCPV(originalEnableCPV);
+ TSFileDescriptor.getInstance().getConfig().setUseChunkIndex(originalUseChunkIndex);
}
@Test
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest2.java b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest2.java
index eb3ca32127..c6fc8dd3e1 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest2.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest2.java
@@ -59,6 +59,7 @@ public class MyTest2 {
private static int originalAvgSeriesPointNumberThreshold;
private static long originalSeqTsFileSize;
private static long originalUnSeqTsFileSize;
+ private static boolean originalUseChunkIndex;
@Before
public void setUp() throws Exception {
@@ -72,6 +73,9 @@ public class MyTest2 {
originalSeqTsFileSize = config.getSeqTsFileSize();
originalUnSeqTsFileSize = config.getUnSeqTsFileSize();
+ originalUseChunkIndex = TSFileDescriptor.getInstance().getConfig().isUseChunkIndex();
+ TSFileDescriptor.getInstance().getConfig().setUseChunkIndex(false);
+
config.setCompactionStrategy(CompactionStrategy.NO_COMPACTION);
config.setSeqTsFileSize(1024 * 1024 * 1024); // 1G
@@ -92,6 +96,7 @@ public class MyTest2 {
config.setEnableCPV(originalEnableCPV);
config.setSeqTsFileSize(originalSeqTsFileSize);
config.setUnSeqTsFileSize(originalUnSeqTsFileSize);
+ TSFileDescriptor.getInstance().getConfig().setUseChunkIndex(originalUseChunkIndex);
}
@Test
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest3.java b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest3.java
index eb7b06b28d..ad3f99f9a7 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest3.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest3.java
@@ -57,6 +57,7 @@ public class MyTest3 {
private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
private static boolean originalEnableCPV;
private static CompactionStrategy originalCompactionStrategy;
+ private static boolean originalUseChunkIndex;
@Before
public void setUp() throws Exception {
@@ -68,6 +69,9 @@ public class MyTest3 {
// config.setEnableCPV(false); // MOC
config.setEnableCPV(true); // CPV
+ originalUseChunkIndex = TSFileDescriptor.getInstance().getConfig().isUseChunkIndex();
+ TSFileDescriptor.getInstance().getConfig().setUseChunkIndex(false);
+
EnvironmentUtils.envSetUp();
Class.forName(Config.JDBC_DRIVER_NAME);
config.setTimestampPrecision("ms");
@@ -78,6 +82,7 @@ public class MyTest3 {
EnvironmentUtils.cleanEnv();
config.setCompactionStrategy(originalCompactionStrategy);
config.setEnableCPV(originalEnableCPV);
+ TSFileDescriptor.getInstance().getConfig().setUseChunkIndex(originalUseChunkIndex);
}
@Test
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest4.java b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest4.java
index 70930a9b13..7b4bd91c41 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest4.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest4.java
@@ -58,6 +58,8 @@ public class MyTest4 {
private static boolean originalEnableCPV;
private static CompactionStrategy originalCompactionStrategy;
+ private static boolean originalUseChunkIndex;
+
@Before
public void setUp() throws Exception {
TSFileDescriptor.getInstance().getConfig().setTimeEncoder("PLAIN");
@@ -67,6 +69,9 @@ public class MyTest4 {
originalEnableCPV = config.isEnableCPV();
config.setEnableCPV(true); // CPV
+ originalUseChunkIndex = TSFileDescriptor.getInstance().getConfig().isUseChunkIndex();
+ TSFileDescriptor.getInstance().getConfig().setUseChunkIndex(false);
+
EnvironmentUtils.envSetUp();
Class.forName(Config.JDBC_DRIVER_NAME);
config.setTimestampPrecision("ms");
@@ -77,6 +82,7 @@ public class MyTest4 {
EnvironmentUtils.cleanEnv();
config.setCompactionStrategy(originalCompactionStrategy);
config.setEnableCPV(originalEnableCPV);
+ TSFileDescriptor.getInstance().getConfig().setUseChunkIndex(originalUseChunkIndex);
}
@Test
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
index 2905a63900..49b385a658 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
@@ -27,6 +27,8 @@ import java.nio.charset.Charset;
/** TSFileConfig is a configure class. Every variables is public and has default value. */
public class TSFileConfig implements Serializable {
+ private boolean useChunkIndex = true;
+
/** encoding configuration */
public static final int RLE_MIN_REPEATED_NUM = 8;
@@ -147,6 +149,14 @@ public class TSFileConfig implements Serializable {
public TSFileConfig() {}
+ public boolean isUseChunkIndex() {
+ return useChunkIndex;
+ }
+
+ public void setUseChunkIndex(boolean useChunkIndex) {
+ this.useChunkIndex = useChunkIndex;
+ }
+
public int getGroupSizeInByte() {
return groupSizeInByte;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java
index d0b6ff49fd..342be83a06 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java
@@ -105,6 +105,9 @@ public class TSFileDescriptor {
Properties properties = new Properties();
try {
properties.load(inputStream);
+ conf.setUseChunkIndex(
+ Boolean.parseBoolean(
+ properties.getProperty("use_ChunkIndex", Boolean.toString(conf.isUseChunkIndex()))));
conf.setGroupSizeInByte(
Integer.parseInt(
properties.getProperty(
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
index d61d14e193..59ba46ed2c 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.tsfile.read.common;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.statistics.DoubleStatistics;
@@ -244,61 +245,107 @@ public class ChunkSuit4CPV {
* @return the position of the point, starting from 0
*/
public int updateFPwithTheClosetPointEqualOrAfter(long targetTimestamp) throws IOException {
- StepRegress stepRegress = chunkMetadata.getStatistics().getStepRegress();
- // infer position starts from 1, so minus 1 here
- // TODO debug buffer.get(index)
- int estimatedPos = (int) Math.round(stepRegress.infer(targetTimestamp)) - 1;
-
- // search from estimatePos in the timeBuffer to find the closet timestamp equal to or larger
- // than the given timestamp
- if (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
- while (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
+ if (TSFileDescriptor.getInstance().getConfig().isUseChunkIndex()) {
+ StepRegress stepRegress = chunkMetadata.getStatistics().getStepRegress();
+ // infer position starts from 1, so minus 1 here
+ int estimatedPos = (int) Math.round(stepRegress.infer(targetTimestamp)) - 1;
+
+ // search from estimatePos in the timeBuffer to find the closet timestamp equal to or larger
+ // than the given timestamp
+ if (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
+ while (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
+ estimatedPos++;
+ IOMonitor.incPointsTravered();
+ }
+ } else if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
+ while (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
+ estimatedPos--;
+ IOMonitor.incPointsTravered();
+ }
+ if (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
+ estimatedPos++;
+ IOMonitor.incPointsTravered();
+ } // else equal
+ } // else equal
+ this.startPos = estimatedPos; // note this
+
+ // since we have constrained that targetTimestamp must be within the chunk time range
+ // [startTime, endTime],
+ // we can definitely find such a point with the closet timestamp equal to or larger than the
+ // given timestamp in the chunk.
+ long timestamp = pageReader.timeBuffer.getLong(estimatedPos * 8);
+ statistics.setStartTime(timestamp);
+ switch (chunkMetadata.getDataType()) {
+ // iotdb的int类型的plain编码用的是自制的不支持random access
+ // case INT32:
+ // return new MinMaxInfo(pageReader.valueBuffer.getInt(estimatedPos * 4),
+ // pageReader.timeBuffer.getLong(estimatedPos * 8));
+ case INT64:
+ long longVal =
+ pageReader.valueBuffer.getLong(pageReader.timeBufferLength + estimatedPos * 8);
+ ((LongStatistics) statistics).setFirstValue(longVal);
+ break;
+ case FLOAT:
+ float floatVal =
+ pageReader.valueBuffer.getFloat(pageReader.timeBufferLength + estimatedPos * 4);
+ ((FloatStatistics) statistics).setFirstValue(floatVal);
+ break;
+ case DOUBLE:
+ double doubleVal =
+ pageReader.valueBuffer.getDouble(pageReader.timeBufferLength + estimatedPos * 8);
+ ((DoubleStatistics) statistics).setFirstValue(doubleVal);
+ break;
+ default:
+ throw new IOException("Unsupported data type!");
+ }
+ return estimatedPos;
+ } else {
+ // search from estimatePos in the timeBuffer to find the closet timestamp equal to or larger
+ // than the given timestamp
+ int estimatedPos = -1;
+ pageReader.timeBuffer.position(0);
+ pageReader.valueBuffer.position(pageReader.timeBufferLength);
+ while (pageReader.timeBuffer.remaining() > 0) {
estimatedPos++;
IOMonitor.incPointsTravered();
+ long t = pageReader.timeBuffer.getLong();
+ if (t >= targetTimestamp) {
+ break;
+ }
}
- } else if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
- while (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
- estimatedPos--;
- IOMonitor.incPointsTravered();
+ this.startPos = estimatedPos; // note this
+
+ // since we have constrained that targetTimestamp must be within the chunk time range
+ // [startTime, endTime],
+ // we can definitely find such a point with the closet timestamp equal to or larger than the
+ // given timestamp in the chunk.
+ long timestamp = pageReader.timeBuffer.getLong(estimatedPos * 8);
+ statistics.setStartTime(timestamp);
+ switch (chunkMetadata.getDataType()) {
+ // iotdb的int类型的plain编码用的是自制的不支持random access
+ // case INT32:
+ // return new MinMaxInfo(pageReader.valueBuffer.getInt(estimatedPos * 4),
+ // pageReader.timeBuffer.getLong(estimatedPos * 8));
+ case INT64:
+ long longVal =
+ pageReader.valueBuffer.getLong(pageReader.timeBufferLength + estimatedPos * 8);
+ ((LongStatistics) statistics).setFirstValue(longVal);
+ break;
+ case FLOAT:
+ float floatVal =
+ pageReader.valueBuffer.getFloat(pageReader.timeBufferLength + estimatedPos * 4);
+ ((FloatStatistics) statistics).setFirstValue(floatVal);
+ break;
+ case DOUBLE:
+ double doubleVal =
+ pageReader.valueBuffer.getDouble(pageReader.timeBufferLength + estimatedPos * 8);
+ ((DoubleStatistics) statistics).setFirstValue(doubleVal);
+ break;
+ default:
+ throw new IOException("Unsupported data type!");
}
- if (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
- estimatedPos++;
- IOMonitor.incPointsTravered();
- } // else equal
- } // else equal
- this.startPos = estimatedPos; // note this
-
- // since we have constrained that targetTimestamp must be within the chunk time range
- // [startTime, endTime],
- // we can definitely find such a point with the closet timestamp equal to or larger than the
- // given timestamp in the chunk.
- long timestamp = pageReader.timeBuffer.getLong(estimatedPos * 8);
- statistics.setStartTime(timestamp);
- switch (chunkMetadata.getDataType()) {
- // iotdb的int类型的plain编码用的是自制的不支持random access
- // case INT32:
- // return new MinMaxInfo(pageReader.valueBuffer.getInt(estimatedPos * 4),
- // pageReader.timeBuffer.getLong(estimatedPos * 8));
- case INT64:
- long longVal =
- pageReader.valueBuffer.getLong(pageReader.timeBufferLength + estimatedPos * 8);
- ((LongStatistics) statistics).setFirstValue(longVal);
- break;
- case FLOAT:
- float floatVal =
- pageReader.valueBuffer.getFloat(pageReader.timeBufferLength + estimatedPos * 4);
- ((FloatStatistics) statistics).setFirstValue(floatVal);
- break;
- case DOUBLE:
- double doubleVal =
- pageReader.valueBuffer.getDouble(pageReader.timeBufferLength + estimatedPos * 8);
- ((DoubleStatistics) statistics).setFirstValue(doubleVal);
- break;
- default:
- throw new IOException("Unsupported data type!");
+ return estimatedPos;
}
-
- return estimatedPos;
}
/**
@@ -309,59 +356,112 @@ public class ChunkSuit4CPV {
* @return the position of the point, starting from 0
*/
public int updateLPwithTheClosetPointEqualOrBefore(long targetTimestamp) throws IOException {
- StepRegress stepRegress = chunkMetadata.getStatistics().getStepRegress();
- // infer position starts from 1, so minus 1 here
- int estimatedPos = (int) Math.round(stepRegress.infer(targetTimestamp)) - 1;
-
- // search from estimatePos in the timeBuffer to find the closet timestamp equal to or smaller
- // than the given timestamp
- if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
- while (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
- estimatedPos--;
- IOMonitor.incPointsTravered();
+ if (TSFileDescriptor.getInstance().getConfig().isUseChunkIndex()) {
+ StepRegress stepRegress = chunkMetadata.getStatistics().getStepRegress();
+ // infer position starts from 1, so minus 1 here
+ int estimatedPos = (int) Math.round(stepRegress.infer(targetTimestamp)) - 1;
+
+ // search from estimatePos in the timeBuffer to find the closet timestamp equal to or smaller
+ // than the given timestamp
+ if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
+ while (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
+ estimatedPos--;
+ IOMonitor.incPointsTravered();
+ }
+ } else if (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
+ while (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
+ estimatedPos++;
+ IOMonitor.incPointsTravered();
+ }
+ if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
+ estimatedPos--;
+ IOMonitor.incPointsTravered();
+ } // else equal
+ } // else equal
+ this.endPos = estimatedPos; // note this
+
+ // since we have constrained that targetTimestamp must be within the chunk time range
+ // [startTime, endTime],
+ // we can definitely find such a point with the closet timestamp equal to or smaller than the
+ // given timestamp in the chunk.
+ long timestamp = pageReader.timeBuffer.getLong(estimatedPos * 8);
+ statistics.setEndTime(timestamp);
+ switch (chunkMetadata.getDataType()) {
+ // iotdb的int类型的plain编码用的是自制的不支持random access
+ // case INT32:
+ // return new MinMaxInfo(pageReader.valueBuffer.getInt(estimatedPos * 4),
+ // pageReader.timeBuffer.getLong(estimatedPos * 8));
+ case INT64:
+ long longVal =
+ pageReader.valueBuffer.getLong(pageReader.timeBufferLength + estimatedPos * 8);
+ ((LongStatistics) statistics).setLastValue(longVal);
+ break;
+ case FLOAT:
+ float floatVal =
+ pageReader.valueBuffer.getFloat(pageReader.timeBufferLength + estimatedPos * 4);
+ ((FloatStatistics) statistics).setLastValue(floatVal);
+ break;
+ case DOUBLE:
+ double doubleVal =
+ pageReader.valueBuffer.getDouble(pageReader.timeBufferLength + estimatedPos * 8);
+ ((DoubleStatistics) statistics).setLastValue(doubleVal);
+ break;
+ default:
+ throw new IOException("Unsupported data type!");
}
- } else if (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
- while (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
+ return estimatedPos;
+ } else {
+ // to find the closet timestamp equal to or smaller
+ // than the given timestamp
+ int estimatedPos = -1;
+ pageReader.timeBuffer.position(0);
+ pageReader.valueBuffer.position(pageReader.timeBufferLength);
+ while (pageReader.timeBuffer.remaining() > 0) {
estimatedPos++;
IOMonitor.incPointsTravered();
+ long t = pageReader.timeBuffer.getLong();
+ if (t >= targetTimestamp) {
+ break;
+ }
}
if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
- estimatedPos--;
IOMonitor.incPointsTravered();
- } // else equal
- } // else equal
- this.endPos = estimatedPos; // note this
-
- // since we have constrained that targetTimestamp must be within the chunk time range
- // [startTime, endTime],
- // we can definitely find such a point with the closet timestamp equal to or smaller than the
- // given timestamp in the chunk.
- long timestamp = pageReader.timeBuffer.getLong(estimatedPos * 8);
- statistics.setEndTime(timestamp);
- switch (chunkMetadata.getDataType()) {
- // iotdb的int类型的plain编码用的是自制的不支持random access
- // case INT32:
- // return new MinMaxInfo(pageReader.valueBuffer.getInt(estimatedPos * 4),
- // pageReader.timeBuffer.getLong(estimatedPos * 8));
- case INT64:
- long longVal =
- pageReader.valueBuffer.getLong(pageReader.timeBufferLength + estimatedPos * 8);
- ((LongStatistics) statistics).setLastValue(longVal);
- break;
- case FLOAT:
- float floatVal =
- pageReader.valueBuffer.getFloat(pageReader.timeBufferLength + estimatedPos * 4);
- ((FloatStatistics) statistics).setLastValue(floatVal);
- break;
- case DOUBLE:
- double doubleVal =
- pageReader.valueBuffer.getDouble(pageReader.timeBufferLength + estimatedPos * 8);
- ((DoubleStatistics) statistics).setLastValue(doubleVal);
- break;
- default:
- throw new IOException("Unsupported data type!");
+ estimatedPos--;
+ } // else equals no need to minus 1
+
+ this.endPos = estimatedPos; // note this
+
+ // since we have constrained that targetTimestamp must be within the chunk time range
+ // [startTime, endTime],
+ // we can definitely find such a point with the closet timestamp equal to or smaller than the
+ // given timestamp in the chunk.
+ long timestamp = pageReader.timeBuffer.getLong(estimatedPos * 8);
+ statistics.setEndTime(timestamp);
+ switch (chunkMetadata.getDataType()) {
+ // iotdb的int类型的plain编码用的是自制的不支持random access
+ // case INT32:
+ // return new MinMaxInfo(pageReader.valueBuffer.getInt(estimatedPos * 4),
+ // pageReader.timeBuffer.getLong(estimatedPos * 8));
+ case INT64:
+ long longVal =
+ pageReader.valueBuffer.getLong(pageReader.timeBufferLength + estimatedPos * 8);
+ ((LongStatistics) statistics).setLastValue(longVal);
+ break;
+ case FLOAT:
+ float floatVal =
+ pageReader.valueBuffer.getFloat(pageReader.timeBufferLength + estimatedPos * 4);
+ ((FloatStatistics) statistics).setLastValue(floatVal);
+ break;
+ case DOUBLE:
+ double doubleVal =
+ pageReader.valueBuffer.getDouble(pageReader.timeBufferLength + estimatedPos * 8);
+ ((DoubleStatistics) statistics).setLastValue(doubleVal);
+ break;
+ default:
+ throw new IOException("Unsupported data type!");
+ }
+ return estimatedPos;
}
- return estimatedPos;
}
/**
@@ -371,33 +471,54 @@ public class ChunkSuit4CPV {
* @return true if exists; false not exist
*/
public boolean checkIfExist(long targetTimestamp) throws IOException {
- StepRegress stepRegress = chunkMetadata.getStatistics().getStepRegress();
- // infer position starts from 1, so minus 1 here
- // TODO debug buffer.get(index)
- int estimatedPos = (int) Math.round(stepRegress.infer(targetTimestamp)) - 1;
-
- // search from estimatePos in the timeBuffer to find the closet timestamp equal to or smaller
- // than the given timestamp
- if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
- while (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
- estimatedPos--;
- IOMonitor.incPointsTravered();
- }
- } else if (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
- while (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
+ if (TSFileDescriptor.getInstance().getConfig().isUseChunkIndex()) {
+ StepRegress stepRegress = chunkMetadata.getStatistics().getStepRegress();
+ // infer position starts from 1, so minus 1 here
+ // TODO debug buffer.get(index)
+ int estimatedPos = (int) Math.round(stepRegress.infer(targetTimestamp)) - 1;
+
+ // search from estimatePos in the timeBuffer to find the closet timestamp equal to or smaller
+ // than the given timestamp
+ if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
+ while (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
+ estimatedPos--;
+ IOMonitor.incPointsTravered();
+ }
+ } else if (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
+ while (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) {
+ estimatedPos++;
+ IOMonitor.incPointsTravered();
+ }
+ if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
+ estimatedPos--;
+ IOMonitor.incPointsTravered();
+ } // else equal
+ } // else equal
+
+ // since we have constrained that targetTimestamp must be within the chunk time range
+ // [startTime, endTime],
+ // estimatedPos will not be out of range.
+ return pageReader.timeBuffer.getLong(estimatedPos * 8) == targetTimestamp;
+ } else {
+ // search from estimatePos in the timeBuffer to find the closet timestamp equal to or smaller
+ // than the given timestamp
+ int estimatedPos = -1;
+ pageReader.timeBuffer.position(0);
+ pageReader.valueBuffer.position(pageReader.timeBufferLength);
+ while (pageReader.timeBuffer.remaining() > 0) {
estimatedPos++;
IOMonitor.incPointsTravered();
+ long t = pageReader.timeBuffer.getLong();
+ if (t >= targetTimestamp) {
+ break;
+ }
}
- if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) {
- estimatedPos--;
- IOMonitor.incPointsTravered();
- } // else equal
- } // else equal
- // since we have constrained that targetTimestamp must be within the chunk time range
- // [startTime, endTime],
- // estimatedPos will not be out of range.
- return pageReader.timeBuffer.getLong(estimatedPos * 8) == targetTimestamp;
+ // since we have constrained that targetTimestamp must be within the chunk time range
+ // [startTime, endTime],
+ // estimatedPos will not be out of range.
+ return pageReader.timeBuffer.getLong(estimatedPos * 8) == targetTimestamp;
+ }
}
public void updateFP(MinMaxInfo point) {