You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2022/01/15 01:53:48 UTC
[iotdb] branch master updated: [IOTDB-2400] Fix series reader bug (#4819)
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 53f48d2 [IOTDB-2400] Fix series reader bug (#4819)
53f48d2 is described below
commit 53f48d2f6bf3ffe716ca9067fb0b9275f7665780
Author: Jackie Tien <ja...@gmail.com>
AuthorDate: Sat Jan 15 09:53:15 2022 +0800
[IOTDB-2400] Fix series reader bug (#4819)
---
.../aligned/IOTDBInsertAlignedValuesIT.java | 3 +
.../IoTDBRawQueryWithoutValueFilter3IT.java | 64 ++++++++++++++++++++++
.../iotdb/db/query/reader/series/SeriesReader.java | 19 +++++--
3 files changed, 81 insertions(+), 5 deletions(-)
diff --git a/integration/src/test/java/org/apache/iotdb/db/integration/aligned/IOTDBInsertAlignedValuesIT.java b/integration/src/test/java/org/apache/iotdb/db/integration/aligned/IOTDBInsertAlignedValuesIT.java
index 14eb63d..f2d5ced 100644
--- a/integration/src/test/java/org/apache/iotdb/db/integration/aligned/IOTDBInsertAlignedValuesIT.java
+++ b/integration/src/test/java/org/apache/iotdb/db/integration/aligned/IOTDBInsertAlignedValuesIT.java
@@ -42,6 +42,7 @@ import java.util.Objects;
@Category({LocalStandaloneTest.class})
public class IOTDBInsertAlignedValuesIT {
private static Connection connection;
+ private int numOfPointsPerPage;
@Before
public void setUp() throws Exception {
@@ -51,11 +52,13 @@ public class IOTDBInsertAlignedValuesIT {
Class.forName(Config.JDBC_DRIVER_NAME);
connection =
DriverManager.getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ numOfPointsPerPage = TSFileDescriptor.getInstance().getConfig().getMaxNumberOfPointsInPage();
}
@After
public void tearDown() throws Exception {
close();
+ TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(numOfPointsPerPage);
EnvironmentUtils.cleanEnv();
}
diff --git a/integration/src/test/java/org/apache/iotdb/db/integration/aligned/IoTDBRawQueryWithoutValueFilter3IT.java b/integration/src/test/java/org/apache/iotdb/db/integration/aligned/IoTDBRawQueryWithoutValueFilter3IT.java
new file mode 100644
index 0000000..e28d6a0
--- /dev/null
+++ b/integration/src/test/java/org/apache/iotdb/db/integration/aligned/IoTDBRawQueryWithoutValueFilter3IT.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.integration.aligned;
+
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.utils.EnvironmentUtils;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+/** Let One chunk has more than one page. */
+public class IoTDBRawQueryWithoutValueFilter3IT extends IoTDBRawQueryWithoutValueFilterIT {
+
+ private static int numOfPointsPerPage;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ EnvironmentUtils.closeStatMonitor();
+ EnvironmentUtils.envSetUp();
+ // TODO When the aligned time series support compaction, we need to set compaction to true
+ enableSeqSpaceCompaction =
+ IoTDBDescriptor.getInstance().getConfig().isEnableSeqSpaceCompaction();
+ enableUnseqSpaceCompaction =
+ IoTDBDescriptor.getInstance().getConfig().isEnableUnseqSpaceCompaction();
+ enableCrossSpaceCompaction =
+ IoTDBDescriptor.getInstance().getConfig().isEnableCrossSpaceCompaction();
+ numOfPointsPerPage = TSFileDescriptor.getInstance().getConfig().getMaxNumberOfPointsInPage();
+ IoTDBDescriptor.getInstance().getConfig().setEnableSeqSpaceCompaction(false);
+ IoTDBDescriptor.getInstance().getConfig().setEnableUnseqSpaceCompaction(false);
+ IoTDBDescriptor.getInstance().getConfig().setEnableCrossSpaceCompaction(false);
+ TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(2);
+ AlignedWriteUtil.insertData();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ IoTDBDescriptor.getInstance().getConfig().setEnableSeqSpaceCompaction(enableSeqSpaceCompaction);
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .setEnableUnseqSpaceCompaction(enableUnseqSpaceCompaction);
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .setEnableCrossSpaceCompaction(enableCrossSpaceCompaction);
+ TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(numOfPointsPerPage);
+ EnvironmentUtils.cleanEnv();
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
index 63d28eb..2dfb8f0 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
@@ -473,7 +473,7 @@ public class SeriesReader {
*/
if (hasCachedNextOverlappedPage) {
return true;
- } else if (mergeReader.hasNextTimeValuePair()) {
+ } else if (mergeReader.hasNextTimeValuePair() || firstPageOverlapped()) {
if (hasNextOverlappedPage()) {
cachedBatchData = nextOverlappedPage();
if (cachedBatchData != null && cachedBatchData.hasCurrent()) {
@@ -551,6 +551,12 @@ public class SeriesReader {
if (firstPageReader == null) {
return false;
}
+
+ long endpointTime = orderUtils.getOverlapCheckTime(firstPageReader.getStatistics());
+ unpackAllOverlappedTsFilesToTimeSeriesMetadata(endpointTime);
+ unpackAllOverlappedTimeSeriesMetadataToCachedChunkMetadata(endpointTime, false);
+ unpackAllOverlappedChunkMetadataToPageReaders(endpointTime, false);
+
return (!seqPageReaders.isEmpty()
&& orderUtils.isOverlapped(
firstPageReader.getStatistics(), seqPageReaders.get(0).getStatistics()))
@@ -558,8 +564,9 @@ public class SeriesReader {
&& orderUtils.isOverlapped(
firstPageReader.getStatistics(), unSeqPageReaders.peek().getStatistics())
|| (mergeReader.hasNextTimeValuePair()
- && mergeReader.currentTimeValuePair().getTimestamp()
- >= firstPageReader.getStatistics().getStartTime()));
+ && orderUtils.isOverlapped(
+ mergeReader.currentTimeValuePair().getTimestamp(),
+ firstPageReader.getStatistics())));
}
private void unpackAllOverlappedChunkMetadataToPageReaders(long endpointTime, boolean init)
@@ -808,7 +815,8 @@ public class SeriesReader {
cachedBatchData.flip();
hasCachedNextOverlappedPage = cachedBatchData.hasCurrent();
return hasCachedNextOverlappedPage;
- } else {
+ } else if (orderUtils.isOverlapped(
+ timeValuePair.getTimestamp(), firstPageReader.getStatistics())) {
// current timeValuePair is overlapped with firstPageReader, add it to merged reader
// and update endTime to the max end time
mergeReader.addReader(
@@ -835,7 +843,8 @@ public class SeriesReader {
cachedBatchData.flip();
hasCachedNextOverlappedPage = cachedBatchData.hasCurrent();
return hasCachedNextOverlappedPage;
- } else {
+ } else if (orderUtils.isOverlapped(
+ timeValuePair.getTimestamp(), seqPageReaders.get(0).getStatistics())) {
VersionPageReader pageReader = seqPageReaders.remove(0);
mergeReader.addReader(
pageReader