You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2021/06/01 12:38:13 UTC
[iotdb] 01/01: Fix overlapped data bug
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a commit to branch 0.11overlap
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5f374eee8e54e2965ff831411a2f933061264812
Author: Alima777 <wx...@gmail.com>
AuthorDate: Tue Jun 1 20:37:12 2021 +0800
Fix overlapped data bug
---
.../iotdb/db/query/reader/series/SeriesReader.java | 3 +
.../db/integration/IoTDBOverlappedPageIT.java | 80 ++++++++++++++++++----
2 files changed, 70 insertions(+), 13 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
index 46b8c1c..411579c 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
@@ -617,6 +617,9 @@ public class SeriesReader {
unpackAllOverlappedChunkMetadataToPageReaders(timeValuePair.getTimestamp(), false);
unpackAllOverlappedUnseqPageReadersToMergeReader(timeValuePair.getTimestamp());
+ // update if there are unSeqPageReaders are unpacked
+ timeValuePair = mergeReader.currentTimeValuePair();
+
if (firstPageReader != null) {
if ((orderUtils.getAscending() && timeValuePair.getTimestamp() > firstPageReader
.getStatistics().getEndTime()) || (!orderUtils.getAscending()
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBOverlappedPageIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBOverlappedPageIT.java
index 76131ad..c575b17 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBOverlappedPageIT.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBOverlappedPageIT.java
@@ -26,6 +26,7 @@ import org.apache.iotdb.jdbc.Config;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.junit.AfterClass;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -44,31 +45,50 @@ import static org.junit.Assert.fail;
public class IoTDBOverlappedPageIT {
private static int beforeMaxNumberOfPointsInPage;
- private static long beforeMemtableSizeThreshold;
+ private static boolean enableUnseqCompaction;
+
+ private static String[] dataSet1 = new String[]{
+ "SET STORAGE GROUP TO root.sg1",
+ "CREATE TIMESERIES root.sg1.d1.s1 WITH DATATYPE=INT32, ENCODING=PLAIN",
+ "INSERT INTO root.sg1.d1(time,s1) values(1, 1)",
+ "INSERT INTO root.sg1.d1(time,s1) values(10, 10)",
+ "flush",
+ "INSERT INTO root.sg1.d1(time,s1) values(20, 20)",
+ "INSERT INTO root.sg1.d1(time,s1) values(30, 30)",
+ "flush",
+ "INSERT INTO root.sg1.d1(time,s1) values(110, 110)",
+ "flush",
+ "INSERT INTO root.sg1.d1(time,s1) values(5, 5)",
+ "INSERT INTO root.sg1.d1(time,s1) values(50, 50)",
+ "INSERT INTO root.sg1.d1(time,s1) values(100, 100)",
+ "flush",
+ "INSERT INTO root.sg1.d1(time,s1) values(15, 15)",
+ "INSERT INTO root.sg1.d1(time,s1) values(25, 25)",
+ "flush",
+ };
@BeforeClass
public static void setUp() throws Exception {
+ Class.forName(Config.JDBC_DRIVER_NAME);
EnvironmentUtils.closeStatMonitor();
IoTDBDescriptor.getInstance().getConfig()
.setCompactionStrategy(CompactionStrategy.NO_COMPACTION);
- beforeMemtableSizeThreshold = IoTDBDescriptor.getInstance().getConfig().getMemtableSizeThreshold();
- IoTDBDescriptor.getInstance().getConfig().setMemtableSizeThreshold(1024 * 16);
+ enableUnseqCompaction = IoTDBDescriptor.getInstance().getConfig().isEnableUnseqCompaction();
+ IoTDBDescriptor.getInstance().getConfig().setEnableUnseqCompaction(false);
+
// max_number_of_points_in_page = 10
beforeMaxNumberOfPointsInPage = TSFileDescriptor.getInstance().getConfig().getMaxNumberOfPointsInPage();
TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(10);
EnvironmentUtils.envSetUp();
- Class.forName(Config.JDBC_DRIVER_NAME);
- insertData();
}
@AfterClass
public static void tearDown() throws Exception {
// recovery value
- TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(beforeMaxNumberOfPointsInPage);
EnvironmentUtils.cleanEnv();
- IoTDBDescriptor.getInstance().getConfig().setMemtableSizeThreshold(beforeMemtableSizeThreshold);
- IoTDBDescriptor.getInstance().getConfig()
- .setCompactionStrategy(CompactionStrategy.LEVEL_COMPACTION);
+ IoTDBDescriptor.getInstance().getConfig().setCompactionStrategy(CompactionStrategy.LEVEL_COMPACTION);
+ IoTDBDescriptor.getInstance().getConfig().setEnableUnseqCompaction(enableUnseqCompaction);
+ IoTDBDescriptor.getInstance().getConfig().setMemtableSizeThreshold(beforeMaxNumberOfPointsInPage);
}
@Test
@@ -89,17 +109,51 @@ public class IoTDBOverlappedPageIT {
try (Connection connection = DriverManager
.getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
Statement statement = connection.createStatement()) {
+ insertData();
+
String sql = "select s0 from root.vehicle.d0 where time >= 1 and time <= 110 AND root.vehicle.d0.s0 > 110";
ResultSet resultSet = statement.executeQuery(sql);
int cnt = 0;
- try {
+ while (resultSet.next()) {
+ String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString("root.vehicle.d0.s0");
+ Assert.assertEquals(res[cnt], ans);
+ cnt++;
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ /**
+ * Test to check if timeValuePair is updated when there are unSeqPageReaders are unpacked
+ * in method hasNextOverlappedPage() in SeriesReader.java
+ */
+ @Test
+ public void selectOverlappedPageTest2() {
+ String[] res = {
+ "0,10"
+ };
+
+ try (Connection connection = DriverManager
+ .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+
+ for (String insertSql : dataSet1) {
+ statement.execute(insertSql);
+ }
+
+ boolean hasResultSet = statement.execute("select count(s1) from root.sg1.d1");
+ Assert.assertTrue(hasResultSet);
+ int cnt = 0;
+ try (ResultSet resultSet = statement.getResultSet()) {
while (resultSet.next()) {
- String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString("root.vehicle.d0.s0");
+ String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet
+ .getString("count(root.sg1.d1.s1)");
Assert.assertEquals(res[cnt], ans);
cnt++;
}
- } finally {
- resultSet.close();
+ Assert.assertEquals(res.length, cnt);
}
} catch (Exception e) {
e.printStackTrace();