You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2022/01/26 07:06:46 UTC
[iotdb] branch OverlappedPageBug created (now 09b68bb)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a change to branch OverlappedPageBug
in repository https://gitbox.apache.org/repos/asf/iotdb.git.
at 09b68bb [To rel/0.12] [IOTDB-2445] Fix overlapped data should be consumed first bug
This branch includes the following new commits:
new 09b68bb [To rel/0.12] [IOTDB-2445] Fix overlapped data should be consumed first bug
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: [To rel/0.12] [IOTDB-2445] Fix overlapped data should be consumed first bug
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch OverlappedPageBug
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 09b68bb7696d7282e6d12a187ed0f9ccbdb33dd9
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Wed Jan 26 15:05:49 2022 +0800
[To rel/0.12] [IOTDB-2445] Fix overlapped data should be consumed first bug
---
.../iotdb/db/query/reader/series/SeriesReader.java | 12 ++-
.../db/integration/IoTDBUnseqOverlappedPageIT.java | 116 +++++++++++++++++++++
2 files changed, 127 insertions(+), 1 deletion(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
index 64141d6..f25f37a 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
@@ -816,6 +816,17 @@ public class SeriesReader {
return;
}
+ long currentPageEndpointTime;
+
+ if (!unSeqPageReaders.isEmpty()) {
+ currentPageEndpointTime =
+ orderUtils.getOverlapCheckTime(unSeqPageReaders.peek().getStatistics());
+ // unpack all overlapped data for first unseq page
+ unpackAllOverlappedTsFilesToTimeSeriesMetadata(currentPageEndpointTime);
+ unpackAllOverlappedTimeSeriesMetadataToCachedChunkMetadata(currentPageEndpointTime, false);
+ unpackAllOverlappedChunkMetadataToPageReaders(currentPageEndpointTime, false);
+ }
+
/*
* init firstPageReader
*/
@@ -823,7 +834,6 @@ public class SeriesReader {
initFirstPageReader();
}
- long currentPageEndpointTime;
if (mergeReader.hasNextTimeValuePair()) {
currentPageEndpointTime = mergeReader.getCurrentReadStopTime();
} else {
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBUnseqOverlappedPageIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBUnseqOverlappedPageIT.java
new file mode 100644
index 0000000..211aa7b
--- /dev/null
+++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBUnseqOverlappedPageIT.java
@@ -0,0 +1,116 @@
+package org.apache.iotdb.db.integration;
+
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.utils.EnvironmentUtils;
+import org.apache.iotdb.jdbc.Config;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
+
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.Statement;
+
+import static org.apache.iotdb.db.constant.TestConstant.TIMESTAMP_STR;
+import static org.junit.Assert.fail;
+
+/**
+ * Notice that, all test begins with "IoTDB" is integration test. All test which will start the
+ * IoTDB server should be defined as integration test.
+ *
+ * <p>You can comprehensively view the generated data in the following online doc:
+ *
+ * <p>https://docs.google.com/spreadsheets/d/1kfrSR1_paSd9B1Z0jnPBD3WQIMDslDuNm4R0mpWx9Ms/edit?usp=sharing
+ */
+public class IoTDBUnseqOverlappedPageIT {
+
+ private static int beforeMaxNumberOfPointsInPage;
+
+ private static String[] dataSet1 =
+ new String[] {
+ "SET STORAGE GROUP TO root.sg1",
+ "CREATE TIMESERIES root.sg1.d1.s1 WITH DATATYPE=INT32, ENCODING=PLAIN",
+ "INSERT INTO root.sg1.d1(time,s1) values(1, 1)",
+ "INSERT INTO root.sg1.d1(time,s1) values(2, 2)",
+ "INSERT INTO root.sg1.d1(time,s1) values(3, 3)",
+ "INSERT INTO root.sg1.d1(time,s1) values(4, 4)",
+ "INSERT INTO root.sg1.d1(time,s1) values(5, 5)",
+ "INSERT INTO root.sg1.d1(time,s1) values(6, 6)",
+ "flush",
+ "INSERT INTO root.sg1.d1(time,s1) values(8, 8)",
+ "INSERT INTO root.sg1.d1(time,s1) values(9, 9)",
+ "INSERT INTO root.sg1.d1(time,s1) values(10, 10)",
+ "INSERT INTO root.sg1.d1(time,s1) values(11, 11)",
+ "INSERT INTO root.sg1.d1(time,s1) values(12, 12)",
+ "INSERT INTO root.sg1.d1(time,s1) values(13, 13)",
+ "flush",
+ "INSERT INTO root.sg1.d1(time,s1) values(5, 50)",
+ "INSERT INTO root.sg1.d1(time,s1) values(7, 70)",
+ "INSERT INTO root.sg1.d1(time,s1) values(11, 110)",
+ "INSERT INTO root.sg1.d1(time,s1) values(12, 120)",
+ "flush",
+ };
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ Class.forName(Config.JDBC_DRIVER_NAME);
+ EnvironmentUtils.closeStatMonitor();
+ // max_number_of_points_in_page = 10
+ beforeMaxNumberOfPointsInPage =
+ TSFileDescriptor.getInstance().getConfig().getMaxNumberOfPointsInPage();
+ TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(3);
+ EnvironmentUtils.envSetUp();
+ try (Connection connection =
+ DriverManager.getConnection(
+ Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+ for (String insertSql : dataSet1) {
+ statement.execute(insertSql);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ // recovery value
+ EnvironmentUtils.cleanEnv();
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .setMemtableSizeThreshold(beforeMaxNumberOfPointsInPage);
+ }
+
+ @Test
+ public void selectOverlappedPageTest() {
+ String[] res = {
+ "1,1", "2,2", "3,3", "4,4", "5,50", "6,6", "7,70", "8,8", "9,9", "10,10", "11,110", "12,120",
+ "13,13"
+ };
+
+ try (Connection connection =
+ DriverManager.getConnection(
+ Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+
+ String sql = "select s1 from root.sg1.d1";
+ try (ResultSet resultSet = statement.executeQuery(sql)) {
+ int cnt = 0;
+ while (resultSet.next()) {
+ String ans =
+ resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString("root.sg1.d1.s1");
+ Assert.assertEquals(res[cnt], ans);
+ cnt++;
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+}