You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2023/03/23 07:27:17 UTC
[iotdb] branch master updated: [IOTDB-5718] Fix wrong time order due to overflow in MergeSort
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 6ed01068d0 [IOTDB-5718] Fix wrong time order due to overflow in MergeSort
6ed01068d0 is described below
commit 6ed01068d04dffc0af03be1f79a464d0c22f978f
Author: YangCaiyin <yc...@gmail.com>
AuthorDate: Thu Mar 23 15:27:11 2023 +0800
[IOTDB-5718] Fix wrong time order due to overflow in MergeSort
---
.../IoTDBOrderByWithAlignByDeviceIT.java | 54 ++++++++++++++++++++++
.../process/join/merge/MergeSortComparator.java | 28 ++++++-----
2 files changed, 70 insertions(+), 12 deletions(-)
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBOrderByWithAlignByDeviceIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBOrderByWithAlignByDeviceIT.java
index fba9177148..988b495432 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBOrderByWithAlignByDeviceIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBOrderByWithAlignByDeviceIT.java
@@ -75,6 +75,7 @@ public class IoTDBOrderByWithAlignByDeviceIT {
public static void setUp() throws Exception {
EnvFactory.getEnv().initClusterEnvironment();
insertData();
+ insertData2();
}
@AfterClass
@@ -173,6 +174,30 @@ public class IoTDBOrderByWithAlignByDeviceIT {
}
}
+ // use to test if the compare result of time will overflow.
+ protected static void insertData2() {
+ try (Connection iotDBConnection = EnvFactory.getEnv().getConnection();
+ Statement statement = iotDBConnection.createStatement()) {
+ long startTime = 1;
+ String createSql = "CREATE TIMESERIES root.overflow.value WITH DATATYPE=INT32, ENCODING=RLE";
+ statement.execute(createSql);
+ for (int i = 0; i < 20; i++) {
+ String insertTime =
+ "INSERT INTO "
+ + "root.overflow"
+ + "(timestamp,value) VALUES("
+ + (startTime + 2147483648L)
+ + ","
+ + i
+ + ")";
+ statement.execute(insertTime);
+ startTime += 2147483648L;
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
private void checkHeader(ResultSetMetaData resultSetMetaData, String title) throws SQLException {
String[] headers = title.split(",");
for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) {
@@ -180,6 +205,35 @@ public class IoTDBOrderByWithAlignByDeviceIT {
}
}
+ @Test
+ public void overFlowTest() {
+ String sql = "SELECT value from root.overflow ALIGN BY DEVICE";
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+
+ try (ResultSet resultSet = statement.executeQuery(sql)) {
+ long startTime = 1;
+ ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
+ checkHeader(resultSetMetaData, "Time,Device,value");
+ int cnt = 0;
+ while (resultSet.next()) {
+ long actualTimeStamp = resultSet.getLong(1);
+ String actualDevice = resultSet.getString(2);
+ assertEquals("root.overflow", actualDevice);
+ assertEquals(startTime + 2147483648L, actualTimeStamp);
+
+ String value = resultSet.getString(3);
+ assertEquals(value, Integer.toString(cnt));
+ startTime += 2147483648L;
+ cnt++;
+ }
+ }
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
// ORDER BY DEVICE
@Test
public void orderByDeviceTest1() {
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/join/merge/MergeSortComparator.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/join/merge/MergeSortComparator.java
index b25add1a25..16b66b8dd3 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/join/merge/MergeSortComparator.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/join/merge/MergeSortComparator.java
@@ -36,7 +36,8 @@ public class MergeSortComparator {
public static final Comparator<MergeSortKey> ASC_TIME_ASC_DEVICE =
(MergeSortKey o1, MergeSortKey o2) -> {
int timeComparing =
- (int) (o1.tsBlock.getTimeByIndex(o1.rowIndex) - o2.tsBlock.getTimeByIndex(o2.rowIndex));
+ Long.compare(
+ o1.tsBlock.getTimeByIndex(o1.rowIndex), o2.tsBlock.getTimeByIndex(o2.rowIndex));
return timeComparing == 0
? o1.tsBlock
.getColumn(0)
@@ -47,7 +48,8 @@ public class MergeSortComparator {
public static final Comparator<MergeSortKey> ASC_TIME_DESC_DEVICE =
(MergeSortKey o1, MergeSortKey o2) -> {
int timeComparing =
- (int) (o1.tsBlock.getTimeByIndex(o1.rowIndex) - o2.tsBlock.getTimeByIndex(o2.rowIndex));
+ Long.compare(
+ o1.tsBlock.getTimeByIndex(o1.rowIndex), o2.tsBlock.getTimeByIndex(o2.rowIndex));
return timeComparing == 0
? o2.tsBlock
.getColumn(0)
@@ -58,7 +60,8 @@ public class MergeSortComparator {
public static final Comparator<MergeSortKey> DESC_TIME_ASC_DEVICE =
(MergeSortKey o1, MergeSortKey o2) -> {
int timeComparing =
- (int) (o2.tsBlock.getTimeByIndex(o2.rowIndex) - o1.tsBlock.getTimeByIndex(o1.rowIndex));
+ Long.compare(
+ o2.tsBlock.getTimeByIndex(o2.rowIndex), o1.tsBlock.getTimeByIndex(o1.rowIndex));
return timeComparing == 0
? o1.tsBlock
.getColumn(0)
@@ -69,7 +72,8 @@ public class MergeSortComparator {
public static final Comparator<MergeSortKey> DESC_TIME_DESC_DEVICE =
(MergeSortKey o1, MergeSortKey o2) -> {
int timeComparing =
- (int) (o2.tsBlock.getTimeByIndex(o2.rowIndex) - o1.tsBlock.getTimeByIndex(o1.rowIndex));
+ Long.compare(
+ o2.tsBlock.getTimeByIndex(o2.rowIndex), o1.tsBlock.getTimeByIndex(o1.rowIndex));
return timeComparing == 0
? o2.tsBlock
.getColumn(0)
@@ -86,8 +90,8 @@ public class MergeSortComparator {
.getBinary(o1.rowIndex)
.compareTo(o2.tsBlock.getColumn(0).getBinary(o2.rowIndex));
return deviceComparing == 0
- ? (int)
- (o1.tsBlock.getTimeByIndex(o1.rowIndex) - o2.tsBlock.getTimeByIndex(o2.rowIndex))
+ ? Long.compare(
+ o1.tsBlock.getTimeByIndex(o1.rowIndex), o2.tsBlock.getTimeByIndex(o2.rowIndex))
: deviceComparing;
};
public static final Comparator<MergeSortKey> ASC_DEVICE_DESC_TIME =
@@ -98,8 +102,8 @@ public class MergeSortComparator {
.getBinary(o1.rowIndex)
.compareTo(o2.tsBlock.getColumn(0).getBinary(o2.rowIndex));
return deviceComparing == 0
- ? (int)
- (o2.tsBlock.getTimeByIndex(o2.rowIndex) - o1.tsBlock.getTimeByIndex(o1.rowIndex))
+ ? Long.compare(
+ o2.tsBlock.getTimeByIndex(o2.rowIndex), o1.tsBlock.getTimeByIndex(o1.rowIndex))
: deviceComparing;
};
public static final Comparator<MergeSortKey> DESC_DEVICE_ASC_TIME =
@@ -110,8 +114,8 @@ public class MergeSortComparator {
.getBinary(o2.rowIndex)
.compareTo(o1.tsBlock.getColumn(0).getBinary(o1.rowIndex));
return deviceComparing == 0
- ? (int)
- (o1.tsBlock.getTimeByIndex(o1.rowIndex) - o2.tsBlock.getTimeByIndex(o2.rowIndex))
+ ? Long.compare(
+ o1.tsBlock.getTimeByIndex(o1.rowIndex), o2.tsBlock.getTimeByIndex(o2.rowIndex))
: deviceComparing;
};
public static final Comparator<MergeSortKey> DESC_DEVICE_DESC_TIME =
@@ -122,8 +126,8 @@ public class MergeSortComparator {
.getBinary(o2.rowIndex)
.compareTo(o1.tsBlock.getColumn(0).getBinary(o1.rowIndex));
return deviceComparing == 0
- ? (int)
- (o2.tsBlock.getTimeByIndex(o2.rowIndex) - o1.tsBlock.getTimeByIndex(o1.rowIndex))
+ ? Long.compare(
+ o2.tsBlock.getTimeByIndex(o2.rowIndex), o1.tsBlock.getTimeByIndex(o1.rowIndex))
: deviceComparing;
};