You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2021/06/04 08:12:15 UTC
[iotdb] branch rel/0.12 updated (63127bc -> e597ab9)
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a change to branch rel/0.12
in repository https://gitbox.apache.org/repos/asf/iotdb.git.
from 63127bc change default force full merge config (#3195)
new c0b2c8b jdbc - get col type bug (#3220)
new 824b4c6 Bug Fix: Overlapped data should be consumed first (#3270)
new 6ffda68 Fix group by data inconsistence bug (#3317)
new e597ab9 Fix bug in disable align it (#3333)
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../iotdb/jdbc/AbstractIoTDBJDBCResultSet.java | 4 +-
.../iotdb/jdbc/IoTDBNonAlignJDBCResultSet.java | 3 +
.../org/apache/iotdb/jdbc/IoTDBResultMetadata.java | 8 +-
.../apache/iotdb/jdbc/IoTDBResultMetadataTest.java | 2 +-
.../db/query/aggregation/impl/CountAggrResult.java | 24 ++---
.../iotdb/db/query/reader/series/SeriesReader.java | 8 +-
.../iotdb/db/integration/IoTDBDisableAlignIT.java | 2 +-
.../iotdb/db/integration/IoTDBGroupByUnseqIT.java | 103 ++++++++++++++++-----
.../db/integration/IoTDBOverlappedPageIT.java | 86 ++++++++++++++---
.../iotdb/db/integration/IoTDBSimpleQueryIT.java | 39 ++++++++
.../java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java | 8 +-
.../org/apache/iotdb/session/SessionDataSet.java | 2 +-
.../iotdb/session/pool/SessionDataSetWrapper.java | 3 +-
.../apache/iotdb/session/IoTDBSessionSimpleIT.java | 2 +-
14 files changed, 218 insertions(+), 76 deletions(-)
[iotdb] 01/04: jdbc - get col type bug (#3220)
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a commit to branch rel/0.12
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit c0b2c8bef5733dbec2c7408db66a84ed663ddc0d
Author: Haimei Guo <68...@users.noreply.github.com>
AuthorDate: Mon May 31 15:32:21 2021 +0800
jdbc - get col type bug (#3220)
(cherry picked from commit 275918e52fb030355b2e60b7e160ca7ed2f4e63d)
---
.../iotdb/jdbc/AbstractIoTDBJDBCResultSet.java | 4 ++-
.../iotdb/jdbc/IoTDBNonAlignJDBCResultSet.java | 3 ++
.../org/apache/iotdb/jdbc/IoTDBResultMetadata.java | 8 ++---
.../apache/iotdb/jdbc/IoTDBResultMetadataTest.java | 2 +-
.../iotdb/db/integration/IoTDBSimpleQueryIT.java | 39 ++++++++++++++++++++++
.../java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java | 8 ++---
.../org/apache/iotdb/session/SessionDataSet.java | 2 +-
.../iotdb/session/pool/SessionDataSetWrapper.java | 3 +-
.../apache/iotdb/session/IoTDBSessionSimpleIT.java | 2 +-
9 files changed, 55 insertions(+), 16 deletions(-)
diff --git a/jdbc/src/main/java/org/apache/iotdb/jdbc/AbstractIoTDBJDBCResultSet.java b/jdbc/src/main/java/org/apache/iotdb/jdbc/AbstractIoTDBJDBCResultSet.java
index 36f74fd..5e13b77 100644
--- a/jdbc/src/main/java/org/apache/iotdb/jdbc/AbstractIoTDBJDBCResultSet.java
+++ b/jdbc/src/main/java/org/apache/iotdb/jdbc/AbstractIoTDBJDBCResultSet.java
@@ -399,7 +399,9 @@ public abstract class AbstractIoTDBJDBCResultSet implements ResultSet {
@Override
public ResultSetMetaData getMetaData() {
return new IoTDBResultMetadata(
- ioTDBRpcDataSet.columnNameList, columnTypeList, ioTDBRpcDataSet.ignoreTimeStamp);
+ ioTDBRpcDataSet.columnNameList,
+ ioTDBRpcDataSet.columnTypeList,
+ ioTDBRpcDataSet.ignoreTimeStamp);
}
@Override
diff --git a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBNonAlignJDBCResultSet.java b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBNonAlignJDBCResultSet.java
index f293bad..da0831a 100644
--- a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBNonAlignJDBCResultSet.java
+++ b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBNonAlignJDBCResultSet.java
@@ -81,6 +81,7 @@ public class IoTDBNonAlignJDBCResultSet extends AbstractIoTDBJDBCResultSet {
times = new byte[columnNameList.size()][Long.BYTES];
ioTDBRpcDataSet.columnNameList = new ArrayList<>();
+ ioTDBRpcDataSet.columnTypeList = new ArrayList<>();
// deduplicate and map
ioTDBRpcDataSet.columnOrdinalMap = new HashMap<>();
ioTDBRpcDataSet.columnOrdinalMap.put(TIMESTAMP_STR, 1);
@@ -93,6 +94,8 @@ public class IoTDBNonAlignJDBCResultSet extends AbstractIoTDBJDBCResultSet {
String name = columnNameList.get(i);
ioTDBRpcDataSet.columnNameList.add(TIMESTAMP_STR + name);
ioTDBRpcDataSet.columnNameList.add(name);
+ ioTDBRpcDataSet.columnTypeList.add(String.valueOf(TSDataType.INT64));
+ ioTDBRpcDataSet.columnTypeList.add(columnTypeList.get(i));
if (!ioTDBRpcDataSet.columnOrdinalMap.containsKey(name)) {
int index = columnNameIndex.get(name);
ioTDBRpcDataSet.columnOrdinalMap.put(name, index + START_INDEX);
diff --git a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBResultMetadata.java b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBResultMetadata.java
index 5fa70a0..e4ed8e5 100644
--- a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBResultMetadata.java
+++ b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBResultMetadata.java
@@ -115,12 +115,8 @@ public class IoTDBResultMetadata implements ResultSetMetaData {
return Types.TIMESTAMP;
}
// BOOLEAN, INT32, INT64, FLOAT, DOUBLE, TEXT,
- String columnType;
- if (!ignoreTimestamp) {
- columnType = columnTypeList.get(column - 2);
- } else {
- columnType = columnTypeList.get(column - 1);
- }
+ String columnType = columnTypeList.get(column - 1);
+
switch (columnType.toUpperCase()) {
case "BOOLEAN":
return Types.BOOLEAN;
diff --git a/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBResultMetadataTest.java b/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBResultMetadataTest.java
index cd208f8..a10bd05 100644
--- a/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBResultMetadataTest.java
+++ b/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBResultMetadataTest.java
@@ -143,7 +143,7 @@ public class IoTDBResultMetadataTest {
"root.a.b.double",
"root.a.b.text"
};
- String[] typesString = {"BOOLEAN", "INT32", "INT64", "FLOAT", "DOUBLE", "TEXT"};
+ String[] typesString = {"TIMESTAMP", "BOOLEAN", "INT32", "INT64", "FLOAT", "DOUBLE", "TEXT"};
int[] types = {
Types.BOOLEAN, Types.INTEGER, Types.BIGINT, Types.FLOAT, Types.DOUBLE, Types.VARCHAR
};
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSimpleQueryIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSimpleQueryIT.java
index 3a6fb43..1a309fd 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSimpleQueryIT.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSimpleQueryIT.java
@@ -38,6 +38,7 @@ import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
+import java.sql.Types;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -1276,4 +1277,42 @@ public class IoTDBSimpleQueryIT {
fail();
}
}
+
+ @Test
+ public void testDisableAlign() throws Exception {
+ Class.forName(Config.JDBC_DRIVER_NAME);
+ try (Connection connection =
+ DriverManager.getConnection(
+ Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+ statement.execute("CREATE TIMESERIES root.sg1.d1.s1 WITH DATATYPE=INT32");
+ statement.execute("CREATE TIMESERIES root.sg1.d1.s2 WITH DATATYPE=BOOLEAN");
+ ResultSet resultSet = statement.executeQuery("select s1, s2 from root.sg1.d1 disable align");
+ ResultSetMetaData metaData = resultSet.getMetaData();
+ int[] types = {Types.TIMESTAMP, Types.INTEGER, Types.BIGINT, Types.BOOLEAN};
+ int columnCount = metaData.getColumnCount();
+ for (int i = 0; i < columnCount; i++) {
+ Assert.assertEquals(types[i], metaData.getColumnType(i + 1));
+ }
+ }
+ }
+
+ @Test
+ public void testEnableAlign() throws Exception {
+ Class.forName(Config.JDBC_DRIVER_NAME);
+ try (Connection connection =
+ DriverManager.getConnection(
+ Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+ statement.execute("CREATE TIMESERIES root.sg1.d1.s1 WITH DATATYPE=INT32");
+ statement.execute("CREATE TIMESERIES root.sg1.d1.s2 WITH DATATYPE=BOOLEAN");
+ ResultSet resultSet = statement.executeQuery("select s1, s2 from root.sg1.d1");
+ ResultSetMetaData metaData = resultSet.getMetaData();
+ int[] types = {Types.TIMESTAMP, Types.INTEGER, Types.BOOLEAN};
+ int columnCount = metaData.getColumnCount();
+ for (int i = 0; i < columnCount; i++) {
+ Assert.assertEquals(types[i], metaData.getColumnType(i + 1));
+ }
+ }
+ }
}
diff --git a/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java b/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java
index a85b21f..fe2ab91 100644
--- a/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java
+++ b/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java
@@ -50,7 +50,7 @@ public class IoTDBRpcDataSet {
public boolean isClosed = false;
public TSIService.Iface client;
public List<String> columnNameList; // no deduplication
- public List<TSDataType> columnTypeList; // no deduplication
+ public List<String> columnTypeList; // no deduplication
public Map<String, Integer>
columnOrdinalMap; // used because the server returns deduplicated columns
public List<TSDataType> columnTypeDeduplicatedList; // deduplicated from columnTypeList
@@ -105,7 +105,7 @@ public class IoTDBRpcDataSet {
this.columnTypeList = new ArrayList<>();
if (!ignoreTimeStamp) {
this.columnNameList.add(TIMESTAMP_STR);
- this.columnTypeList.add(TSDataType.INT64);
+ this.columnTypeList.add(String.valueOf(TSDataType.INT64));
}
// deduplicate and map
this.columnOrdinalMap = new HashMap<>();
@@ -122,7 +122,7 @@ public class IoTDBRpcDataSet {
for (int i = 0; i < columnNameList.size(); i++) {
String name = columnNameList.get(i);
this.columnNameList.add(name);
- this.columnTypeList.add(TSDataType.valueOf(columnTypeList.get(i)));
+ this.columnTypeList.add(columnTypeList.get(i));
if (!columnOrdinalMap.containsKey(name)) {
int index = columnNameIndex.get(name);
columnOrdinalMap.put(name, index + START_INDEX);
@@ -135,7 +135,7 @@ public class IoTDBRpcDataSet {
for (int i = 0; i < columnNameList.size(); i++) {
String name = columnNameList.get(i);
this.columnNameList.add(name);
- this.columnTypeList.add(TSDataType.valueOf(columnTypeList.get(i)));
+ this.columnTypeList.add(columnTypeList.get(i));
if (!columnOrdinalMap.containsKey(name)) {
columnOrdinalMap.put(name, index++);
columnTypeDeduplicatedList.add(TSDataType.valueOf(columnTypeList.get(i)));
diff --git a/session/src/main/java/org/apache/iotdb/session/SessionDataSet.java b/session/src/main/java/org/apache/iotdb/session/SessionDataSet.java
index 5e6bef5..541d6e4 100644
--- a/session/src/main/java/org/apache/iotdb/session/SessionDataSet.java
+++ b/session/src/main/java/org/apache/iotdb/session/SessionDataSet.java
@@ -110,7 +110,7 @@ public class SessionDataSet {
return new ArrayList<>(ioTDBRpcDataSet.columnNameList);
}
- public List<TSDataType> getColumnTypes() {
+ public List<String> getColumnTypes() {
return new ArrayList<>(ioTDBRpcDataSet.columnTypeList);
}
diff --git a/session/src/main/java/org/apache/iotdb/session/pool/SessionDataSetWrapper.java b/session/src/main/java/org/apache/iotdb/session/pool/SessionDataSetWrapper.java
index fa5eb4c..e2e3b2c 100644
--- a/session/src/main/java/org/apache/iotdb/session/pool/SessionDataSetWrapper.java
+++ b/session/src/main/java/org/apache/iotdb/session/pool/SessionDataSetWrapper.java
@@ -23,7 +23,6 @@ import org.apache.iotdb.rpc.StatementExecutionException;
import org.apache.iotdb.session.Session;
import org.apache.iotdb.session.SessionDataSet;
import org.apache.iotdb.session.SessionDataSet.DataIterator;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.RowRecord;
import java.util.List;
@@ -88,7 +87,7 @@ public class SessionDataSetWrapper implements AutoCloseable {
return sessionDataSet.getColumnNames();
}
- public List<TSDataType> getColumnTypes() {
+ public List<String> getColumnTypes() {
return sessionDataSet.getColumnTypes();
}
diff --git a/session/src/test/java/org/apache/iotdb/session/IoTDBSessionSimpleIT.java b/session/src/test/java/org/apache/iotdb/session/IoTDBSessionSimpleIT.java
index ed72527..a227a3d 100644
--- a/session/src/test/java/org/apache/iotdb/session/IoTDBSessionSimpleIT.java
+++ b/session/src/test/java/org/apache/iotdb/session/IoTDBSessionSimpleIT.java
@@ -610,7 +610,7 @@ public class IoTDBSessionSimpleIT {
"root.sg.d1.s2"
});
Assert.assertArrayEquals(
- dataSet.getColumnTypes().toArray(new TSDataType[0]),
+ dataSet.getColumnTypes().toArray(new String[0]),
new TSDataType[] {
TSDataType.INT64,
TSDataType.INT64,
[iotdb] 04/04: Fix bug in disable align it (#3333)
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a commit to branch rel/0.12
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit e597ab91bb8a31b19b4886dbacd83d2a396ad4ba
Author: Haimei Guo <68...@users.noreply.github.com>
AuthorDate: Fri Jun 4 15:59:40 2021 +0800
Fix bug in disable align it (#3333)
(cherry picked from commit 22d8e0d5d19e87431d6bc41f66dbe7572ef09719)
---
.../test/java/org/apache/iotdb/db/integration/IoTDBDisableAlignIT.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBDisableAlignIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBDisableAlignIT.java
index f96648d..d953d04 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBDisableAlignIT.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBDisableAlignIT.java
@@ -410,7 +410,7 @@ public class IoTDBDisableAlignIT {
String columnName = resultSetMetaData.getColumnName(i);
Integer typeIndex = expectedHeaderToTypeIndexMap.get(columnName);
if (typeIndex != null) {
- Assert.assertEquals(expectedTypes[typeIndex], resultSetMetaData.getColumnType(1 + i / 2));
+ Assert.assertEquals(expectedTypes[typeIndex], resultSetMetaData.getColumnType(i));
}
actualIndexToExpectedIndexList.add(expectedHeaderToColumnIndexMap.get(columnName));
}
[iotdb] 02/04: Bug Fix: Overlapped data should be consumed first
(#3270)
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a commit to branch rel/0.12
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 824b4c6a7b4f51feb8ffefb4f7ea84c9c2a8e11b
Author: Xiangwei Wei <34...@users.noreply.github.com>
AuthorDate: Thu Jun 3 10:15:01 2021 +0800
Bug Fix: Overlapped data should be consumed first (#3270)
Overlapped data should be consumed first
(cherry picked from commit 22086d318668c344aa60dae997c6f45a1db5a522)
---
.../iotdb/db/query/reader/series/SeriesReader.java | 5 +-
.../db/integration/IoTDBOverlappedPageIT.java | 86 ++++++++++++++++++----
2 files changed, 75 insertions(+), 16 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
index 01cb4f1..aa78aeb 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
@@ -480,7 +480,7 @@ public class SeriesReader {
firstPageReader.getStatistics(), unSeqPageReaders.peek().getStatistics())
|| (mergeReader.hasNextTimeValuePair()
&& mergeReader.currentTimeValuePair().getTimestamp()
- > firstPageReader.getStatistics().getStartTime()));
+ >= firstPageReader.getStatistics().getStartTime()));
}
private void unpackAllOverlappedChunkMetadataToPageReaders(long endpointTime, boolean init)
@@ -683,6 +683,9 @@ public class SeriesReader {
unpackAllOverlappedChunkMetadataToPageReaders(timeValuePair.getTimestamp(), false);
unpackAllOverlappedUnseqPageReadersToMergeReader(timeValuePair.getTimestamp());
+ // update if there are unpacked unSeqPageReaders
+ timeValuePair = mergeReader.currentTimeValuePair();
+
// from now, the unsequence reader is all unpacked, so we don't need to consider it
// we has first page reader now
if (firstPageReader != null) {
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBOverlappedPageIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBOverlappedPageIT.java
index d62c1fc..8cb9f27 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBOverlappedPageIT.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBOverlappedPageIT.java
@@ -45,37 +45,57 @@ import static org.junit.Assert.fail;
public class IoTDBOverlappedPageIT {
private static int beforeMaxNumberOfPointsInPage;
- private static long beforeMemtableSizeThreshold;
+ private static boolean enableUnseqCompaction;
+
+ private static String[] dataSet1 =
+ new String[] {
+ "SET STORAGE GROUP TO root.sg1",
+ "CREATE TIMESERIES root.sg1.d1.s1 WITH DATATYPE=INT32, ENCODING=PLAIN",
+ "INSERT INTO root.sg1.d1(time,s1) values(1, 1)",
+ "INSERT INTO root.sg1.d1(time,s1) values(10, 10)",
+ "flush",
+ "INSERT INTO root.sg1.d1(time,s1) values(20, 20)",
+ "INSERT INTO root.sg1.d1(time,s1) values(30, 30)",
+ "flush",
+ "INSERT INTO root.sg1.d1(time,s1) values(110, 110)",
+ "flush",
+ "INSERT INTO root.sg1.d1(time,s1) values(5, 5)",
+ "INSERT INTO root.sg1.d1(time,s1) values(50, 50)",
+ "INSERT INTO root.sg1.d1(time,s1) values(100, 100)",
+ "flush",
+ "INSERT INTO root.sg1.d1(time,s1) values(15, 15)",
+ "INSERT INTO root.sg1.d1(time,s1) values(25, 25)",
+ "flush",
+ };
@BeforeClass
public static void setUp() throws Exception {
+ Class.forName(Config.JDBC_DRIVER_NAME);
EnvironmentUtils.closeStatMonitor();
IoTDBDescriptor.getInstance()
.getConfig()
.setCompactionStrategy(CompactionStrategy.NO_COMPACTION);
- beforeMemtableSizeThreshold =
- IoTDBDescriptor.getInstance().getConfig().getMemtableSizeThreshold();
- IoTDBDescriptor.getInstance().getConfig().setMemtableSizeThreshold(1024 * 16);
+ enableUnseqCompaction = IoTDBDescriptor.getInstance().getConfig().isEnableUnseqCompaction();
+ IoTDBDescriptor.getInstance().getConfig().setEnableUnseqCompaction(false);
+
// max_number_of_points_in_page = 10
beforeMaxNumberOfPointsInPage =
TSFileDescriptor.getInstance().getConfig().getMaxNumberOfPointsInPage();
TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(10);
EnvironmentUtils.envSetUp();
- Class.forName(Config.JDBC_DRIVER_NAME);
- insertData();
}
@AfterClass
public static void tearDown() throws Exception {
// recovery value
- TSFileDescriptor.getInstance()
- .getConfig()
- .setMaxNumberOfPointsInPage(beforeMaxNumberOfPointsInPage);
EnvironmentUtils.cleanEnv();
- IoTDBDescriptor.getInstance().getConfig().setMemtableSizeThreshold(beforeMemtableSizeThreshold);
IoTDBDescriptor.getInstance()
.getConfig()
.setCompactionStrategy(CompactionStrategy.LEVEL_COMPACTION);
+ IoTDBDescriptor.getInstance().getConfig().setEnableUnseqCompaction(enableUnseqCompaction);
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .setMemtableSizeThreshold(beforeMaxNumberOfPointsInPage);
}
@Test
@@ -89,19 +109,55 @@ public class IoTDBOverlappedPageIT {
DriverManager.getConnection(
Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
Statement statement = connection.createStatement()) {
+ insertData();
+
String sql =
"select s0 from root.vehicle.d0 where time >= 1 and time <= 110 AND root.vehicle.d0.s0 > 110";
- ResultSet resultSet = statement.executeQuery(sql);
- int cnt = 0;
- try {
+ try (ResultSet resultSet = statement.executeQuery(sql)) {
+ int cnt = 0;
while (resultSet.next()) {
String ans =
resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString("root.vehicle.d0.s0");
Assert.assertEquals(res[cnt], ans);
cnt++;
}
- } finally {
- resultSet.close();
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ /**
+ * Test to check if timeValuePair is updated when there are unSeqPageReaders are unpacked in
+ * method hasNextOverlappedPage() in SeriesReader.java
+ */
+ @Test
+ public void selectOverlappedPageTest2() {
+ String[] res = {"0,10"};
+
+ try (Connection connection =
+ DriverManager.getConnection(
+ Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+
+ for (String insertSql : dataSet1) {
+ statement.execute(insertSql);
+ }
+
+ boolean hasResultSet = statement.execute("select count(s1) from root.sg1.d1");
+ Assert.assertTrue(hasResultSet);
+ int cnt = 0;
+ try (ResultSet resultSet = statement.getResultSet()) {
+ while (resultSet.next()) {
+ String ans =
+ resultSet.getString(TIMESTAMP_STR)
+ + ","
+ + resultSet.getString("count(root.sg1.d1.s1)");
+ Assert.assertEquals(res[cnt], ans);
+ cnt++;
+ }
+ Assert.assertEquals(res.length, cnt);
}
} catch (Exception e) {
e.printStackTrace();
[iotdb] 03/04: Fix group by data inconsistence bug (#3317)
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a commit to branch rel/0.12
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 6ffda68ca690f66b0b13cc7140ec617660521dbb
Author: Xiangwei Wei <34...@users.noreply.github.com>
AuthorDate: Thu Jun 3 10:22:35 2021 +0800
Fix group by data inconsistence bug (#3317)
Fix group by data inconsistence bug
(cherry picked from commit 9883106d9324067bcc5c302cd2fd901809cac695)
---
.../db/query/aggregation/impl/CountAggrResult.java | 24 ++---
.../iotdb/db/query/reader/series/SeriesReader.java | 3 +-
.../iotdb/db/integration/IoTDBGroupByUnseqIT.java | 103 ++++++++++++++++-----
3 files changed, 87 insertions(+), 43 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/aggregation/impl/CountAggrResult.java b/server/src/main/java/org/apache/iotdb/db/query/aggregation/impl/CountAggrResult.java
index 7b40fda..00bfcad 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/aggregation/impl/CountAggrResult.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/aggregation/impl/CountAggrResult.java
@@ -45,29 +45,25 @@ public class CountAggrResult extends AggregateResult {
@Override
public void updateResultFromStatistics(Statistics statistics) {
- long preValue = getLongValue();
- preValue += statistics.getCount();
- setLongValue(preValue);
+ setLongValue(getLongValue() + statistics.getCount());
}
@Override
public void updateResultFromPageData(BatchData dataInThisPage) {
- int cnt = dataInThisPage.length();
- long preValue = getLongValue();
- preValue += cnt;
- setLongValue(preValue);
+ setLongValue(getLongValue() + dataInThisPage.length());
}
@Override
public void updateResultFromPageData(BatchData dataInThisPage, long minBound, long maxBound) {
+ int cnt = 0;
while (dataInThisPage.hasCurrent()) {
if (dataInThisPage.currentTime() >= maxBound || dataInThisPage.currentTime() < minBound) {
break;
}
- long preValue = getLongValue();
- setLongValue(++preValue);
+ cnt++;
dataInThisPage.next();
}
+ setLongValue(getLongValue() + cnt);
}
@Override
@@ -80,10 +76,7 @@ public class CountAggrResult extends AggregateResult {
cnt++;
}
}
-
- long preValue = getLongValue();
- preValue += cnt;
- setLongValue(preValue);
+ setLongValue(getLongValue() + cnt);
}
@Override
@@ -94,10 +87,7 @@ public class CountAggrResult extends AggregateResult {
cnt++;
}
}
-
- long preValue = getLongValue();
- preValue += cnt;
- setLongValue(preValue);
+ setLongValue(getLongValue() + cnt);
}
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
index aa78aeb..dc3b4bb 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
@@ -295,11 +295,12 @@ public class SeriesReader {
* first time series metadata is already unpacked, consume cached ChunkMetadata
*/
if (!cachedChunkMetadata.isEmpty()) {
- firstChunkMetadata = cachedChunkMetadata.poll();
+ firstChunkMetadata = cachedChunkMetadata.peek();
unpackAllOverlappedTsFilesToTimeSeriesMetadata(
orderUtils.getOverlapCheckTime(firstChunkMetadata.getStatistics()));
unpackAllOverlappedTimeSeriesMetadataToCachedChunkMetadata(
orderUtils.getOverlapCheckTime(firstChunkMetadata.getStatistics()), false);
+ firstChunkMetadata = cachedChunkMetadata.poll();
}
}
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBGroupByUnseqIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBGroupByUnseqIT.java
index c8c69b3..e515612 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBGroupByUnseqIT.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBGroupByUnseqIT.java
@@ -33,14 +33,10 @@ import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.Statement;
+import static org.apache.iotdb.db.constant.TestConstant.TIMESTAMP_STR;
import static org.apache.iotdb.db.constant.TestConstant.count;
import static org.junit.Assert.fail;
-/**
- * This test contains one seq file and one unseq file. In the seq file, it contains two pages:
- * 1,2,3,4 in one page, 8,10,11,12 in another page. In the unseq file, it only contains one page: 7,
- * 9. The unseq page is overlapped with the second seq page.
- */
public class IoTDBGroupByUnseqIT {
private static String[] dataSet1 =
@@ -61,13 +57,29 @@ public class IoTDBGroupByUnseqIT {
"flush"
};
- private static final String TIMESTAMP_STR = "Time";
+ private static String[] dataSet2 =
+ new String[] {
+ "SET STORAGE GROUP TO root.sg2",
+ "CREATE TIMESERIES root.sg2.d1.s1 WITH DATATYPE=INT32, ENCODING=PLAIN",
+ "INSERT INTO root.sg2.d1(time,s1) values(1, 1)",
+ "INSERT INTO root.sg2.d1(time,s1) values(10, 10)",
+ "flush",
+ "INSERT INTO root.sg2.d1(time,s1) values(19, 19)",
+ "INSERT INTO root.sg2.d1(time,s1) values(30, 30)",
+ "flush",
+ "INSERT INTO root.sg2.d1(time,s1) values(5, 5)",
+ "INSERT INTO root.sg2.d1(time,s1) values(15, 15)",
+ "INSERT INTO root.sg2.d1(time,s1) values(26, 26)",
+ "INSERT INTO root.sg2.d1(time,s1) values(30, 30)",
+ "flush"
+ };
private boolean enableUnseqCompaction;
private int maxNumberOfPointsInPage;
@Before
public void setUp() throws Exception {
+ Class.forName(Config.JDBC_DRIVER_NAME);
EnvironmentUtils.closeStatMonitor();
enableUnseqCompaction = IoTDBDescriptor.getInstance().getConfig().isEnableUnseqCompaction();
IoTDBDescriptor.getInstance().getConfig().setEnableUnseqCompaction(false);
@@ -75,23 +87,6 @@ public class IoTDBGroupByUnseqIT {
TSFileDescriptor.getInstance().getConfig().getMaxNumberOfPointsInPage();
TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(4);
EnvironmentUtils.envSetUp();
- Class.forName(Config.JDBC_DRIVER_NAME);
- prepareData();
- }
-
- private void prepareData() {
- try (Connection connection =
- DriverManager.getConnection(
- Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
- Statement statement = connection.createStatement(); ) {
-
- for (String sql : dataSet1) {
- statement.execute(sql);
- }
-
- } catch (Exception e) {
- e.printStackTrace();
- }
}
@After
@@ -101,6 +96,11 @@ public class IoTDBGroupByUnseqIT {
TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(maxNumberOfPointsInPage);
}
+ /**
+ * This test contains one seq file and one unseq file. In the seq file, it contains two pages:
+ * 1,2,3,4 in one page, 8,10,11,12 in another page. In the unseq file, it only contains one page:
+ * 7, 9. The unseq page is overlapped with the second seq page.
+ */
@Test
public void test1() {
String[] retArray1 =
@@ -111,13 +111,17 @@ public class IoTDBGroupByUnseqIT {
try (Connection connection =
DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
Statement statement = connection.createStatement()) {
+
+ for (String sql : dataSet1) {
+ statement.execute(sql);
+ }
+
boolean hasResultSet =
statement.execute("select count(s1) from root.sg1.d1 group by ([1, 13), 3ms)");
Assert.assertTrue(hasResultSet);
- int cnt;
+ int cnt = 0;
try (ResultSet resultSet = statement.getResultSet()) {
- cnt = 0;
while (resultSet.next()) {
String ans =
resultSet.getString(TIMESTAMP_STR)
@@ -133,4 +137,53 @@ public class IoTDBGroupByUnseqIT {
fail(e.getMessage());
}
}
+
+ /**
+ * This test contains two seq files and one unseq file. In the first seq file, it contains two
+ * points: [1, 10]. In the second seq file, it contains two points: [19, 30]. In the unseq file,
+ * it contains two CHUNKS: [5, 15], [26, 30]. The unseq file is overlapped with two seq files.
+ * While the chunk [19,30] in the second seq file is unpacked, it should replace [26,30] as the
+ * first chunk.
+ */
+ @Test
+ public void test2() {
+ String[] retArray1 = new String[] {"5,1", "10,1", "15,2", "20,0", "25,1"};
+
+ int preAvgSeriesPointNumberThreshold =
+ IoTDBDescriptor.getInstance().getConfig().getAvgSeriesPointNumberThreshold();
+ try (Connection connection =
+ DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+
+ IoTDBDescriptor.getInstance().getConfig().setAvgSeriesPointNumberThreshold(2);
+
+ for (String sql : dataSet2) {
+ statement.execute(sql);
+ }
+
+ boolean hasResultSet =
+ statement.execute("select count(s1) from root.sg2.d1 group by ([5, 30), 5ms)");
+
+ Assert.assertTrue(hasResultSet);
+ int cnt = 0;
+ try (ResultSet resultSet = statement.getResultSet()) {
+ while (resultSet.next()) {
+ String ans =
+ resultSet.getString(TIMESTAMP_STR)
+ + ","
+ + resultSet.getString(count("root.sg2.d1.s1"));
+ Assert.assertEquals(retArray1[cnt], ans);
+ cnt++;
+ }
+ Assert.assertEquals(retArray1.length, cnt);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ } finally {
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .setAvgSeriesPointNumberThreshold(preAvgSeriesPointNumberThreshold);
+ }
+ }
}