You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by er...@apache.org on 2022/03/08 08:44:21 UTC
[iotdb] branch master updated: [IOTDB-2697] Add metadata modification validation before accelerating query by statistics (#5182)
This is an automated email from the ASF dual-hosted git repository.
ericpai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 5286d42 [IOTDB-2697] Add metadata modification validation before accelerating query by statistics (#5182)
5286d42 is described below
commit 5286d42d5bc5e2befcf3a254163e7c0d3394d286
Author: BaiJian <er...@hotmail.com>
AuthorDate: Tue Mar 8 16:43:33 2022 +0800
[IOTDB-2697] Add metadata modification validation before accelerating query by statistics (#5182)
---
.../IoTDBQueryWithComplexValueFilterIT.java | 64 ++++++++----------
.../IoTDBQueryWithRecreatedTimeseriesIT.java | 79 ++++++++++++++++++++++
.../iotdb/db/query/reader/series/SeriesReader.java | 2 +
3 files changed, 109 insertions(+), 36 deletions(-)
diff --git a/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBQueryWithComplexValueFilterIT.java b/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBQueryWithComplexValueFilterIT.java
index c104b89..4cb9c91 100644
--- a/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBQueryWithComplexValueFilterIT.java
+++ b/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBQueryWithComplexValueFilterIT.java
@@ -18,55 +18,53 @@
*/
package org.apache.iotdb.db.integration;
-import org.apache.iotdb.db.utils.EnvironmentUtils;
-import org.apache.iotdb.jdbc.Config;
+import org.apache.iotdb.integration.env.EnvFactory;
+import org.apache.iotdb.itbase.category.ClusterTest;
+import org.apache.iotdb.itbase.category.LocalStandaloneTest;
+import org.apache.iotdb.itbase.category.RemoteTest;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.junit.experimental.categories.Category;
import java.sql.Connection;
-import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.Statement;
import static org.junit.Assert.fail;
+@Category({LocalStandaloneTest.class, ClusterTest.class, RemoteTest.class})
public class IoTDBQueryWithComplexValueFilterIT {
@BeforeClass
public static void setUp() throws Exception {
- EnvironmentUtils.envSetUp();
+ EnvFactory.getEnv().initBeforeClass();
prepareData();
}
@AfterClass
public static void tearDown() throws Exception {
- EnvironmentUtils.cleanEnv();
+ EnvFactory.getEnv().cleanAfterClass();
}
@Test
- public void testRawQuery1() throws ClassNotFoundException {
- Class.forName(Config.JDBC_DRIVER_NAME);
- try (Connection connection =
- DriverManager.getConnection(
- Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ public void testRawQuery1() {
+ try (Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement()) {
boolean hasResultSet =
statement.execute(
"select s1 from root.sg1.d1 where (time > 400 and s1 <= 600) or (s2 > 300 and time <= 500)");
Assert.assertTrue(hasResultSet);
- ResultSet resultSet = statement.getResultSet();
- int cnt = 0;
-
- while (resultSet.next()) {
- cnt++;
+ try (ResultSet resultSet = statement.getResultSet()) {
+ int cnt = 0;
+ while (resultSet.next()) {
+ cnt++;
+ }
+ Assert.assertEquals(300, cnt);
}
-
- Assert.assertEquals(300, cnt);
-
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
@@ -74,46 +72,40 @@ public class IoTDBQueryWithComplexValueFilterIT {
}
@Test
- public void testRawQuery2() throws ClassNotFoundException {
- Class.forName(Config.JDBC_DRIVER_NAME);
- try (Connection connection =
- DriverManager.getConnection(
- Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ public void testRawQuery2() {
+ try (Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement()) {
boolean hasResultSet =
statement.execute(
"select s1 from root.sg1.d1 where (time > 400 and s1 <= 600) and (s2 > 300 and time <= 500)");
Assert.assertTrue(hasResultSet);
- ResultSet resultSet = statement.getResultSet();
- int cnt = 0;
-
- while (resultSet.next()) {
- cnt++;
+ try (ResultSet resultSet = statement.getResultSet()) {
+ int cnt = 0;
+ while (resultSet.next()) {
+ cnt++;
+ }
+ Assert.assertEquals(100, cnt);
}
- Assert.assertEquals(100, cnt);
-
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
- private static void prepareData() throws ClassNotFoundException {
- Class.forName(Config.JDBC_DRIVER_NAME);
- try (Connection connection =
- DriverManager.getConnection(
- Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ private static void prepareData() {
+ try (Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement()) {
statement.execute("create storage group root.sg1");
statement.execute("create timeseries root.sg1.d1.s1 with datatype=INT32,encoding=PLAIN");
statement.execute("create timeseries root.sg1.d1.s2 with datatype=DOUBLE,encoding=PLAIN");
for (int i = 0; i < 1000; i++) {
- statement.execute(
+ statement.addBatch(
String.format(
"insert into root.sg1.d1(time,s1,s2) values(%d,%d,%f)", i, i, (double) i));
}
+ statement.executeBatch();
} catch (Exception e) {
e.printStackTrace();
}
diff --git a/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBQueryWithRecreatedTimeseriesIT.java b/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBQueryWithRecreatedTimeseriesIT.java
new file mode 100644
index 0000000..a9de25a
--- /dev/null
+++ b/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBQueryWithRecreatedTimeseriesIT.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.integration;
+
+import org.apache.iotdb.integration.env.EnvFactory;
+import org.apache.iotdb.itbase.category.ClusterTest;
+import org.apache.iotdb.itbase.category.LocalStandaloneTest;
+import org.apache.iotdb.itbase.category.RemoteTest;
+
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.Statement;
+
+import static org.junit.Assert.fail;
+
+@Category({LocalStandaloneTest.class, ClusterTest.class, RemoteTest.class})
+public class IoTDBQueryWithRecreatedTimeseriesIT {
+ @BeforeClass
+ public static void setUp() throws Exception {
+ EnvFactory.getEnv().initBeforeClass();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ EnvFactory.getEnv().cleanAfterClass();
+ }
+
+ @Test
+ // https://issues.apache.org/jira/browse/IOTDB-2697
+ public void testQueryDiffTypeTimeseries() {
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute("SET STORAGE GROUP TO root.sg");
+ Thread.sleep(100);
+ statement.execute("CREATE TIMESERIES root.sg.d1.s1 with datatype=FLOAT,encoding=PLAIN");
+ statement.execute("CREATE TIMESERIES root.sg.d1.s2 with datatype=INT64,encoding=PLAIN");
+ statement.execute("INSERT INTO root.sg.d1(time, s1, s2) VALUES(11000, 10, 20)");
+ Thread.sleep(100);
+ statement.execute("FLUSH");
+ Thread.sleep(100);
+ statement.execute("DELETE TIMESERIES root.sg.d1.s1");
+ statement.execute("CREATE TIMESERIES root.sg.d1.s1 with datatype=INT32,encoding=PLAIN");
+ try (ResultSet resultSet =
+ statement.executeQuery("SELECT s1 FROM root.sg.d1 WHERE s1 > 10")) {
+ Assert.assertFalse(resultSet.next());
+ }
+
+ try (ResultSet resultSet =
+ statement.executeQuery("SELECT s1 FROM root.sg.d1 WHERE s1 <= 10")) {
+ Assert.assertFalse(resultSet.next());
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
index 02058aa..615c685 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
@@ -368,6 +368,7 @@ public class SeriesReader {
if (valueFilter != null
&& firstChunkMetadata != null
&& !isChunkOverlapped()
+ && !firstChunkMetadata.isModified()
&& !valueFilter.satisfy(firstChunkMetadata.getStatistics())) {
skipCurrentChunk();
}
@@ -1067,6 +1068,7 @@ public class SeriesReader {
if (valueFilter != null
&& firstTimeSeriesMetadata != null
&& !isFileOverlapped()
+ && !firstTimeSeriesMetadata.isModified()
&& !valueFilter.satisfy(firstTimeSeriesMetadata.getStatistics())) {
firstTimeSeriesMetadata = null;
}