You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2020/04/14 12:15:55 UTC
[incubator-iotdb] 01/01: Solve it
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch TyUnseqTimeSeriesMetadata
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 247ab2cbeeca19d72ea86edf692be914ebe62e79
Author: JackieTien97 <Ja...@foxmail.com>
AuthorDate: Tue Apr 14 20:15:27 2020 +0800
Solve it
---
.../apache/iotdb/db/query/fill/PreviousFill.java | 6 +-
.../iotdb/db/query/reader/series/SeriesReader.java | 8 +-
.../org/apache/iotdb/db/utils/FileLoaderUtils.java | 5 +-
.../IoTDBMultiOverlappedChunkInUnseqIT.java | 109 +++++++++++++++++++++
4 files changed, 120 insertions(+), 8 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/fill/PreviousFill.java b/server/src/main/java/org/apache/iotdb/db/query/fill/PreviousFill.java
index b209c58..fdd9642 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/fill/PreviousFill.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/fill/PreviousFill.java
@@ -139,7 +139,7 @@ public class PreviousFill extends IFill {
TsFileResource resource = seqFileResource.get(index);
TimeseriesMetadata timeseriesMetadata =
FileLoaderUtils.loadTimeSeriesMetadata(
- resource, seriesPath, context, timeFilter, allSensors);
+ resource, seriesPath, context, timeFilter, allSensors, false);
if (timeseriesMetadata != null) {
if (timeseriesMetadata.getStatistics().canUseStatistics()
&& endtimeContainedByTimeFilter(timeseriesMetadata.getStatistics())) {
@@ -180,7 +180,7 @@ public class PreviousFill extends IFill {
}
TimeseriesMetadata timeseriesMetadata =
FileLoaderUtils.loadTimeSeriesMetadata(
- unseqFileResource.poll(), seriesPath, context, timeFilter, allSensors);
+ unseqFileResource.poll(), seriesPath, context, timeFilter, allSensors, true);
if (timeseriesMetadata != null && timeseriesMetadata.getStatistics().canUseStatistics()
&& lBoundTime <= timeseriesMetadata.getStatistics().getEndTime()) {
// The last timeseriesMetadata will be used as a pivot to filter the rest unseq files.
@@ -196,7 +196,7 @@ public class PreviousFill extends IFill {
&& (lBoundTime <= unseqFileResource.peek().getEndTimeMap().get(seriesPath.getDevice()))) {
TimeseriesMetadata timeseriesMetadata =
FileLoaderUtils.loadTimeSeriesMetadata(
- unseqFileResource.poll(), seriesPath, context, timeFilter, allSensors);
+ unseqFileResource.poll(), seriesPath, context, timeFilter, allSensors, true);
unseqTimeseriesMetadataList.add(timeseriesMetadata);
// update lBoundTime if current unseq timeseriesMetadata's last point is a valid result
if (timeseriesMetadata.getStatistics().canUseStatistics()
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
index 5590615..04c3ff9 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
@@ -543,7 +543,7 @@ class SeriesReader {
* Fill sequence TimeSeriesMetadata List until it is not empty
*/
while (seqTimeSeriesMetadata.isEmpty() && !seqFileResource.isEmpty()) {
- TimeseriesMetadata timeseriesMetadata = FileLoaderUtils.loadTimeSeriesMetadata(seqFileResource.remove(0), seriesPath, context, timeFilter, allSensors);
+ TimeseriesMetadata timeseriesMetadata = FileLoaderUtils.loadTimeSeriesMetadata(seqFileResource.remove(0), seriesPath, context, timeFilter, allSensors, false);
if (timeseriesMetadata != null) {
seqTimeSeriesMetadata.add(timeseriesMetadata);
}
@@ -553,7 +553,7 @@ class SeriesReader {
* Fill unSequence TimeSeriesMetadata Priority Queue until it is not empty
*/
while (unSeqTimeSeriesMetadata.isEmpty() && !unseqFileResource.isEmpty()) {
- TimeseriesMetadata timeseriesMetadata = FileLoaderUtils.loadTimeSeriesMetadata(unseqFileResource.remove(0), seriesPath, context, timeFilter, allSensors);
+ TimeseriesMetadata timeseriesMetadata = FileLoaderUtils.loadTimeSeriesMetadata(unseqFileResource.remove(0), seriesPath, context, timeFilter, allSensors, true);
if (timeseriesMetadata != null) {
unSeqTimeSeriesMetadata.add(timeseriesMetadata);
}
@@ -608,13 +608,13 @@ class SeriesReader {
private void unpackAllOverlappedTsFilesToTimeSeriesMetadata(long endTime) throws IOException {
while (!unseqFileResource.isEmpty() && endTime >= unseqFileResource.get(0).getStartTimeMap().get(seriesPath.getDevice())) {
- TimeseriesMetadata timeseriesMetadata = FileLoaderUtils.loadTimeSeriesMetadata(unseqFileResource.remove(0), seriesPath, context, timeFilter, allSensors);
+ TimeseriesMetadata timeseriesMetadata = FileLoaderUtils.loadTimeSeriesMetadata(unseqFileResource.remove(0), seriesPath, context, timeFilter, allSensors, true);
if (timeseriesMetadata != null) {
unSeqTimeSeriesMetadata.add(timeseriesMetadata);
}
}
while (!seqFileResource.isEmpty() && endTime >= seqFileResource.get(0).getStartTimeMap().get(seriesPath.getDevice())) {
- TimeseriesMetadata timeseriesMetadata = FileLoaderUtils.loadTimeSeriesMetadata(seqFileResource.remove(0), seriesPath, context, timeFilter, allSensors);
+ TimeseriesMetadata timeseriesMetadata = FileLoaderUtils.loadTimeSeriesMetadata(seqFileResource.remove(0), seriesPath, context, timeFilter, allSensors, false);
if (timeseriesMetadata != null) {
seqTimeSeriesMetadata.add(timeseriesMetadata);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
index 8884a7d..fa1f9bc 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
@@ -90,7 +90,7 @@ public class FileLoaderUtils {
* @param allSensors measurements queried at the same time of this device
*/
public static TimeseriesMetadata loadTimeSeriesMetadata(TsFileResource resource, Path seriesPath,
- QueryContext context, Filter timeFilter, Set<String> allSensors) throws IOException {
+ QueryContext context, Filter timeFilter, Set<String> allSensors, boolean isUnseq) throws IOException {
TimeseriesMetadata timeSeriesMetadata;
if (resource.isClosed()) {
timeSeriesMetadata = TimeSeriesMetadataCache.getInstance()
@@ -122,6 +122,9 @@ public class FileLoaderUtils {
return null;
}
}
+ if (timeSeriesMetadata != null && timeSeriesMetadata.getStatistics() != null && isUnseq) {
+ timeSeriesMetadata.getStatistics().setCanUseStatistics(false);
+ }
return timeSeriesMetadata;
}
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBMultiOverlappedChunkInUnseqIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBMultiOverlappedChunkInUnseqIT.java
new file mode 100644
index 0000000..294481d
--- /dev/null
+++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBMultiOverlappedChunkInUnseqIT.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.integration;
+
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.utils.EnvironmentUtils;
+import org.apache.iotdb.jdbc.Config;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.Statement;
+
+import static org.apache.iotdb.db.constant.TestConstant.count;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * Notice that, all test begins with "IoTDB" is integration test. All test which will start the
+ * IoTDB server should be defined as integration test.
+ */
+public class IoTDBMultiOverlappedChunkInUnseqIT {
+
+ private static long beforeMemtableSizeThreshold;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ EnvironmentUtils.closeStatMonitor();
+ beforeMemtableSizeThreshold = IoTDBDescriptor.getInstance().getConfig().getMemtableSizeThreshold();
+ IoTDBDescriptor.getInstance().getConfig().setMemtableSizeThreshold(1024);
+ EnvironmentUtils.envSetUp();
+ Class.forName(Config.JDBC_DRIVER_NAME);
+ insertData();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ // recovery value
+ EnvironmentUtils.cleanEnv();
+ IoTDBDescriptor.getInstance().getConfig().setMemtableSizeThreshold(beforeMemtableSizeThreshold);
+ }
+
+ @Test
+ public void selectOverlappedPageTest() {
+
+ try (Connection connection = DriverManager
+ .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+ String sql = "select count(s0) from root.vehicle.d0 where time < 1000000";
+ ResultSet resultSet = statement.executeQuery(sql);
+ while (resultSet.next()) {
+ String ans = resultSet.getString(count("root.vehicle.d0.s0"));
+ assertEquals("1000", ans);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ private static void insertData() {
+ try (Connection connection = DriverManager
+ .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+
+ statement.execute("CREATE TIMESERIES root.vehicle.d0.s0 WITH DATATYPE=INT32, ENCODING=RLE");
+
+ String sql = String
+ .format("insert into root.vehicle.d0(timestamp,s0) values(%s,%s)", 1000000, 1000000);
+ statement.execute(sql);
+
+ statement.execute("flush");
+ for (long time = 1; time <= 1000; time++) {
+ sql = String
+ .format("insert into root.vehicle.d0(timestamp,s0) values(%s,%s)", time, time);
+ statement.execute(sql);
+ }
+ for (long time = 2; time <= 1000; time++) {
+ sql = String
+ .format("insert into root.vehicle.d0(timestamp,s0) values(%s,%s)", time, 1000);
+ statement.execute(sql);
+ }
+ statement.execute("flush");
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+}
\ No newline at end of file