You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2020/04/14 12:15:54 UTC
[incubator-iotdb] branch TyUnseqTimeSeriesMetadata created (now
247ab2c)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a change to branch TyUnseqTimeSeriesMetadata
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.
at 247ab2c Solve it
This branch includes the following new commits:
new 247ab2c Solve it
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-iotdb] 01/01: Solve it
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch TyUnseqTimeSeriesMetadata
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 247ab2cbeeca19d72ea86edf692be914ebe62e79
Author: JackieTien97 <Ja...@foxmail.com>
AuthorDate: Tue Apr 14 20:15:27 2020 +0800
Solve it
---
.../apache/iotdb/db/query/fill/PreviousFill.java | 6 +-
.../iotdb/db/query/reader/series/SeriesReader.java | 8 +-
.../org/apache/iotdb/db/utils/FileLoaderUtils.java | 5 +-
.../IoTDBMultiOverlappedChunkInUnseqIT.java | 109 +++++++++++++++++++++
4 files changed, 120 insertions(+), 8 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/fill/PreviousFill.java b/server/src/main/java/org/apache/iotdb/db/query/fill/PreviousFill.java
index b209c58..fdd9642 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/fill/PreviousFill.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/fill/PreviousFill.java
@@ -139,7 +139,7 @@ public class PreviousFill extends IFill {
TsFileResource resource = seqFileResource.get(index);
TimeseriesMetadata timeseriesMetadata =
FileLoaderUtils.loadTimeSeriesMetadata(
- resource, seriesPath, context, timeFilter, allSensors);
+ resource, seriesPath, context, timeFilter, allSensors, false);
if (timeseriesMetadata != null) {
if (timeseriesMetadata.getStatistics().canUseStatistics()
&& endtimeContainedByTimeFilter(timeseriesMetadata.getStatistics())) {
@@ -180,7 +180,7 @@ public class PreviousFill extends IFill {
}
TimeseriesMetadata timeseriesMetadata =
FileLoaderUtils.loadTimeSeriesMetadata(
- unseqFileResource.poll(), seriesPath, context, timeFilter, allSensors);
+ unseqFileResource.poll(), seriesPath, context, timeFilter, allSensors, true);
if (timeseriesMetadata != null && timeseriesMetadata.getStatistics().canUseStatistics()
&& lBoundTime <= timeseriesMetadata.getStatistics().getEndTime()) {
// The last timeseriesMetadata will be used as a pivot to filter the rest unseq files.
@@ -196,7 +196,7 @@ public class PreviousFill extends IFill {
&& (lBoundTime <= unseqFileResource.peek().getEndTimeMap().get(seriesPath.getDevice()))) {
TimeseriesMetadata timeseriesMetadata =
FileLoaderUtils.loadTimeSeriesMetadata(
- unseqFileResource.poll(), seriesPath, context, timeFilter, allSensors);
+ unseqFileResource.poll(), seriesPath, context, timeFilter, allSensors, true);
unseqTimeseriesMetadataList.add(timeseriesMetadata);
// update lBoundTime if current unseq timeseriesMetadata's last point is a valid result
if (timeseriesMetadata.getStatistics().canUseStatistics()
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
index 5590615..04c3ff9 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
@@ -543,7 +543,7 @@ class SeriesReader {
* Fill sequence TimeSeriesMetadata List until it is not empty
*/
while (seqTimeSeriesMetadata.isEmpty() && !seqFileResource.isEmpty()) {
- TimeseriesMetadata timeseriesMetadata = FileLoaderUtils.loadTimeSeriesMetadata(seqFileResource.remove(0), seriesPath, context, timeFilter, allSensors);
+ TimeseriesMetadata timeseriesMetadata = FileLoaderUtils.loadTimeSeriesMetadata(seqFileResource.remove(0), seriesPath, context, timeFilter, allSensors, false);
if (timeseriesMetadata != null) {
seqTimeSeriesMetadata.add(timeseriesMetadata);
}
@@ -553,7 +553,7 @@ class SeriesReader {
* Fill unSequence TimeSeriesMetadata Priority Queue until it is not empty
*/
while (unSeqTimeSeriesMetadata.isEmpty() && !unseqFileResource.isEmpty()) {
- TimeseriesMetadata timeseriesMetadata = FileLoaderUtils.loadTimeSeriesMetadata(unseqFileResource.remove(0), seriesPath, context, timeFilter, allSensors);
+ TimeseriesMetadata timeseriesMetadata = FileLoaderUtils.loadTimeSeriesMetadata(unseqFileResource.remove(0), seriesPath, context, timeFilter, allSensors, true);
if (timeseriesMetadata != null) {
unSeqTimeSeriesMetadata.add(timeseriesMetadata);
}
@@ -608,13 +608,13 @@ class SeriesReader {
private void unpackAllOverlappedTsFilesToTimeSeriesMetadata(long endTime) throws IOException {
while (!unseqFileResource.isEmpty() && endTime >= unseqFileResource.get(0).getStartTimeMap().get(seriesPath.getDevice())) {
- TimeseriesMetadata timeseriesMetadata = FileLoaderUtils.loadTimeSeriesMetadata(unseqFileResource.remove(0), seriesPath, context, timeFilter, allSensors);
+ TimeseriesMetadata timeseriesMetadata = FileLoaderUtils.loadTimeSeriesMetadata(unseqFileResource.remove(0), seriesPath, context, timeFilter, allSensors, true);
if (timeseriesMetadata != null) {
unSeqTimeSeriesMetadata.add(timeseriesMetadata);
}
}
while (!seqFileResource.isEmpty() && endTime >= seqFileResource.get(0).getStartTimeMap().get(seriesPath.getDevice())) {
- TimeseriesMetadata timeseriesMetadata = FileLoaderUtils.loadTimeSeriesMetadata(seqFileResource.remove(0), seriesPath, context, timeFilter, allSensors);
+ TimeseriesMetadata timeseriesMetadata = FileLoaderUtils.loadTimeSeriesMetadata(seqFileResource.remove(0), seriesPath, context, timeFilter, allSensors, false);
if (timeseriesMetadata != null) {
seqTimeSeriesMetadata.add(timeseriesMetadata);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
index 8884a7d..fa1f9bc 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
@@ -90,7 +90,7 @@ public class FileLoaderUtils {
* @param allSensors measurements queried at the same time of this device
*/
public static TimeseriesMetadata loadTimeSeriesMetadata(TsFileResource resource, Path seriesPath,
- QueryContext context, Filter timeFilter, Set<String> allSensors) throws IOException {
+ QueryContext context, Filter timeFilter, Set<String> allSensors, boolean isUnseq) throws IOException {
TimeseriesMetadata timeSeriesMetadata;
if (resource.isClosed()) {
timeSeriesMetadata = TimeSeriesMetadataCache.getInstance()
@@ -122,6 +122,9 @@ public class FileLoaderUtils {
return null;
}
}
+ if (timeSeriesMetadata != null && timeSeriesMetadata.getStatistics() != null && isUnseq) {
+ timeSeriesMetadata.getStatistics().setCanUseStatistics(false);
+ }
return timeSeriesMetadata;
}
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBMultiOverlappedChunkInUnseqIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBMultiOverlappedChunkInUnseqIT.java
new file mode 100644
index 0000000..294481d
--- /dev/null
+++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBMultiOverlappedChunkInUnseqIT.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.integration;
+
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.utils.EnvironmentUtils;
+import org.apache.iotdb.jdbc.Config;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.Statement;
+
+import static org.apache.iotdb.db.constant.TestConstant.count;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * Notice that, all test begins with "IoTDB" is integration test. All test which will start the
+ * IoTDB server should be defined as integration test.
+ */
+public class IoTDBMultiOverlappedChunkInUnseqIT {
+
+ private static long beforeMemtableSizeThreshold;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ EnvironmentUtils.closeStatMonitor();
+ beforeMemtableSizeThreshold = IoTDBDescriptor.getInstance().getConfig().getMemtableSizeThreshold();
+ IoTDBDescriptor.getInstance().getConfig().setMemtableSizeThreshold(1024);
+ EnvironmentUtils.envSetUp();
+ Class.forName(Config.JDBC_DRIVER_NAME);
+ insertData();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ // recovery value
+ EnvironmentUtils.cleanEnv();
+ IoTDBDescriptor.getInstance().getConfig().setMemtableSizeThreshold(beforeMemtableSizeThreshold);
+ }
+
+ @Test
+ public void selectOverlappedPageTest() {
+
+ try (Connection connection = DriverManager
+ .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+ String sql = "select count(s0) from root.vehicle.d0 where time < 1000000";
+ ResultSet resultSet = statement.executeQuery(sql);
+ while (resultSet.next()) {
+ String ans = resultSet.getString(count("root.vehicle.d0.s0"));
+ assertEquals("1000", ans);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ private static void insertData() {
+ try (Connection connection = DriverManager
+ .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+
+ statement.execute("CREATE TIMESERIES root.vehicle.d0.s0 WITH DATATYPE=INT32, ENCODING=RLE");
+
+ String sql = String
+ .format("insert into root.vehicle.d0(timestamp,s0) values(%s,%s)", 1000000, 1000000);
+ statement.execute(sql);
+
+ statement.execute("flush");
+ for (long time = 1; time <= 1000; time++) {
+ sql = String
+ .format("insert into root.vehicle.d0(timestamp,s0) values(%s,%s)", time, time);
+ statement.execute(sql);
+ }
+ for (long time = 2; time <= 1000; time++) {
+ sql = String
+ .format("insert into root.vehicle.d0(timestamp,s0) values(%s,%s)", time, 1000);
+ statement.execute(sql);
+ }
+ statement.execute("flush");
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+}
\ No newline at end of file