You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2020/04/22 06:37:03 UTC
[incubator-iotdb] branch master updated: Add modified flag in
Metadatas and remove canUseStatistics() (#1083)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 11949b8 Add modified flag in Metadatas and remove canUseStatistics() (#1083)
11949b8 is described below
commit 11949b8957685abd98627ad7029ce3f9220be2c0
Author: wshao08 <59...@users.noreply.github.com>
AuthorDate: Wed Apr 22 14:36:53 2020 +0800
Add modified flag in Metadatas and remove canUseStatistics() (#1083)
* Add hasModification() in Metadatas and remove canUseStatistics() in Statistics
---
.../iotdb/db/query/executor/LastQueryExecutor.java | 2 +-
.../apache/iotdb/db/query/fill/PreviousFill.java | 8 +++---
.../iotdb/db/query/reader/chunk/MemPageReader.java | 5 ++++
.../query/reader/series/SeriesAggregateReader.java | 19 ++++++++------
.../iotdb/db/query/reader/series/SeriesReader.java | 29 ++++++++++++++++++++--
.../org/apache/iotdb/db/utils/FileLoaderUtils.java | 2 +-
.../java/org/apache/iotdb/db/utils/QueryUtils.java | 2 +-
.../iotdb/tsfile/file/header/PageHeader.java | 9 +++++++
.../iotdb/tsfile/file/metadata/ChunkMetadata.java | 10 ++++++++
.../tsfile/file/metadata/TimeseriesMetadata.java | 11 +++++++-
.../file/metadata/statistics/Statistics.java | 13 ----------
.../iotdb/tsfile/read/reader/IPageReader.java | 2 ++
.../tsfile/read/reader/chunk/ChunkReader.java | 2 +-
.../iotdb/tsfile/read/reader/page/PageReader.java | 5 ++++
14 files changed, 87 insertions(+), 32 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/executor/LastQueryExecutor.java b/server/src/main/java/org/apache/iotdb/db/query/executor/LastQueryExecutor.java
index 91cfb82..746cc65 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/executor/LastQueryExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/executor/LastQueryExecutor.java
@@ -134,7 +134,7 @@ public class LastQueryExecutor {
TimeseriesMetadata timeseriesMetadata = FileLoaderUtils.loadTimeSeriesMetadata(
seqFileResources.get(i), seriesPath, context, null, sensors);
if (timeseriesMetadata != null) {
- if (timeseriesMetadata.getStatistics().canUseStatistics()) {
+ if (!timeseriesMetadata.isModified()) {
Statistics timeseriesMetadataStats = timeseriesMetadata.getStatistics();
resultPair = constructLastPair(
timeseriesMetadataStats.getEndTime(),
diff --git a/server/src/main/java/org/apache/iotdb/db/query/fill/PreviousFill.java b/server/src/main/java/org/apache/iotdb/db/query/fill/PreviousFill.java
index 7b7e508..be88b15 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/fill/PreviousFill.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/fill/PreviousFill.java
@@ -176,12 +176,12 @@ public class PreviousFill extends IFill {
FileLoaderUtils.loadTimeSeriesMetadata(
unseqFileResource.poll(), seriesPath, context, timeFilter, allSensors);
- if (timeseriesMetadata == null || (timeseriesMetadata.getStatistics().canUseStatistics()
+ if (timeseriesMetadata == null || (!timeseriesMetadata.isModified()
&& timeseriesMetadata.getStatistics().getEndTime() < lBoundTime)) {
continue;
}
unseqTimeseriesMetadataList.add(timeseriesMetadata);
- if (timeseriesMetadata.getStatistics().canUseStatistics()) {
+ if (!timeseriesMetadata.isModified()) {
if (endtimeContainedByTimeFilter(timeseriesMetadata.getStatistics())) {
lBoundTime = Math.max(lBoundTime, timeseriesMetadata.getStatistics().getEndTime());
} else {
@@ -198,7 +198,7 @@ public class PreviousFill extends IFill {
}
Statistics chunkStatistics = chunkMetaData.getStatistics();
- if (chunkStatistics.canUseStatistics() && endtimeContainedByTimeFilter(chunkStatistics)) {
+ if (!chunkMetaData.isModified() && endtimeContainedByTimeFilter(chunkStatistics)) {
return constructLastPair(
chunkStatistics.getEndTime(), chunkStatistics.getLastValue(), dataType);
}
@@ -206,7 +206,7 @@ public class PreviousFill extends IFill {
for (int i = pageReaders.size() - 1; i >= 0; i--) {
IPageReader pageReader = pageReaders.get(i);
Statistics pageStatistics = pageReader.getStatistics();
- if (pageStatistics.canUseStatistics() && endtimeContainedByTimeFilter(pageStatistics)) {
+ if (!pageReader.isModified() && endtimeContainedByTimeFilter(pageStatistics)) {
lastPoint = constructLastPair(
pageStatistics.getEndTime(), pageStatistics.getLastValue(), dataType);
} else {
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/MemPageReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/MemPageReader.java
index e40e99c..a9106c5 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/MemPageReader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/MemPageReader.java
@@ -58,4 +58,9 @@ public class MemPageReader implements IPageReader {
public void setFilter(Filter filter) {
this.valueFilter = filter;
}
+
+ @Override
+ public boolean isModified() {
+ return false;
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesAggregateReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesAggregateReader.java
index 7f48b2f..be9afc4 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesAggregateReader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesAggregateReader.java
@@ -35,10 +35,11 @@ public class SeriesAggregateReader implements IAggregateReader {
private final SeriesReader seriesReader;
- public SeriesAggregateReader(Path seriesPath, Set<String> allSensors, TSDataType dataType, QueryContext context,
- QueryDataSource dataSource, Filter timeFilter, Filter valueFilter, TsFileFilter fileFilter) {
- this.seriesReader = new SeriesReader(seriesPath, allSensors, dataType, context, dataSource, timeFilter,
- valueFilter, fileFilter);
+ public SeriesAggregateReader(Path seriesPath, Set<String> allSensors, TSDataType dataType,
+ QueryContext context, QueryDataSource dataSource, Filter timeFilter, Filter valueFilter,
+ TsFileFilter fileFilter) {
+ this.seriesReader = new SeriesReader(seriesPath, allSensors, dataType, context, dataSource,
+ timeFilter, valueFilter, fileFilter);
}
@Override
@@ -49,7 +50,8 @@ public class SeriesAggregateReader implements IAggregateReader {
@Override
public boolean canUseCurrentFileStatistics() throws IOException {
Statistics fileStatistics = currentFileStatistics();
- return !seriesReader.isFileOverlapped() && containedByTimeFilter(fileStatistics) && fileStatistics.canUseStatistics();
+ return !seriesReader.isFileOverlapped() && containedByTimeFilter(fileStatistics)
+ && !seriesReader.currentFileModified();
}
@Override
@@ -70,7 +72,8 @@ public class SeriesAggregateReader implements IAggregateReader {
@Override
public boolean canUseCurrentChunkStatistics() throws IOException {
Statistics chunkStatistics = currentChunkStatistics();
- return !seriesReader.isChunkOverlapped() && containedByTimeFilter(chunkStatistics) && chunkStatistics.canUseStatistics();
+ return !seriesReader.isChunkOverlapped() && containedByTimeFilter(chunkStatistics)
+ && !seriesReader.currentChunkModified();
}
@Override
@@ -88,14 +91,14 @@ public class SeriesAggregateReader implements IAggregateReader {
return seriesReader.hasNextPage();
}
-
@Override
public boolean canUseCurrentPageStatistics() throws IOException {
Statistics currentPageStatistics = currentPageStatistics();
if (currentPageStatistics == null) {
return false;
}
- return !seriesReader.isPageOverlapped() && containedByTimeFilter(currentPageStatistics) && currentPageStatistics.canUseStatistics();
+ return !seriesReader.isPageOverlapped() && containedByTimeFilter(currentPageStatistics)
+ && !seriesReader.currentPageModified();
}
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
index cedd326..0988cd9 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
@@ -184,6 +184,13 @@ class SeriesReader {
return firstTimeSeriesMetadata.getStatistics();
}
+ boolean currentFileModified() throws IOException {
+ if (firstTimeSeriesMetadata == null) {
+ throw new IOException("no first file");
+ }
+ return firstTimeSeriesMetadata.isModified();
+ }
+
void skipCurrentFile() {
firstTimeSeriesMetadata = null;
}
@@ -266,6 +273,13 @@ class SeriesReader {
return firstChunkMetadata.getStatistics();
}
+ boolean currentChunkModified() throws IOException {
+ if (firstChunkMetadata == null) {
+ throw new IOException("no first chunk");
+ }
+ return firstChunkMetadata.isModified();
+ }
+
void skipCurrentChunk() {
firstChunkMetadata = null;
}
@@ -402,6 +416,13 @@ class SeriesReader {
return firstPageReader.getStatistics();
}
+ boolean currentPageModified() throws IOException {
+ if (firstPageReader == null) {
+ throw new IOException("no first page");
+ }
+ return firstPageReader.isModified();
+ }
+
void skipCurrentPage() {
firstPageReader = null;
}
@@ -584,7 +605,7 @@ class SeriesReader {
.loadTimeSeriesMetadata(unseqFileResource.remove(0), seriesPath, context, getAnyFilter(),
allSensors);
if (timeseriesMetadata != null) {
- timeseriesMetadata.getStatistics().setCanUseStatistics(false);
+ timeseriesMetadata.setModified(true);
unSeqTimeSeriesMetadata.add(timeseriesMetadata);
}
}
@@ -640,7 +661,7 @@ class SeriesReader {
while (!unseqFileResource.isEmpty() && endTime >= unseqFileResource.get(0).getStartTimeMap().get(seriesPath.getDevice())) {
TimeseriesMetadata timeseriesMetadata = FileLoaderUtils.loadTimeSeriesMetadata(unseqFileResource.remove(0), seriesPath, context, getAnyFilter(), allSensors);
if (timeseriesMetadata != null) {
- timeseriesMetadata.getStatistics().setCanUseStatistics(false);
+ timeseriesMetadata.setModified(true);
unSeqTimeSeriesMetadata.add(timeseriesMetadata);
}
}
@@ -693,5 +714,9 @@ class SeriesReader {
void setFilter(Filter filter) {
data.setFilter(filter);
}
+
+ boolean isModified() {
+ return data.isModified();
+ }
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
index b03360b..8f4661a 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
@@ -112,7 +112,7 @@ public class FileLoaderUtils {
if (timeSeriesMetadata != null) {
List<Modification> pathModifications =
context.getPathModifications(resource.getModFile(), seriesPath.getFullPath());
- timeSeriesMetadata.getStatistics().setCanUseStatistics(pathModifications.isEmpty());
+ timeSeriesMetadata.setModified(!pathModifications.isEmpty());
if (timeSeriesMetadata.getStatistics().getStartTime() > timeSeriesMetadata.getStatistics()
.getEndTime()) {
return null;
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java
index 48e4e84..7a3a6c5 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java
@@ -69,7 +69,7 @@ public class QueryUtils {
return true;
} else {
if (metaData.getDeletedAt() >= metaData.getStartTime()) {
- metaData.getStatistics().setCanUseStatistics(false);
+ metaData.setModified(true);
}
return false;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/PageHeader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/PageHeader.java
index a5589da..fa24f93 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/PageHeader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/PageHeader.java
@@ -33,6 +33,7 @@ public class PageHeader {
private int uncompressedSize;
private int compressedSize;
private Statistics statistics;
+ private boolean modified;
public PageHeader(int uncompressedSize, int compressedSize, Statistics statistics) {
this.uncompressedSize = uncompressedSize;
@@ -102,4 +103,12 @@ public class PageHeader {
return "PageHeader{" + "uncompressedSize=" + uncompressedSize + ", compressedSize="
+ compressedSize + ", statistics=" + statistics + "}";
}
+
+ public boolean isModified() {
+ return modified;
+ }
+
+ public void setModified(boolean modified) {
+ this.modified = modified;
+ }
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
index a15fb76..12d96c7 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
@@ -53,6 +53,8 @@ public class ChunkMetadata {
*/
private long deletedAt = Long.MIN_VALUE;
+ private boolean modified;
+
/**
* ChunkLoader of metadata, used to create ChunkReaderWrap
*/
@@ -199,4 +201,12 @@ public class ChunkMetadata {
return Objects.hash(measurementUid, deletedAt, tsDataType, statistics,
version, offsetOfChunkHeader);
}
+
+ public boolean isModified() {
+ return modified;
+ }
+
+ public void setModified(boolean modified) {
+ this.modified = modified;
+ }
}
\ No newline at end of file
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
index f6051bf..e0fa68f 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
@@ -38,7 +38,8 @@ public class TimeseriesMetadata {
private TSDataType tsDataType;
private Statistics<?> statistics;
-
+// modified is true when there are modifications of the series, or from unseq file
+ private boolean modified;
private IChunkMetadataLoader chunkMetadataLoader;
@@ -116,4 +117,12 @@ public class TimeseriesMetadata {
public List<ChunkMetadata> loadChunkMetadataList() throws IOException {
return chunkMetadataLoader.loadChunkMetadataList();
}
+
+ public boolean isModified() {
+ return modified;
+ }
+
+ public void setModified(boolean modified) {
+ this.modified = modified;
+ }
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java
index a833086..9be295f 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java
@@ -55,11 +55,6 @@ public abstract class Statistics<T> {
private long endTime = Long.MIN_VALUE;
/**
- * If the statistics has been modified, it can't be used.
- */
- private boolean canUseStatistics = true;
-
- /**
* static method providing statistic instance for respective data type.
*
* @param type - data type
@@ -416,14 +411,6 @@ public abstract class Statistics<T> {
this.count = count;
}
- public boolean canUseStatistics() {
- return canUseStatistics;
- }
-
- public void setCanUseStatistics(boolean canUseStatistics) {
- this.canUseStatistics = canUseStatistics;
- }
-
@Override
public String toString() {
return "startTime: " + startTime + " endTime: " + endTime + " count: " + count;
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/IPageReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/IPageReader.java
index ac5d29e..03a7dba 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/IPageReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/IPageReader.java
@@ -31,4 +31,6 @@ public interface IPageReader {
Statistics getStatistics();
void setFilter(Filter filter);
+
+ boolean isModified();
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReader.java
index 86f74ee..b6680f5 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReader.java
@@ -121,7 +121,7 @@ public class ChunkReader implements IChunkReader {
if (pageHeader.getEndTime() <= deletedAt) {
return false;
} else if (pageHeader.getStartTime() <= deletedAt) {
- pageHeader.getStatistics().setCanUseStatistics(false);
+ pageHeader.setModified(true);
}
return filter == null || filter.satisfy(pageHeader.getStatistics());
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
index 0a35554..f95c8e3 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
@@ -162,4 +162,9 @@ public class PageReader implements IPageReader {
public void setDeletedAt(long deletedAt) {
this.deletedAt = deletedAt;
}
+
+ @Override
+ public boolean isModified() {
+ return pageHeader.isModified();
+ }
}