You are viewing a plain text version of this content. The canonical link for it is here.
Posted to reviews@iotdb.apache.org by GitBox <gi...@apache.org> on 2020/04/10 11:06:07 UTC
[GitHub] [incubator-iotdb] wshao08 commented on a change in pull request
#931: [IOTDB-540] Accelerate Previous Fill
wshao08 commented on a change in pull request #931: [IOTDB-540] Accelerate Previous Fill
URL: https://github.com/apache/incubator-iotdb/pull/931#discussion_r406711350
##########
File path: server/src/main/java/org/apache/iotdb/db/query/fill/PreviousFill.java
##########
@@ -59,28 +84,183 @@ public long getBeforeRange() {
return beforeRange;
}
+ @Override
+ public void configureFill(Path path, TSDataType dataType, long queryTime,
+ Set<String> sensors, QueryContext context)
+ throws StorageEngineException, QueryProcessException {
+ this.seriesPath = path;
+ this.dataType = dataType;
+ this.context = context;
+ this.queryTime = queryTime;
+ this.allSensors = sensors;
+ this.timeFilter = constructFilter();
+ this.dataSource = QueryResourceManager.getInstance().getQueryDataSource(path, context, timeFilter);
+ // update filter by TTL
+ timeFilter = dataSource.updateFilterUsingTTL(timeFilter);
+ }
+
@Override
public TimeValuePair getFillResult() throws IOException {
- TimeValuePair beforePair = null;
- TimeValuePair cachedPair;
- while (batchData.hasCurrent() || allDataReader.hasNextBatch()) {
- if (!batchData.hasCurrent() && allDataReader.hasNextBatch()) {
- batchData = allDataReader.nextBatch();
+ TimeValuePair lastPointResult = retrieveValidLastPointFromSeqFiles();
+ UnpackOverlappedUnseqFiles(lastPointResult.getTimestamp());
+
+ long lastVersion = 0;
+ PriorityQueue<ChunkMetadata> sortedChunkMetatdataList = sortUnseqChunkMetadatasByEndtime();
+ while (!sortedChunkMetatdataList.isEmpty()
+ && lastPointResult.getTimestamp() <= sortedChunkMetatdataList.peek().getEndTime()) {
+ ChunkMetadata chunkMetadata = sortedChunkMetatdataList.poll();
+ TimeValuePair lastChunkPoint = getChunkLastPoint(chunkMetadata);
+ if (shouldUpdate(lastPointResult.getTimestamp(), lastVersion,
+ lastChunkPoint.getTimestamp(), chunkMetadata.getVersion())) {
+ lastPointResult = lastChunkPoint;
+ lastVersion = chunkMetadata.getVersion();
}
- cachedPair = new TimeValuePair(batchData.currentTime(), batchData.currentTsPrimitiveType());
- batchData.next();
- if (cachedPair.getTimestamp() <= queryTime) {
- beforePair = cachedPair;
- } else {
+ }
+ return lastPointResult;
+ }
+
+ /** Pick up and cache the last sequence TimeseriesMetadata that satisfies timeFilter */
+ private TimeValuePair retrieveValidLastPointFromSeqFiles() throws IOException {
+ List<TsFileResource> seqFileResource = dataSource.getSeqResources();
+ TimeValuePair lastPoint = new TimeValuePair(Long.MIN_VALUE, null);
+ for (int index = seqFileResource.size() - 1; index >= 0; index--) {
+ TsFileResource resource = seqFileResource.get(index);
+ TimeseriesMetadata timeseriesMetadata =
+ FileLoaderUtils.loadTimeSeriesMetadata(
+ resource, seriesPath, context, timeFilter, allSensors);
+ if (timeseriesMetadata != null) {
+ if (endtimeContainedByTimeFilter(timeseriesMetadata.getStatistics())) {
+ return constructLastPair(
+ timeseriesMetadata.getStatistics().getEndTime(),
+ timeseriesMetadata.getStatistics().getLastValue(),
+ dataType);
+ } else {
+ List<ChunkMetadata> seqChunkMetadataList =
+ FileLoaderUtils.loadChunkMetadataList(timeseriesMetadata);
+
+ for (int i = seqChunkMetadataList.size() - 1; i >= 0; i--) {
+ lastPoint = getChunkLastPoint(seqChunkMetadataList.get(i));
+ // last point of this sequence chunk is valid, quit the loop
+ if (lastPoint.getValue() != null) {
+ return lastPoint;
+ }
+ }
+ }
+ }
+ }
+
+ return lastPoint;
+ }
+
+ /**
+ * find the last TimeseriesMetadata and unpack all overlapped seq/unseq files
+ */
+ private void UnpackOverlappedUnseqFiles(long lBoundTime) throws IOException {
+ PriorityQueue<TsFileResource> unseqFileResource =
+ sortUnSeqFileResourcesInDecendingOrder(dataSource.getUnseqResources());
+
+ while (!unseqFileResource.isEmpty()) {
+ // The very end time of unseq files is smaller than lBoundTime,
+ // then skip all the rest unseq files
+ if (unseqFileResource.peek().getEndTimeMap().get(seriesPath.getDevice()) < lBoundTime) {
+ return;
+ }
+ TimeseriesMetadata timeseriesMetadata =
+ FileLoaderUtils.loadTimeSeriesMetadata(
+ unseqFileResource.poll(), seriesPath, context, timeFilter, allSensors);
+ if (timeseriesMetadata != null
+ && lBoundTime <= timeseriesMetadata.getStatistics().getEndTime()) {
+ lBoundTime = Math.max(lBoundTime, timeseriesMetadata.getStatistics().getStartTime());
+ unseqTimeseriesMetadataList.add(timeseriesMetadata);
break;
}
}
- if (beforePair != null) {
- beforePair.setTimestamp(queryTime);
- } else {
- beforePair = new TimeValuePair(queryTime, null);
+ // unpack all overlapped unseq files and fill unseqTimeseriesMetadata list
+ while (!unseqFileResource.isEmpty()
+ && (lBoundTime <= unseqFileResource.peek().getEndTimeMap().get(seriesPath.getDevice()))) {
+ TimeseriesMetadata timeseriesMetadata =
+ FileLoaderUtils.loadTimeSeriesMetadata(
+ unseqFileResource.poll(), seriesPath, context, timeFilter, allSensors);
+ unseqTimeseriesMetadataList.add(timeseriesMetadata);
+ // current unseq timeseriesMetadata's last point is a valid result,
+ // then skip the rest unseq files
+ if (endtimeContainedByTimeFilter(timeseriesMetadata.getStatistics())
+ && timeseriesMetadata.getStatistics().getEndTime()
+ > unseqFileResource.peek().getEndTimeMap().get(seriesPath.getDevice())) {
+ break;
+ }
}
- return beforePair;
+ }
+
+ private TimeValuePair getChunkLastPoint(ChunkMetadata chunkMetaData) throws IOException {
+ TimeValuePair lastPoint = new TimeValuePair(Long.MIN_VALUE, null);
+ Statistics chunkStatistics = chunkMetaData.getStatistics();
+
+ if (endtimeContainedByTimeFilter(chunkStatistics)) {
+ return constructLastPair(
+ chunkStatistics.getEndTime(), chunkStatistics.getLastValue(), dataType);
+ }
+ List<IPageReader> pageReaders = FileLoaderUtils.loadPageReaderList(chunkMetaData, timeFilter);
+ for (int i = pageReaders.size() - 1; i >= 0; i--) {
+ IPageReader pageReader = pageReaders.get(i);
+ Statistics pageStatistics = pageReader.getStatistics();
+ if (endtimeContainedByTimeFilter(pageStatistics)) {
+ lastPoint = constructLastPair(
+ pageStatistics.getEndTime(), pageStatistics.getLastValue(), dataType);
+ } else {
+ BatchData batchData = pageReader.getAllSatisfiedPageData();
+ lastPoint = batchData.getLastPairBeforeOrEqualTimestamp(queryTime);
+ }
+ break;
+ }
Review comment:
Fixed
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services