[GitHub] [incubator-iotdb] qiaojialin commented on a change in pull request #931: [IOTDB-540]Add time filter for LAST query and refactor Previous Fill

[GitBox <gi...@apache.org>]

qiaojialin commented on a change in pull request #931: [IOTDB-540]Add time filter for LAST query and refactor Previous Fill
URL: https://github.com/apache/incubator-iotdb/pull/931#discussion_r396964121
 
 

 ##########
 File path: server/src/main/java/org/apache/iotdb/db/query/executor/LastQueryExecutor.java
 ##########
 @@ -115,55 +116,47 @@ private TimeValuePair calculateLastPairForOneSeries(
     } catch (MetadataException e) {
       throw new QueryProcessException(e);
     }
-    if (((LeafMNode) node).getCachedLast() != null) {
+    if (((LeafMNode) node).getCachedLast() != null && timeFilter == null) {
       return ((LeafMNode) node).getCachedLast();
     }
 
-    QueryDataSource dataSource =
-        QueryResourceManager.getInstance().getQueryDataSource(seriesPath, context, null);
-
-    List<TsFileResource> seqFileResources = dataSource.getSeqResources();
-    List<TsFileResource> unseqFileResources = dataSource.getUnseqResources();
-
     TimeValuePair resultPair = new TimeValuePair(Long.MIN_VALUE, null);
-
-    if (!seqFileResources.isEmpty()) {
-      for (int i = seqFileResources.size() - 1; i >= 0; i--) {
-        List<ChunkMetaData> chunkMetadata =
-            FileLoaderUtils.loadChunkMetadataFromTsFileResource(
-                seqFileResources.get(i), seriesPath, context);
-        if (!chunkMetadata.isEmpty()) {
-          ChunkMetaData lastChunkMetaData = chunkMetadata.get(chunkMetadata.size() - 1);
-          Statistics chunkStatistics = lastChunkMetaData.getStatistics();
-          resultPair =
-              constructLastPair(
+    QueryDataSource queryDataSource =
+        QueryResourceManager.getInstance().getQueryDataSource(seriesPath, context, timeFilter);
+    timeFilter = queryDataSource.updateFilterUsingTTL(timeFilter);
+
+    InvertedSeriesReader seriesReader =
+        new InvertedSeriesReader(
+            seriesPath, tsDataType, context, queryDataSource, timeFilter, null, null);
+
+    while (seriesReader.hasNextChunk()) {
+      // cal by chunk statistics
+      if (seriesReader.canUseCurrentChunkStatistics()) {
+        Statistics chunkStatistics = seriesReader.currentChunkStatistics();
+        if (resultPair == null || resultPair.getTimestamp() < chunkStatistics.getEndTime()) {
+          resultPair = constructLastPair(
                   chunkStatistics.getEndTime(), chunkStatistics.getLastValue(), tsDataType);
-          break;
         }
-      }
-    }
+        seriesReader.skipCurrentChunk();
+      } else {
+        BatchData lastBatchData = null;
+        while (seriesReader.hasNextPage()) {
+          lastBatchData = seriesReader.nextPage();
+        }
 
-    long version = 0;
-    for (TsFileResource resource : unseqFileResources) {
-      if (resource.getEndTimeMap().get(seriesPath.getDevice()) < resultPair.getTimestamp()) {
-        continue;
-      }
-      List<ChunkMetaData> chunkMetadata =
-          FileLoaderUtils.loadChunkMetadataFromTsFileResource(resource, seriesPath, context);
-      for (ChunkMetaData chunkMetaData : chunkMetadata) {
-        if (chunkMetaData.getEndTime() == resultPair.getTimestamp()
-            && chunkMetaData.getVersion() > version) {
-          Statistics chunkStatistics = chunkMetaData.getStatistics();
-          resultPair =
-              constructLastPair(
-                  chunkStatistics.getEndTime(), chunkStatistics.getLastValue(), tsDataType);
-          version = chunkMetaData.getVersion();
+        if (lastBatchData != null) {
+          if (resultPair == null || resultPair.getTimestamp() < lastBatchData.getMaxTimestamp()) {
+            resultPair = new TimeValuePair(
+                lastBatchData.getMaxTimestamp(),
+                lastBatchData.getTsPrimitiveTypeByIndex(lastBatchData.length() - 1));
+          }
         }
       }
     }
 
     // Update cached last value with low priority
-    ((LeafMNode) node).updateCachedLast(resultPair, false, Long.MIN_VALUE);
+    if (timeFilter == null)
 
 Review comment:
   what if the filter is time > 0 ?
   or the latest time point is 10 and the filter is time< 20?

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services