[incubator-iotdb] branch dev_new_merge_strategy updated: optimize cost calculation

[ji...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

jiangtian pushed a commit to branch dev_new_merge_strategy
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git


The following commit(s) were added to refs/heads/dev_new_merge_strategy by this push:
     new 3747c79  optimize cost calculation
3747c79 is described below

commit 3747c79820c19a71c4bf4d51a4dcce702e46a8b6
Author: jt2594838 <jt...@163.com>
AuthorDate: Sat Nov 23 00:08:33 2019 +0800

    optimize cost calculation
---
 .../iotdb/db/engine/merge/BaseFileSelector.java    |  1 +
 .../squeeze/selector/SqueezeMaxFileSelector.java   |  1 +
 .../java/org/apache/iotdb/db/utils/MergeUtils.java | 49 ++++++++++------------
 3 files changed, 23 insertions(+), 28 deletions(-)

diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/BaseFileSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/BaseFileSelector.java
index c244a99..4fa0708 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/merge/BaseFileSelector.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/BaseFileSelector.java
@@ -152,6 +152,7 @@ public abstract class BaseFileSelector implements IMergeFileSelector{
       long[] chunkNums = MergeUtils.findTotalAndLargestSeriesChunkNum(seqFile, resource.getFileReader(seqFile));
       long totalChunkNum = chunkNums[0];
       long maxChunkNum = chunkNums[1];
+      logger.debug("File {} has {} chunks, max chunk num {}", seqFile, totalChunkNum, maxChunkNum);
       cost = measurement.measure(seqFile) * maxChunkNum / totalChunkNum;
       maxSeriesQueryCostMap.put(seqFile, cost);
       logger.debug(LOG_FILE_COST, seqFile, cost);
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/squeeze/selector/SqueezeMaxFileSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/squeeze/selector/SqueezeMaxFileSelector.java
index cbf1748..3a5b08e 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/merge/squeeze/selector/SqueezeMaxFileSelector.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/squeeze/selector/SqueezeMaxFileSelector.java
@@ -142,6 +142,7 @@ public class SqueezeMaxFileSelector extends BaseFileSelector {
     long fileCost = 0;
     long fileReadCost = useTightBound ? calculateTightSeqMemoryCost(seqFile) :
         calculateMetadataSize(seqFile);
+    logger.debug("File read cost of {} is {}", seqFile, fileReadCost);
     if (fileReadCost > tempMaxSeqFileCost) {
       // memory used when read data from a seq file:
       // only one file will be read at the same time, so only the largest one is recorded here
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/MergeUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/MergeUtils.java
index 9e01aa6..038c7c6 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/MergeUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/MergeUtils.java
@@ -20,19 +20,13 @@
 package org.apache.iotdb.db.utils;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Map;
-import java.util.PriorityQueue;
-import java.util.Set;
+import java.util.*;
+import java.util.function.BiFunction;
+
 import org.apache.iotdb.db.engine.merge.manage.MergeResource;
 import org.apache.iotdb.db.engine.modification.Modification;
 import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
-import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
-import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadataIndex;
-import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData;
+import org.apache.iotdb.tsfile.file.metadata.*;
 import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
 import org.apache.iotdb.tsfile.read.common.BatchData;
 import org.apache.iotdb.tsfile.read.common.Chunk;
@@ -76,19 +70,6 @@ public class MergeUtils {
     }
   }
 
-  private static List<Path> collectFileSeries(TsFileSequenceReader sequenceReader) throws IOException {
-    TsFileMetaData metaData = sequenceReader.readFileMetadata();
-    Set<String> deviceIds = metaData.getDeviceMap().keySet();
-    Set<String> measurements = metaData.getMeasurementSchema().keySet();
-    List<Path> paths = new ArrayList<>();
-    for (String deviceId : deviceIds) {
-      for (String measurement : measurements) {
-        paths.add(new Path(deviceId, measurement));
-      }
-    }
-    return paths;
-  }
-
   public static long collectFileSizes(List<TsFileResource> seqFiles, List<TsFileResource> unseqFiles) {
     long totalSize = 0;
     for (TsFileResource tsFileResource : seqFiles) {
@@ -144,12 +125,24 @@ public class MergeUtils {
       throws IOException {
     long totalChunkNum = 0;
     long maxChunkNum = Long.MIN_VALUE;
-    List<Path> paths = collectFileSeries(sequenceReader);
+    Map<String, Long> seriesChunkNums = new HashMap<>();
+    TsFileMetaData metaData = sequenceReader.readFileMetadata();
+    for (TsDeviceMetadataIndex tsDeviceMetadataIndex : metaData.getDeviceMap().values()) {
+      TsDeviceMetadata deviceMetadata = sequenceReader.readTsDeviceMetaData(tsDeviceMetadataIndex);
+      List<ChunkGroupMetaData> chunkGroupMetaDataList = deviceMetadata.getChunkGroupMetaDataList();
+      if (!chunkGroupMetaDataList.isEmpty()) {
+        String device = chunkGroupMetaDataList.get(0).getDeviceID();
+        for (ChunkGroupMetaData chunkGroupMetaData : chunkGroupMetaDataList) {
+          for (ChunkMetaData chunkMetaData : chunkGroupMetaData.getChunkMetaDataList()) {
+            seriesChunkNums.compute(device + chunkMetaData.getMeasurementUid(), (s, aLong) -> aLong == null ? 1 : aLong + 1);
+            totalChunkNum ++;
+          }
+        }
+      }
+    }
 
-    for (Path path : paths) {
-      List<ChunkMetaData> chunkMetaDataList = sequenceReader.getChunkMetadataList(path);
-      totalChunkNum += chunkMetaDataList.size();
-      maxChunkNum = chunkMetaDataList.size() > maxChunkNum ? chunkMetaDataList.size() : maxChunkNum;
+    for (Map.Entry<String, Long> stringLongEntry : seriesChunkNums.entrySet()) {
+      maxChunkNum = Math.max(maxChunkNum, stringLongEntry.getValue());
     }
     logger.debug("In file {}, total chunk num {}, series max chunk num {}", tsFileResource,
         totalChunkNum, maxChunkNum);