You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by su...@apache.org on 2019/07/04 07:25:54 UTC

[incubator-iotdb] 01/01: add meta data cache in query

This is an automated email from the ASF dual-hosted git repository.

suyue pushed a commit to branch feature_query_cache
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git

commit 4380ab3ce1fdca3f6d361069ebd15bf3d648d0c4
Author: suyue <23...@qq.com>
AuthorDate: Thu Jul 4 15:24:47 2019 +0800

    add meta data cache in query
---
 .../iotdb/db/engine/cache/DeviceMetaDataCache.java | 55 ++++++++++++-------
 .../org/apache/iotdb/db/engine/cache/Test.java     | 64 ++++++++++++++++++++++
 .../iotdb/db/engine/cache/TsFileMetadataUtils.java | 50 +++++++++++++----
 .../apache/iotdb/db/engine/cache/TsResource.java   | 16 ++++++
 .../iotdb/db/query/control/JobFileManager.java     |  4 +-
 .../db/query/factory/SeriesReaderFactoryImpl.java  |  7 +--
 .../reader/sequence/SequenceSeriesReader.java      |  6 +-
 .../sequence/SequenceSeriesReaderByTimestamp.java  |  5 +-
 8 files changed, 164 insertions(+), 43 deletions(-)

diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
index 994622a..cc06cec 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
@@ -19,12 +19,17 @@
 package org.apache.iotdb.db.engine.cache;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicLong;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
 import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadata;
 import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData;
+import org.apache.iotdb.tsfile.read.common.Path;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -37,9 +42,9 @@ public class DeviceMetaDataCache {
 
   private static final int CACHE_SIZE = 100;
   /**
-   * key: the file path + deviceId.
+   * key: the file path. value: key: series path, value: list of chunkMetaData.
    */
-  private LinkedHashMap<String, TsDeviceMetadata> lruCache;
+  private LinkedHashMap<String, ConcurrentHashMap<Path, List<ChunkMetaData>>> lruCache;
 
   private AtomicLong cacheHintNum = new AtomicLong();
   private AtomicLong cacheRequestNum = new AtomicLong();
@@ -55,15 +60,13 @@ public class DeviceMetaDataCache {
   /**
    * get {@link TsDeviceMetadata}. THREAD SAFE.
    */
-  public TsDeviceMetadata get(String filePath, String deviceId, TsFileMetaData fileMetaData)
+  public List<ChunkMetaData> get(String filePath, Path seriesPath)
       throws IOException {
-    // The key(the tsfile path and deviceId) for the lruCache
 
-    String jointPath = filePath + deviceId;
-    Object jointPathObject = jointPath.intern();
+    Object jointPathObject = filePath.intern();
     synchronized (lruCache) {
       cacheRequestNum.incrementAndGet();
-      if (lruCache.containsKey(jointPath)) {
+      if (lruCache.containsKey(filePath) && lruCache.get(filePath).containsKey(seriesPath)) {
         cacheHintNum.incrementAndGet();
         if (logger.isDebugEnabled()) {
           logger.debug(
@@ -71,29 +74,36 @@ public class DeviceMetaDataCache {
                   + "the number of hints for cache is {}",
               cacheRequestNum.get(), cacheHintNum.get());
         }
-        return lruCache.get(jointPath);
+        return new ArrayList<>(lruCache.get(filePath).get(seriesPath));
       }
     }
     synchronized (jointPathObject) {
       synchronized (lruCache) {
-        if (lruCache.containsKey(jointPath)) {
-          return lruCache.get(jointPath);
+        if (lruCache.containsKey(filePath) && lruCache.get(filePath).containsKey(seriesPath)) {
+          return new ArrayList<>(lruCache.get(filePath).get(seriesPath));
         }
       }
       if (logger.isDebugEnabled()) {
         logger.debug("Cache didn't hint: the number of requests for cache is {}",
             cacheRequestNum.get());
       }
+      TsFileMetaData fileMetaData = TsFileMetaDataCache.getInstance().get(filePath);
       TsDeviceMetadata blockMetaData = TsFileMetadataUtils
-          .getTsRowGroupBlockMetaData(filePath, deviceId,
+          .getTsRowGroupBlockMetaData(filePath, seriesPath.getDevice(),
               fileMetaData);
+      ConcurrentHashMap<Path, List<ChunkMetaData>> chunkMetaData = TsFileMetadataUtils
+          .getChunkMetaDataList(blockMetaData);
       synchronized (lruCache) {
-        lruCache.put(jointPath, blockMetaData);
-        return lruCache.get(jointPath);
+        lruCache.put(filePath, chunkMetaData);
+        if(lruCache.get(filePath).containsKey(seriesPath)){
+          return new ArrayList<>(lruCache.get(filePath).get(seriesPath));
+        }
+        return new ArrayList<>();
       }
     }
   }
 
+
   /**
    * clear LRUCache.
    */
@@ -115,9 +125,9 @@ public class DeviceMetaDataCache {
   /**
    * This class is a map used to cache the <code>RowGroupBlockMetaData</code>. The caching strategy
    * is LRU.
-   *
    */
-  private class LruLinkedHashMap extends LinkedHashMap<String, TsDeviceMetadata> {
+  private class LruLinkedHashMap extends
+      LinkedHashMap<String, ConcurrentHashMap<Path, List<ChunkMetaData>>> {
 
     private static final long serialVersionUID = 1290160928914532649L;
     private static final float LOAD_FACTOR_MAP = 0.75f;
@@ -129,7 +139,8 @@ public class DeviceMetaDataCache {
     }
 
     @Override
-    protected boolean removeEldestEntry(Map.Entry<String, TsDeviceMetadata> eldest) {
+    protected boolean removeEldestEntry(
+        Map.Entry<String, ConcurrentHashMap<Path, List<ChunkMetaData>>> eldest) {
       return size() > maxCapacity;
     }
 
@@ -152,12 +163,16 @@ public class DeviceMetaDataCache {
 
   @Override
   public boolean equals(Object o) {
-    if (this == o) return true;
-    if (o == null || getClass() != o.getClass()) return false;
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
     DeviceMetaDataCache that = (DeviceMetaDataCache) o;
     return Objects.equals(lruCache, that.lruCache) &&
-            Objects.equals(cacheHintNum, that.cacheHintNum) &&
-            Objects.equals(cacheRequestNum, that.cacheRequestNum);
+        Objects.equals(cacheHintNum, that.cacheHintNum) &&
+        Objects.equals(cacheRequestNum, that.cacheRequestNum);
   }
 
   @Override
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/Test.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/Test.java
new file mode 100644
index 0000000..c1b17fa
--- /dev/null
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/Test.java
@@ -0,0 +1,64 @@
+package org.apache.iotdb.db.engine.cache;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.Random;
+import org.apache.iotdb.tsfile.read.common.Path;
+
+public class Test {
+
+  public static void main(String[] args) {
+    File tmp = new File("");
+    Path path = new Path("d.s");
+    Random random = new Random();
+
+    HashMap<String, String> stringHashMap = new HashMap<>(10_000_000);
+    HashMap<Long, String> objectHashMap = new HashMap<>(10_000_000);
+    for (int i = 0; i < 10_000_000; i++) {
+      stringHashMap.put(genStr(), "abc");
+      objectHashMap.put(random.nextLong(), "abc");
+    }
+
+    System.out.println("init data finish!");
+    long t1 = System.currentTimeMillis();
+    for (String str : stringHashMap.keySet()) {
+      stringHashMap.get(str);
+    }
+    long t2 = System.currentTimeMillis();
+    System.out.println("str map time = " + (t2 - t1) + "ms");
+    long t3 = System.currentTimeMillis();
+    for (Long obj : objectHashMap.keySet()) {
+      objectHashMap.get(obj);
+    }
+    long t4 = System.currentTimeMillis();
+    System.out
+        .println("obj map time = " + (t4 - t3) + "ms" + ". obj.size = " + objectHashMap.size());
+
+    t3 = System.currentTimeMillis();
+    for (Long obj : objectHashMap.keySet()) {
+      objectHashMap.get(obj);
+    }
+    t4 = System.currentTimeMillis();
+    System.out.println("obj map time = " + (t4 - t3) + "ms");
+    t1 = System.currentTimeMillis();
+    for (String str : stringHashMap.keySet()) {
+      stringHashMap.get(str);
+    }
+    t2 = System.currentTimeMillis();
+    System.out.println("str map time = " + (t2 - t1) + "ms");
+
+
+  }
+
+
+  static String genStr() {
+    StringBuilder builder = new StringBuilder();
+    Random random = new Random();
+    for (int j = 0; j < 32; j++) {
+      char ch = (char) (random.nextInt(27) + 'a');
+      builder.append(ch);
+    }
+    return builder.reverse().toString();
+  }
+
+}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java
index ea63b37..49c42b0 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java
@@ -19,10 +19,17 @@
 package org.apache.iotdb.db.engine.cache;
 
 import java.io.IOException;
-import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import org.apache.iotdb.db.query.control.FileReaderManager;
+import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetaData;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
 import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadata;
+import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadataIndex;
 import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData;
 import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
+import org.apache.iotdb.tsfile.read.common.Path;
 
 /**
  * This class is used to read metadata(<code>TsFileMetaData</code> and
@@ -30,7 +37,7 @@ import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
  */
 public class TsFileMetadataUtils {
 
-  private TsFileMetadataUtils(){
+  private TsFileMetadataUtils() {
 
   }
 
@@ -41,9 +48,8 @@ public class TsFileMetadataUtils {
    * @return -meta data
    */
   public static TsFileMetaData getTsFileMetaData(String filePath) throws IOException {
-    try (TsFileSequenceReader reader = new TsFileSequenceReader(filePath)) {
-      return reader.readFileMetadata();
-    }
+    TsFileSequenceReader reader = FileReaderManager.getInstance().get(filePath, true);
+    return reader.readFileMetadata();
   }
 
   /**
@@ -59,14 +65,34 @@ public class TsFileMetadataUtils {
     if (!fileMetaData.getDeviceMap().containsKey(deviceId)) {
       return null;
     } else {
-      try (TsFileSequenceReader reader = new TsFileSequenceReader(filePath)) {
-        long offset = fileMetaData.getDeviceMap().get(deviceId).getOffset();
-        int size = fileMetaData.getDeviceMap().get(deviceId).getLen();
-        ByteBuffer data = ByteBuffer.allocate(size);
-        reader.readRaw(offset, size, data);
-        data.flip();
-        return TsDeviceMetadata.deserializeFrom(data);
+      // get the index information of TsDeviceMetadata
+      TsDeviceMetadataIndex index = fileMetaData.getDeviceMetadataIndex(deviceId);
+      TsFileSequenceReader tsFileReader = FileReaderManager.getInstance().get(filePath, true);
+      // read TsDeviceMetadata from file
+      TsDeviceMetadata tsDeviceMetadata = tsFileReader.readTsDeviceMetaData(index);
+      return tsDeviceMetadata;
+    }
+  }
+
+  /**
+   * get all ChunkMetaData List included in all ChunkGroups of this device.
+   */
+  public static ConcurrentHashMap<Path, List<ChunkMetaData>> getChunkMetaDataList(
+      TsDeviceMetadata tsDeviceMetadata) {
+    ConcurrentHashMap<Path, List<ChunkMetaData>> pathToChunkMetaDataList = new ConcurrentHashMap<>();
+    for (ChunkGroupMetaData chunkGroupMetaData : tsDeviceMetadata.getChunkGroupMetaDataList()) {
+      List<ChunkMetaData> chunkMetaDataListInOneChunkGroup = chunkGroupMetaData
+          .getChunkMetaDataList();
+      String deviceId = chunkGroupMetaData.getDeviceID();
+      for (ChunkMetaData chunkMetaData : chunkMetaDataListInOneChunkGroup) {
+        Path path = new Path(deviceId, chunkMetaData.getMeasurementUid());
+        pathToChunkMetaDataList.putIfAbsent(path, new ArrayList<>());
+        chunkMetaData.setVersion(chunkGroupMetaData.getVersion());
+        pathToChunkMetaDataList.get(path).add(chunkMetaData);
       }
     }
+    return pathToChunkMetaDataList;
   }
+
+
 }
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsResource.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsResource.java
new file mode 100644
index 0000000..37c4fa0
--- /dev/null
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsResource.java
@@ -0,0 +1,16 @@
+package org.apache.iotdb.db.engine.cache;
+
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.tsfile.read.common.Path;
+
+public class TsResource{
+  TsFileResource file;
+  Path path;
+  public TsResource(){
+
+  }
+  public TsResource(TsFileResource file, Path path){
+    this.file = file;
+    this.path = path;
+  }
+}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/control/JobFileManager.java b/iotdb/src/main/java/org/apache/iotdb/db/query/control/JobFileManager.java
index 315cc48..f102e44 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/control/JobFileManager.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/control/JobFileManager.java
@@ -76,11 +76,11 @@ public class JobFileManager {
    */
   void removeUsedFilesForGivenJob(long jobId) {
       for (String filePath : sealedFilePathsMap.get(jobId)) {
-        FileReaderManager.getInstance().decreaseFileReaderReference(filePath, false);
+        FileReaderManager.getInstance().decreaseFileReaderReference(filePath, true);
       }
       sealedFilePathsMap.remove(jobId);
       for (String filePath : unsealedFilePathsMap.get(jobId)) {
-        FileReaderManager.getInstance().decreaseFileReaderReference(filePath, true);
+        FileReaderManager.getInstance().decreaseFileReaderReference(filePath, false);
       }
       unsealedFilePathsMap.remove(jobId);
   }
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/factory/SeriesReaderFactoryImpl.java b/iotdb/src/main/java/org/apache/iotdb/db/query/factory/SeriesReaderFactoryImpl.java
index 9876f43..02ec4e4 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/factory/SeriesReaderFactoryImpl.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/factory/SeriesReaderFactoryImpl.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.db.query.factory;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import org.apache.iotdb.db.engine.cache.DeviceMetaDataCache;
 import org.apache.iotdb.db.engine.modification.Modification;
 import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
 import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
@@ -107,8 +108,7 @@ public class SeriesReaderFactoryImpl implements ISeriesReaderFactory {
     // get modified chunk metadatas
     List<ChunkMetaData> metaDataList;
     if (tsFileResource.isClosed()) {
-      MetadataQuerierByFileImpl metadataQuerier = new MetadataQuerierByFileImpl(tsFileReader);
-      metaDataList = metadataQuerier.getChunkMetaDataList(seriesPath);
+      metaDataList = DeviceMetaDataCache.getInstance().get(tsFileResource.getFile().getPath(), seriesPath);
       // mod
       List<Modification> pathModifications = context
           .getPathModifications(tsFileResource.getModFile(),
@@ -161,8 +161,7 @@ public class SeriesReaderFactoryImpl implements ISeriesReaderFactory {
 
       List<ChunkMetaData> metaDataList;
       if (tsFileResource.isClosed()) {
-        MetadataQuerierByFileImpl metadataQuerier = new MetadataQuerierByFileImpl(tsFileReader);
-        metaDataList = metadataQuerier.getChunkMetaDataList(seriesPath);
+        metaDataList = DeviceMetaDataCache.getInstance().get(tsFileResource.getFile().getPath(), seriesPath);
         // mod
         List<Modification> pathModifications = context
                 .getPathModifications(tsFileResource.getModFile(),
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceSeriesReader.java b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceSeriesReader.java
index da4571e..b68ab53 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceSeriesReader.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceSeriesReader.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.db.query.reader.sequence;
 import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
+import org.apache.iotdb.db.engine.cache.DeviceMetaDataCache;
 import org.apache.iotdb.db.engine.modification.Modification;
 import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
 import org.apache.iotdb.db.query.context.QueryContext;
@@ -128,9 +129,8 @@ public class SequenceSeriesReader extends IterateReader {
     // to avoid too many opened files
     TsFileSequenceReader tsFileReader = FileReaderManager.getInstance()
         .get(tsfile.getFile().getPath(), true);
-
-    MetadataQuerierByFileImpl metadataQuerier = new MetadataQuerierByFileImpl(tsFileReader);
-    List<ChunkMetaData> metaDataList = metadataQuerier.getChunkMetaDataList(seriesPath);
+    List<ChunkMetaData> metaDataList =  DeviceMetaDataCache
+        .getInstance().get(tsfile.getFile().getPath(), seriesPath);
 
     List<Modification> pathModifications = context.getPathModifications(tsfile.getModFile(),
         seriesPath.getFullPath());
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceSeriesReaderByTimestamp.java b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceSeriesReaderByTimestamp.java
index aeb4bdd..8fdecc7 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceSeriesReaderByTimestamp.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceSeriesReaderByTimestamp.java
@@ -19,6 +19,7 @@
 
 package org.apache.iotdb.db.query.reader.sequence;
 
+import org.apache.iotdb.db.engine.cache.DeviceMetaDataCache;
 import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
 import org.apache.iotdb.db.engine.modification.Modification;
 import org.apache.iotdb.db.query.context.QueryContext;
@@ -140,8 +141,8 @@ public class SequenceSeriesReaderByTimestamp implements IReaderByTimeStamp {
     TsFileSequenceReader tsFileReader = FileReaderManager.getInstance()
             .get(fileNode.getFile().getPath(), true);
 
-    MetadataQuerierByFileImpl metadataQuerier = new MetadataQuerierByFileImpl(tsFileReader);
-    List<ChunkMetaData> metaDataList = metadataQuerier.getChunkMetaDataList(seriesPath);
+    List<ChunkMetaData> metaDataList = DeviceMetaDataCache
+        .getInstance().get(fileNode.getFile().getPath(), seriesPath);
 
     List<Modification> pathModifications = context.getPathModifications(fileNode.getModFile(),
             seriesPath.getFullPath());