[incubator-iotdb] branch add_cache_configuration created (now f94d421)

[su...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

suyue pushed a change to branch add_cache_configuration
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.


      at f94d421  increase configuration parameters of cache

This branch includes the following new commits:

     new f94d421  increase configuration parameters of cache

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.

[incubator-iotdb] 01/01: increase configuration parameters of cache

[su...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

suyue pushed a commit to branch add_cache_configuration
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git

commit f94d42114e4e43e4bc42182a00e1391d187fb7ec
Author: suyue <23...@qq.com>
AuthorDate: Fri Aug 16 14:27:21 2019 +0800

    increase configuration parameters of cache
---
 .../resources/conf/iotdb-engine.properties         | 10 ++++++
 .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 40 +++++++++++++++++++++-
 .../org/apache/iotdb/db/conf/IoTDBDescriptor.java  | 24 +++++++++++++
 .../iotdb/db/engine/cache/DeviceMetaDataCache.java | 20 ++++++++---
 .../iotdb/db/engine/cache/TsFileMetaDataCache.java | 18 +++++++---
 .../iotdb/db/engine/cache/TsFileMetadataUtils.java | 16 +++++++++
 6 files changed, 119 insertions(+), 9 deletions(-)

diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index 01dea2c..7a67eeb 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -157,6 +157,16 @@ concurrent_flush_thread=0
 # (i.e., whether use ChunkBufferPool), value true, false
 chunk_buffer_pool_enable = false
 
+####################
+### Metadata Cache Configuration
+####################
+
+# whether to cache meta data(ChunkMetaData and TsFileMetaData) or not.
+meta_data_cache_enable=true
+# Read memory Allocation Ratio: FileMetaDataCache, ChunkMetaDataCache, and Free Memory Used in Query.
+# The parameter form is a:b:c, where a, b and c are integers. for example: 1:1:1 , 3:6:10
+filemeta_chunkmeta_free_memory_proportion=3:6:10
+
 
 ####################
 ### Statistics Monitor configuration
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 86c3622..1a98a36 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -99,7 +99,7 @@ public class IoTDBConfig {
   private String systemDir = "data/system";
 
   /**
-   *  Schema directory, including storage set of values.
+   * Schema directory, including storage set of values.
    */
   private String schemaDir = "data/system/schema";
 
@@ -157,6 +157,20 @@ public class IoTDBConfig {
   private long memtableSizeThreshold = 128 * 1024 * 1024L;
 
   /**
+   * whether to cache meta data(ChunkMetaData and TsFileMetaData) or not.
+   */
+  private boolean metaDataCacheEnable = true;
+  /**
+   * Memory allocated for fileMetaData cache in read process
+   */
+  private long allocateMemoryForFileMetaDataCache = allocateMemoryForRead * 3 / 19;
+
+  /**
+   * Memory allocated for chunkMetaData cache in read process
+   */
+  private long allocateMemoryForChumkMetaDataCache = allocateMemoryForRead * 6 / 19;
+
+  /**
    * The statMonitor writes statistics info into IoTDB every backLoopPeriodSec secs. The default
    * value is 5s.
    */
@@ -618,4 +632,28 @@ public class IoTDBConfig {
   public void setMemtableSizeThreshold(long memtableSizeThreshold) {
     this.memtableSizeThreshold = memtableSizeThreshold;
   }
+
+  public boolean isMetaDataCacheEnable() {
+    return metaDataCacheEnable;
+  }
+
+  public void setMetaDataCacheEnable(boolean metaDataCacheEnable) {
+    this.metaDataCacheEnable = metaDataCacheEnable;
+  }
+
+  public long getAllocateMemoryForFileMetaDataCache() {
+    return allocateMemoryForFileMetaDataCache;
+  }
+
+  public void setAllocateMemoryForFileMetaDataCache(long allocateMemoryForFileMetaDataCache) {
+    this.allocateMemoryForFileMetaDataCache = allocateMemoryForFileMetaDataCache;
+  }
+
+  public long getAllocateMemoryForChumkMetaDataCache() {
+    return allocateMemoryForChumkMetaDataCache;
+  }
+
+  public void setAllocateMemoryForChumkMetaDataCache(long allocateMemoryForChumkMetaDataCache) {
+    this.allocateMemoryForChumkMetaDataCache = allocateMemoryForChumkMetaDataCache;
+  }
 }
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index c916114..b255154 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -130,6 +130,10 @@ public class IoTDBDescriptor {
           Boolean.parseBoolean(properties.getProperty("enable_parameter_adapter",
               Boolean.toString(conf.isEnableParameterAdapter()))));
 
+      conf.setMetaDataCacheEnable(
+          Boolean.parseBoolean(properties.getProperty("meta_data_cache_enable",
+              Boolean.toString(conf.isMetaDataCacheEnable()))));
+
       initMemoryAllocate(properties);
 
       conf.setEnableWal(Boolean.parseBoolean(properties.getProperty("enable_wal",
@@ -263,6 +267,26 @@ public class IoTDBDescriptor {
       conf.setAllocateMemoryForRead(
           maxMemoryAvailable * Integer.parseInt(proportions[1].trim()) / proportionSum);
     }
+
+    if (!conf.isMetaDataCacheEnable()) {
+      return;
+    }
+
+    String queryMemoryAllocateProportion = properties
+        .getProperty("filemeta_chunkmeta_free_memory_proportion");
+    if (queryMemoryAllocateProportion != null) {
+      String[] proportions = queryMemoryAllocateProportion.split(":");
+      int proportionSum = 0;
+      for (String proportion : proportions) {
+        proportionSum += Integer.parseInt(proportion.trim());
+      }
+      long maxMemoryAvailable = conf.getAllocateMemoryForRead();
+      conf.setAllocateMemoryForFileMetaDataCache(
+          maxMemoryAvailable * Integer.parseInt(proportions[0].trim()) / proportionSum);
+      conf.setAllocateMemoryForChumkMetaDataCache(
+          maxMemoryAvailable * Integer.parseInt(proportions[1].trim()) / proportionSum);
+    }
+
   }
 
   private static class IoTDBDescriptorHolder {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
index 369e5e0..042769c 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
@@ -46,8 +46,8 @@ public class DeviceMetaDataCache {
 
   private static StorageEngine storageEngine = StorageEngine.getInstance();
 
-  private static final long MEMORY_THRESHOLD_IN_B = (long) (0.3 * config
-      .getAllocateMemoryForRead());
+  private static boolean cacheEnable = config.isMetaDataCacheEnable();
+  private static final long MEMORY_THRESHOLD_IN_B = config.getAllocateMemoryForChumkMetaDataCache();
   /**
    * key: file path dot deviceId dot sensorId.
    * <p>
@@ -64,6 +64,9 @@ public class DeviceMetaDataCache {
   private long chunkMetaDataSize = 0;
 
   private DeviceMetaDataCache(long memoryThreshold) {
+    if (!cacheEnable) {
+      return;
+    }
     lruCache = new LRULinkedHashMap<String, List<ChunkMetaData>>(memoryThreshold, true) {
       @Override
       protected long calEntrySize(String key, List<ChunkMetaData> value) {
@@ -84,6 +87,13 @@ public class DeviceMetaDataCache {
    */
   public List<ChunkMetaData> get(String filePath, Path seriesPath)
       throws IOException {
+    if (!cacheEnable) {
+      TsFileMetaData fileMetaData = TsFileMetaDataCache.getInstance().get(filePath);
+      TsDeviceMetadata deviceMetaData = TsFileMetadataUtils
+          .getTsDeviceMetaData(filePath, seriesPath, fileMetaData);
+      return TsFileMetadataUtils.getChunkMetaDataList(seriesPath.getMeasurement(), deviceMetaData);
+    }
+
     StringBuilder builder = new StringBuilder(filePath).append(".").append(seriesPath.getDevice());
     String pathDeviceStr = builder.toString();
     String key = builder.append(".").append(seriesPath.getMeasurement()).toString();
@@ -117,7 +127,7 @@ public class DeviceMetaDataCache {
       TsDeviceMetadata deviceMetaData = TsFileMetadataUtils
           .getTsDeviceMetaData(filePath, seriesPath, fileMetaData);
       // If measurement isn't included in the tsfile, empty list is returned.
-      if(deviceMetaData == null){
+      if (deviceMetaData == null) {
         return new ArrayList<>();
       }
       Map<Path, List<ChunkMetaData>> chunkMetaData = TsFileMetadataUtils
@@ -169,7 +179,9 @@ public class DeviceMetaDataCache {
    */
   public void clear() {
     synchronized (lruCache) {
-      lruCache.clear();
+      if (lruCache != null) {
+        lruCache.clear();
+      }
     }
   }
 
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java
index c29b345..6b7e448 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java
@@ -34,8 +34,8 @@ public class TsFileMetaDataCache {
   private static final Logger logger = LoggerFactory.getLogger(TsFileMetaDataCache.class);
   private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
 
-  private static final long MEMORY_THRESHOLD_IN_B = (long) (0.25 * config
-      .getAllocateMemoryForRead());
+  private static boolean cacheEnable = config.isMetaDataCacheEnable();
+  private static final long MEMORY_THRESHOLD_IN_B = config.getAllocateMemoryForFileMetaDataCache();
   /**
    * key: Tsfile path. value: TsFileMetaData
    */
@@ -57,6 +57,9 @@ public class TsFileMetaDataCache {
   private long versionAndCreatebySize = 10;
 
   private TsFileMetaDataCache() {
+    if (!cacheEnable) {
+      return;
+    }
     cache = new LRULinkedHashMap<String, TsFileMetaData>(MEMORY_THRESHOLD_IN_B, true) {
       @Override
       protected long calEntrySize(String key, TsFileMetaData value) {
@@ -86,6 +89,9 @@ public class TsFileMetaDataCache {
    * @param path -given path
    */
   public TsFileMetaData get(String path) throws IOException {
+    if (!cacheEnable) {
+      return TsFileMetadataUtils.getTsFileMetaData(path);
+    }
 
     Object internPath = path.intern();
     cacheRequestNum.incrementAndGet();
@@ -122,13 +128,17 @@ public class TsFileMetaDataCache {
 
   public void remove(String path) {
     synchronized (cache) {
-      cache.remove(path);
+      if (cache != null) {
+        cache.remove(path);
+      }
     }
   }
 
   public void clear() {
     synchronized (cache) {
-      cache.clear();
+      if (cache != null) {
+        cache.clear();
+      }
     }
   }
 
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java
index 8a4424b..cd53e3b 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java
@@ -98,4 +98,20 @@ public class TsFileMetadataUtils {
     return pathToChunkMetaDataList;
   }
 
+  public static List<ChunkMetaData> getChunkMetaDataList(String sensor,
+      TsDeviceMetadata tsDeviceMetadata) {
+    List<ChunkMetaData> chunkMetaDataList = new ArrayList<>();
+    for (ChunkGroupMetaData chunkGroupMetaData : tsDeviceMetadata.getChunkGroupMetaDataList()) {
+      List<ChunkMetaData> chunkMetaDataListInOneChunkGroup = chunkGroupMetaData
+          .getChunkMetaDataList();
+
+      for (ChunkMetaData chunkMetaData : chunkMetaDataListInOneChunkGroup) {
+        if (sensor.equals(chunkMetaData.getMeasurementUid())) {
+          chunkMetaData.setVersion(chunkGroupMetaData.getVersion());
+          chunkMetaDataList.add(chunkMetaData);
+        }
+      }
+    }
+    return chunkMetaDataList;
+  }
 }