You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by su...@apache.org on 2019/08/16 06:27:50 UTC
[incubator-iotdb] 01/01: increase configuration parameters of cache
This is an automated email from the ASF dual-hosted git repository.
suyue pushed a commit to branch add_cache_configuration
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit f94d42114e4e43e4bc42182a00e1391d187fb7ec
Author: suyue <23...@qq.com>
AuthorDate: Fri Aug 16 14:27:21 2019 +0800
increase configuration parameters of cache
---
.../resources/conf/iotdb-engine.properties | 10 ++++++
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 40 +++++++++++++++++++++-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 24 +++++++++++++
.../iotdb/db/engine/cache/DeviceMetaDataCache.java | 20 ++++++++---
.../iotdb/db/engine/cache/TsFileMetaDataCache.java | 18 +++++++---
.../iotdb/db/engine/cache/TsFileMetadataUtils.java | 16 +++++++++
6 files changed, 119 insertions(+), 9 deletions(-)
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index 01dea2c..7a67eeb 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -157,6 +157,16 @@ concurrent_flush_thread=0
# (i.e., whether use ChunkBufferPool), value true, false
chunk_buffer_pool_enable = false
+####################
+### Metadata Cache Configuration
+####################
+
+# whether to cache meta data(ChunkMetaData and TsFileMetaData) or not.
+meta_data_cache_enable=true
+# Read memory Allocation Ratio: FileMetaDataCache, ChunkMetaDataCache, and Free Memory Used in Query.
+# The parameter form is a:b:c, where a, b and c are integers. for example: 1:1:1 , 3:6:10
+filemeta_chunkmeta_free_memory_proportion=3:6:10
+
####################
### Statistics Monitor configuration
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 86c3622..1a98a36 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -99,7 +99,7 @@ public class IoTDBConfig {
private String systemDir = "data/system";
/**
- * Schema directory, including storage set of values.
+ * Schema directory, including storage set of values.
*/
private String schemaDir = "data/system/schema";
@@ -157,6 +157,20 @@ public class IoTDBConfig {
private long memtableSizeThreshold = 128 * 1024 * 1024L;
/**
+ * whether to cache meta data(ChunkMetaData and TsFileMetaData) or not.
+ */
+ private boolean metaDataCacheEnable = true;
+ /**
+ * Memory allocated for fileMetaData cache in read process
+ */
+ private long allocateMemoryForFileMetaDataCache = allocateMemoryForRead * 3 / 19;
+
+ /**
+ * Memory allocated for chunkMetaData cache in read process
+ */
+ private long allocateMemoryForChumkMetaDataCache = allocateMemoryForRead * 6 / 19;
+
+ /**
* The statMonitor writes statistics info into IoTDB every backLoopPeriodSec secs. The default
* value is 5s.
*/
@@ -618,4 +632,28 @@ public class IoTDBConfig {
public void setMemtableSizeThreshold(long memtableSizeThreshold) {
this.memtableSizeThreshold = memtableSizeThreshold;
}
+
+ public boolean isMetaDataCacheEnable() {
+ return metaDataCacheEnable;
+ }
+
+ public void setMetaDataCacheEnable(boolean metaDataCacheEnable) {
+ this.metaDataCacheEnable = metaDataCacheEnable;
+ }
+
+ public long getAllocateMemoryForFileMetaDataCache() {
+ return allocateMemoryForFileMetaDataCache;
+ }
+
+ public void setAllocateMemoryForFileMetaDataCache(long allocateMemoryForFileMetaDataCache) {
+ this.allocateMemoryForFileMetaDataCache = allocateMemoryForFileMetaDataCache;
+ }
+
+ public long getAllocateMemoryForChumkMetaDataCache() {
+ return allocateMemoryForChumkMetaDataCache;
+ }
+
+ public void setAllocateMemoryForChumkMetaDataCache(long allocateMemoryForChumkMetaDataCache) {
+ this.allocateMemoryForChumkMetaDataCache = allocateMemoryForChumkMetaDataCache;
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index c916114..b255154 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -130,6 +130,10 @@ public class IoTDBDescriptor {
Boolean.parseBoolean(properties.getProperty("enable_parameter_adapter",
Boolean.toString(conf.isEnableParameterAdapter()))));
+ conf.setMetaDataCacheEnable(
+ Boolean.parseBoolean(properties.getProperty("meta_data_cache_enable",
+ Boolean.toString(conf.isMetaDataCacheEnable()))));
+
initMemoryAllocate(properties);
conf.setEnableWal(Boolean.parseBoolean(properties.getProperty("enable_wal",
@@ -263,6 +267,26 @@ public class IoTDBDescriptor {
conf.setAllocateMemoryForRead(
maxMemoryAvailable * Integer.parseInt(proportions[1].trim()) / proportionSum);
}
+
+ if (!conf.isMetaDataCacheEnable()) {
+ return;
+ }
+
+ String queryMemoryAllocateProportion = properties
+ .getProperty("filemeta_chunkmeta_free_memory_proportion");
+ if (queryMemoryAllocateProportion != null) {
+ String[] proportions = queryMemoryAllocateProportion.split(":");
+ int proportionSum = 0;
+ for (String proportion : proportions) {
+ proportionSum += Integer.parseInt(proportion.trim());
+ }
+ long maxMemoryAvailable = conf.getAllocateMemoryForRead();
+ conf.setAllocateMemoryForFileMetaDataCache(
+ maxMemoryAvailable * Integer.parseInt(proportions[0].trim()) / proportionSum);
+ conf.setAllocateMemoryForChumkMetaDataCache(
+ maxMemoryAvailable * Integer.parseInt(proportions[1].trim()) / proportionSum);
+ }
+
}
private static class IoTDBDescriptorHolder {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
index 369e5e0..042769c 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
@@ -46,8 +46,8 @@ public class DeviceMetaDataCache {
private static StorageEngine storageEngine = StorageEngine.getInstance();
- private static final long MEMORY_THRESHOLD_IN_B = (long) (0.3 * config
- .getAllocateMemoryForRead());
+ private static boolean cacheEnable = config.isMetaDataCacheEnable();
+ private static final long MEMORY_THRESHOLD_IN_B = config.getAllocateMemoryForChumkMetaDataCache();
/**
* key: file path dot deviceId dot sensorId.
* <p>
@@ -64,6 +64,9 @@ public class DeviceMetaDataCache {
private long chunkMetaDataSize = 0;
private DeviceMetaDataCache(long memoryThreshold) {
+ if (!cacheEnable) {
+ return;
+ }
lruCache = new LRULinkedHashMap<String, List<ChunkMetaData>>(memoryThreshold, true) {
@Override
protected long calEntrySize(String key, List<ChunkMetaData> value) {
@@ -84,6 +87,13 @@ public class DeviceMetaDataCache {
*/
public List<ChunkMetaData> get(String filePath, Path seriesPath)
throws IOException {
+ if (!cacheEnable) {
+ TsFileMetaData fileMetaData = TsFileMetaDataCache.getInstance().get(filePath);
+ TsDeviceMetadata deviceMetaData = TsFileMetadataUtils
+ .getTsDeviceMetaData(filePath, seriesPath, fileMetaData);
+ return TsFileMetadataUtils.getChunkMetaDataList(seriesPath.getMeasurement(), deviceMetaData);
+ }
+
StringBuilder builder = new StringBuilder(filePath).append(".").append(seriesPath.getDevice());
String pathDeviceStr = builder.toString();
String key = builder.append(".").append(seriesPath.getMeasurement()).toString();
@@ -117,7 +127,7 @@ public class DeviceMetaDataCache {
TsDeviceMetadata deviceMetaData = TsFileMetadataUtils
.getTsDeviceMetaData(filePath, seriesPath, fileMetaData);
// If measurement isn't included in the tsfile, empty list is returned.
- if(deviceMetaData == null){
+ if (deviceMetaData == null) {
return new ArrayList<>();
}
Map<Path, List<ChunkMetaData>> chunkMetaData = TsFileMetadataUtils
@@ -169,7 +179,9 @@ public class DeviceMetaDataCache {
*/
public void clear() {
synchronized (lruCache) {
- lruCache.clear();
+ if (lruCache != null) {
+ lruCache.clear();
+ }
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java
index c29b345..6b7e448 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java
@@ -34,8 +34,8 @@ public class TsFileMetaDataCache {
private static final Logger logger = LoggerFactory.getLogger(TsFileMetaDataCache.class);
private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
- private static final long MEMORY_THRESHOLD_IN_B = (long) (0.25 * config
- .getAllocateMemoryForRead());
+ private static boolean cacheEnable = config.isMetaDataCacheEnable();
+ private static final long MEMORY_THRESHOLD_IN_B = config.getAllocateMemoryForFileMetaDataCache();
/**
* key: Tsfile path. value: TsFileMetaData
*/
@@ -57,6 +57,9 @@ public class TsFileMetaDataCache {
private long versionAndCreatebySize = 10;
private TsFileMetaDataCache() {
+ if (!cacheEnable) {
+ return;
+ }
cache = new LRULinkedHashMap<String, TsFileMetaData>(MEMORY_THRESHOLD_IN_B, true) {
@Override
protected long calEntrySize(String key, TsFileMetaData value) {
@@ -86,6 +89,9 @@ public class TsFileMetaDataCache {
* @param path -given path
*/
public TsFileMetaData get(String path) throws IOException {
+ if (!cacheEnable) {
+ return TsFileMetadataUtils.getTsFileMetaData(path);
+ }
Object internPath = path.intern();
cacheRequestNum.incrementAndGet();
@@ -122,13 +128,17 @@ public class TsFileMetaDataCache {
public void remove(String path) {
synchronized (cache) {
- cache.remove(path);
+ if (cache != null) {
+ cache.remove(path);
+ }
}
}
public void clear() {
synchronized (cache) {
- cache.clear();
+ if (cache != null) {
+ cache.clear();
+ }
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java
index 8a4424b..cd53e3b 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java
@@ -98,4 +98,20 @@ public class TsFileMetadataUtils {
return pathToChunkMetaDataList;
}
+ public static List<ChunkMetaData> getChunkMetaDataList(String sensor,
+ TsDeviceMetadata tsDeviceMetadata) {
+ List<ChunkMetaData> chunkMetaDataList = new ArrayList<>();
+ for (ChunkGroupMetaData chunkGroupMetaData : tsDeviceMetadata.getChunkGroupMetaDataList()) {
+ List<ChunkMetaData> chunkMetaDataListInOneChunkGroup = chunkGroupMetaData
+ .getChunkMetaDataList();
+
+ for (ChunkMetaData chunkMetaData : chunkMetaDataListInOneChunkGroup) {
+ if (sensor.equals(chunkMetaData.getMeasurementUid())) {
+ chunkMetaData.setVersion(chunkGroupMetaData.getVersion());
+ chunkMetaDataList.add(chunkMetaData);
+ }
+ }
+ }
+ return chunkMetaDataList;
+ }
}