You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by su...@apache.org on 2019/07/05 08:27:44 UTC
[incubator-iotdb] branch feature_query_cache updated: add cache in
TsFileMetaData
This is an automated email from the ASF dual-hosted git repository.
suyue pushed a commit to branch feature_query_cache
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/feature_query_cache by this push:
new 48968a8 add cache in TsFileMetaData
48968a8 is described below
commit 48968a83bed88537b1f207a79d9c57d383af2f07
Author: suyue <23...@qq.com>
AuthorDate: Fri Jul 5 16:27:29 2019 +0800
add cache in TsFileMetaData
---
.../iotdb/db/engine/cache/DeviceMetaDataCache.java | 78 +++++---------------
.../iotdb/db/engine/cache/LruLinkedHashMap.java | 60 ++++++++++++++++
.../org/apache/iotdb/db/engine/cache/Test.java | 82 ----------------------
.../iotdb/db/engine/cache/TsFileMetaDataCache.java | 54 ++++++++------
.../apache/iotdb/db/engine/cache/TsResource.java | 34 ---------
5 files changed, 112 insertions(+), 196 deletions(-)
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
index cc06cec..1598afc 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
@@ -94,8 +94,9 @@ public class DeviceMetaDataCache {
ConcurrentHashMap<Path, List<ChunkMetaData>> chunkMetaData = TsFileMetadataUtils
.getChunkMetaDataList(blockMetaData);
synchronized (lruCache) {
- lruCache.put(filePath, chunkMetaData);
- if(lruCache.get(filePath).containsKey(seriesPath)){
+ lruCache.putIfAbsent(filePath, new ConcurrentHashMap<>());
+ lruCache.get(filePath).putAll(chunkMetaData);
+ if (lruCache.get(filePath).containsKey(seriesPath)) {
return new ArrayList<>(lruCache.get(filePath).get(seriesPath));
}
return new ArrayList<>();
@@ -103,6 +104,21 @@ public class DeviceMetaDataCache {
}
}
+ /**
+ * the num of chunkMetaData cached in the class.
+ * @return num of chunkMetaData cached in the LRUCache
+ */
+ public int calChunkMetaDataNum() {
+ int cnt = 0;
+ synchronized (lruCache) {
+ for (ConcurrentHashMap<Path, List<ChunkMetaData>> map : lruCache.values()) {
+ for(List<ChunkMetaData> metaDataList: map.values()){
+ cnt+=metaDataList.size();
+ }
+ }
+ }
+ return cnt;
+ }
/**
* clear LRUCache.
@@ -121,62 +137,4 @@ public class DeviceMetaDataCache {
private static final DeviceMetaDataCache INSTANCE = new
DeviceMetaDataCache(CACHE_SIZE);
}
-
- /**
- * This class is a map used to cache the <code>RowGroupBlockMetaData</code>. The caching strategy
- * is LRU.
- */
- private class LruLinkedHashMap extends
- LinkedHashMap<String, ConcurrentHashMap<Path, List<ChunkMetaData>>> {
-
- private static final long serialVersionUID = 1290160928914532649L;
- private static final float LOAD_FACTOR_MAP = 0.75f;
- private int maxCapacity;
-
- public LruLinkedHashMap(int maxCapacity, boolean isLru) {
- super(maxCapacity, LOAD_FACTOR_MAP, isLru);
- this.maxCapacity = maxCapacity;
- }
-
- @Override
- protected boolean removeEldestEntry(
- Map.Entry<String, ConcurrentHashMap<Path, List<ChunkMetaData>>> eldest) {
- return size() > maxCapacity;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- return super.equals(o);
- }
-
- @Override
- public int hashCode() {
- return super.hashCode();
- }
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- DeviceMetaDataCache that = (DeviceMetaDataCache) o;
- return Objects.equals(lruCache, that.lruCache) &&
- Objects.equals(cacheHintNum, that.cacheHintNum) &&
- Objects.equals(cacheRequestNum, that.cacheRequestNum);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(lruCache, cacheHintNum, cacheRequestNum);
- }
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/LruLinkedHashMap.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/LruLinkedHashMap.java
new file mode 100644
index 0000000..856eabd
--- /dev/null
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/LruLinkedHashMap.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.cache;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+/**
+ * This class is a map used to cache the <code>RowGroupBlockMetaData</code>. The caching strategy
+ * is LRU.
+ */
+public class LruLinkedHashMap<K, V> extends LinkedHashMap<K, V> {
+
+ private static final long serialVersionUID = 1290160928914532649L;
+ private static final float LOAD_FACTOR_MAP = 0.75f;
+ private int maxCapacity;
+
+ public LruLinkedHashMap(int maxCapacity, boolean isLru) {
+ super(maxCapacity, LOAD_FACTOR_MAP, isLru);
+ this.maxCapacity = maxCapacity;
+ }
+
+ @Override
+ protected boolean removeEldestEntry(
+ Map.Entry<K, V> eldest) {
+ return size() > maxCapacity;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ return super.equals(o);
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/Test.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/Test.java
deleted file mode 100644
index ec6a016..0000000
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/Test.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.engine.cache;
-
-import java.io.File;
-import java.util.HashMap;
-import java.util.Random;
-import org.apache.iotdb.tsfile.read.common.Path;
-
-public class Test {
-
- public static void main(String[] args) {
- File tmp = new File("");
- Path path = new Path("d.s");
- Random random = new Random();
-
- HashMap<String, String> stringHashMap = new HashMap<>(10_000_000);
- HashMap<Long, String> objectHashMap = new HashMap<>(10_000_000);
- for (int i = 0; i < 10_000_000; i++) {
- stringHashMap.put(genStr(), "abc");
- objectHashMap.put(random.nextLong(), "abc");
- }
-
- System.out.println("init data finish!");
- long t1 = System.currentTimeMillis();
- for (String str : stringHashMap.keySet()) {
- stringHashMap.get(str);
- }
- long t2 = System.currentTimeMillis();
- System.out.println("str map time = " + (t2 - t1) + "ms");
- long t3 = System.currentTimeMillis();
- for (Long obj : objectHashMap.keySet()) {
- objectHashMap.get(obj);
- }
- long t4 = System.currentTimeMillis();
- System.out
- .println("obj map time = " + (t4 - t3) + "ms" + ". obj.size = " + objectHashMap.size());
-
- t3 = System.currentTimeMillis();
- for (Long obj : objectHashMap.keySet()) {
- objectHashMap.get(obj);
- }
- t4 = System.currentTimeMillis();
- System.out.println("obj map time = " + (t4 - t3) + "ms");
- t1 = System.currentTimeMillis();
- for (String str : stringHashMap.keySet()) {
- stringHashMap.get(str);
- }
- t2 = System.currentTimeMillis();
- System.out.println("str map time = " + (t2 - t1) + "ms");
-
-
- }
-
-
- static String genStr() {
- StringBuilder builder = new StringBuilder();
- Random random = new Random();
- for (int j = 0; j < 32; j++) {
- char ch = (char) (random.nextInt(27) + 'a');
- builder.append(ch);
- }
- return builder.reverse().toString();
- }
-
-}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java
index cff1a06..6b431f0 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java
@@ -19,7 +19,6 @@
package org.apache.iotdb.db.engine.cache;
import java.io.IOException;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData;
import org.slf4j.Logger;
@@ -31,15 +30,17 @@ import org.slf4j.LoggerFactory;
public class TsFileMetaDataCache {
private static final Logger logger = LoggerFactory.getLogger(TsFileMetaDataCache.class);
+
+ private static final int CACHE_SIZE = 100;
/**
* key: The file seriesPath of tsfile.
*/
- private ConcurrentHashMap<String, TsFileMetaData> cache;
+ private LruLinkedHashMap<String, TsFileMetaData> cache;
private AtomicLong cacheHintNum = new AtomicLong();
private AtomicLong cacheRequestNum = new AtomicLong();
private TsFileMetaDataCache() {
- cache = new ConcurrentHashMap<>();
+ cache = new LruLinkedHashMap<>(CACHE_SIZE, true);
}
public static TsFileMetaDataCache getInstance() {
@@ -54,36 +55,48 @@ public class TsFileMetaDataCache {
public TsFileMetaData get(String path) throws IOException {
Object internPath = path.intern();
- synchronized (internPath) {
- cacheRequestNum.incrementAndGet();
- if (!cache.containsKey(path)) {
- // read value from tsfile
- TsFileMetaData fileMetaData = TsFileMetadataUtils.getTsFileMetaData(path);
- cache.put(path, fileMetaData);
- if (logger.isDebugEnabled()) {
- logger.debug("Cache didn't hint: the number of requests for cache is {}",
- cacheRequestNum.get());
- }
- return cache.get(path);
- } else {
+ cacheRequestNum.incrementAndGet();
+ synchronized (cache) {
+ if (cache.containsKey(path)) {
cacheHintNum.incrementAndGet();
if (logger.isDebugEnabled()) {
logger.debug(
- "Cache hint: the number of requests for cache is {}, the number of hints for cache "
- + "is {}",
+ "Cache hint: the number of requests for cache is {}, "
+ + "the number of hints for cache is {}",
cacheRequestNum.get(), cacheHintNum.get());
}
return cache.get(path);
}
}
+ synchronized (internPath) {
+ synchronized (cache) {
+ if (cache.containsKey(path)) {
+ cacheHintNum.incrementAndGet();
+ return cache.get(path);
+ }
+ }
+ if (logger.isDebugEnabled()) {
+ logger.debug("Cache didn't hint: the number of requests for cache is {}",
+ cacheRequestNum.get());
+ }
+ TsFileMetaData fileMetaData = TsFileMetadataUtils.getTsFileMetaData(path);
+ synchronized (cache) {
+ cache.put(path, fileMetaData);
+ return fileMetaData;
+ }
+ }
}
public void remove(String path) {
- cache.remove(path);
+ synchronized (cache) {
+ cache.remove(path);
+ }
}
public void clear() {
- cache.clear();
+ synchronized (cache) {
+ cache.clear();
+ }
}
/*
@@ -91,7 +104,8 @@ public class TsFileMetaDataCache {
*/
private static class TsFileMetaDataCacheHolder {
- private TsFileMetaDataCacheHolder() {}
+ private TsFileMetaDataCacheHolder() {
+ }
private static final TsFileMetaDataCache INSTANCE = new TsFileMetaDataCache();
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsResource.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsResource.java
deleted file mode 100644
index aeb8641..0000000
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsResource.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.engine.cache;
-
-import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
-import org.apache.iotdb.tsfile.read.common.Path;
-
-public class TsResource{
- TsFileResource file;
- Path path;
- public TsResource(){
-
- }
- public TsResource(TsFileResource file, Path path){
- this.file = file;
- this.path = path;
- }
-}