You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by su...@apache.org on 2019/07/04 07:25:54 UTC
[incubator-iotdb] 01/01: add meta data cache in query
This is an automated email from the ASF dual-hosted git repository.
suyue pushed a commit to branch feature_query_cache
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 4380ab3ce1fdca3f6d361069ebd15bf3d648d0c4
Author: suyue <23...@qq.com>
AuthorDate: Thu Jul 4 15:24:47 2019 +0800
add meta data cache in query
---
.../iotdb/db/engine/cache/DeviceMetaDataCache.java | 55 ++++++++++++-------
.../org/apache/iotdb/db/engine/cache/Test.java | 64 ++++++++++++++++++++++
.../iotdb/db/engine/cache/TsFileMetadataUtils.java | 50 +++++++++++++----
.../apache/iotdb/db/engine/cache/TsResource.java | 16 ++++++
.../iotdb/db/query/control/JobFileManager.java | 4 +-
.../db/query/factory/SeriesReaderFactoryImpl.java | 7 +--
.../reader/sequence/SequenceSeriesReader.java | 6 +-
.../sequence/SequenceSeriesReaderByTimestamp.java | 5 +-
8 files changed, 164 insertions(+), 43 deletions(-)
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
index 994622a..cc06cec 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
@@ -19,12 +19,17 @@
package org.apache.iotdb.db.engine.cache;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.LinkedHashMap;
+import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadata;
import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData;
+import org.apache.iotdb.tsfile.read.common.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -37,9 +42,9 @@ public class DeviceMetaDataCache {
private static final int CACHE_SIZE = 100;
/**
- * key: the file path + deviceId.
+ * key: the file path. value: key: series path, value: list of chunkMetaData.
*/
- private LinkedHashMap<String, TsDeviceMetadata> lruCache;
+ private LinkedHashMap<String, ConcurrentHashMap<Path, List<ChunkMetaData>>> lruCache;
private AtomicLong cacheHintNum = new AtomicLong();
private AtomicLong cacheRequestNum = new AtomicLong();
@@ -55,15 +60,13 @@ public class DeviceMetaDataCache {
/**
* get {@link TsDeviceMetadata}. THREAD SAFE.
*/
- public TsDeviceMetadata get(String filePath, String deviceId, TsFileMetaData fileMetaData)
+ public List<ChunkMetaData> get(String filePath, Path seriesPath)
throws IOException {
- // The key(the tsfile path and deviceId) for the lruCache
- String jointPath = filePath + deviceId;
- Object jointPathObject = jointPath.intern();
+ Object jointPathObject = filePath.intern();
synchronized (lruCache) {
cacheRequestNum.incrementAndGet();
- if (lruCache.containsKey(jointPath)) {
+ if (lruCache.containsKey(filePath) && lruCache.get(filePath).containsKey(seriesPath)) {
cacheHintNum.incrementAndGet();
if (logger.isDebugEnabled()) {
logger.debug(
@@ -71,29 +74,36 @@ public class DeviceMetaDataCache {
+ "the number of hints for cache is {}",
cacheRequestNum.get(), cacheHintNum.get());
}
- return lruCache.get(jointPath);
+ return new ArrayList<>(lruCache.get(filePath).get(seriesPath));
}
}
synchronized (jointPathObject) {
synchronized (lruCache) {
- if (lruCache.containsKey(jointPath)) {
- return lruCache.get(jointPath);
+ if (lruCache.containsKey(filePath) && lruCache.get(filePath).containsKey(seriesPath)) {
+ return new ArrayList<>(lruCache.get(filePath).get(seriesPath));
}
}
if (logger.isDebugEnabled()) {
logger.debug("Cache didn't hint: the number of requests for cache is {}",
cacheRequestNum.get());
}
+ TsFileMetaData fileMetaData = TsFileMetaDataCache.getInstance().get(filePath);
TsDeviceMetadata blockMetaData = TsFileMetadataUtils
- .getTsRowGroupBlockMetaData(filePath, deviceId,
+ .getTsRowGroupBlockMetaData(filePath, seriesPath.getDevice(),
fileMetaData);
+ ConcurrentHashMap<Path, List<ChunkMetaData>> chunkMetaData = TsFileMetadataUtils
+ .getChunkMetaDataList(blockMetaData);
synchronized (lruCache) {
- lruCache.put(jointPath, blockMetaData);
- return lruCache.get(jointPath);
+ lruCache.put(filePath, chunkMetaData);
+ if(lruCache.get(filePath).containsKey(seriesPath)){
+ return new ArrayList<>(lruCache.get(filePath).get(seriesPath));
+ }
+ return new ArrayList<>();
}
}
}
+
/**
* clear LRUCache.
*/
@@ -115,9 +125,9 @@ public class DeviceMetaDataCache {
/**
* This class is a map used to cache the <code>RowGroupBlockMetaData</code>. The caching strategy
* is LRU.
- *
*/
- private class LruLinkedHashMap extends LinkedHashMap<String, TsDeviceMetadata> {
+ private class LruLinkedHashMap extends
+ LinkedHashMap<String, ConcurrentHashMap<Path, List<ChunkMetaData>>> {
private static final long serialVersionUID = 1290160928914532649L;
private static final float LOAD_FACTOR_MAP = 0.75f;
@@ -129,7 +139,8 @@ public class DeviceMetaDataCache {
}
@Override
- protected boolean removeEldestEntry(Map.Entry<String, TsDeviceMetadata> eldest) {
+ protected boolean removeEldestEntry(
+ Map.Entry<String, ConcurrentHashMap<Path, List<ChunkMetaData>>> eldest) {
return size() > maxCapacity;
}
@@ -152,12 +163,16 @@ public class DeviceMetaDataCache {
@Override
public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
DeviceMetaDataCache that = (DeviceMetaDataCache) o;
return Objects.equals(lruCache, that.lruCache) &&
- Objects.equals(cacheHintNum, that.cacheHintNum) &&
- Objects.equals(cacheRequestNum, that.cacheRequestNum);
+ Objects.equals(cacheHintNum, that.cacheHintNum) &&
+ Objects.equals(cacheRequestNum, that.cacheRequestNum);
}
@Override
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/Test.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/Test.java
new file mode 100644
index 0000000..c1b17fa
--- /dev/null
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/Test.java
@@ -0,0 +1,64 @@
+package org.apache.iotdb.db.engine.cache;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.Random;
+import org.apache.iotdb.tsfile.read.common.Path;
+
+public class Test {
+
+ public static void main(String[] args) {
+ File tmp = new File("");
+ Path path = new Path("d.s");
+ Random random = new Random();
+
+ HashMap<String, String> stringHashMap = new HashMap<>(10_000_000);
+ HashMap<Long, String> objectHashMap = new HashMap<>(10_000_000);
+ for (int i = 0; i < 10_000_000; i++) {
+ stringHashMap.put(genStr(), "abc");
+ objectHashMap.put(random.nextLong(), "abc");
+ }
+
+ System.out.println("init data finish!");
+ long t1 = System.currentTimeMillis();
+ for (String str : stringHashMap.keySet()) {
+ stringHashMap.get(str);
+ }
+ long t2 = System.currentTimeMillis();
+ System.out.println("str map time = " + (t2 - t1) + "ms");
+ long t3 = System.currentTimeMillis();
+ for (Long obj : objectHashMap.keySet()) {
+ objectHashMap.get(obj);
+ }
+ long t4 = System.currentTimeMillis();
+ System.out
+ .println("obj map time = " + (t4 - t3) + "ms" + ". obj.size = " + objectHashMap.size());
+
+ t3 = System.currentTimeMillis();
+ for (Long obj : objectHashMap.keySet()) {
+ objectHashMap.get(obj);
+ }
+ t4 = System.currentTimeMillis();
+ System.out.println("obj map time = " + (t4 - t3) + "ms");
+ t1 = System.currentTimeMillis();
+ for (String str : stringHashMap.keySet()) {
+ stringHashMap.get(str);
+ }
+ t2 = System.currentTimeMillis();
+ System.out.println("str map time = " + (t2 - t1) + "ms");
+
+
+ }
+
+
+ static String genStr() {
+ StringBuilder builder = new StringBuilder();
+ Random random = new Random();
+ for (int j = 0; j < 32; j++) {
+ char ch = (char) (random.nextInt(27) + 'a');
+ builder.append(ch);
+ }
+ return builder.reverse().toString();
+ }
+
+}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java
index ea63b37..49c42b0 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java
@@ -19,10 +19,17 @@
package org.apache.iotdb.db.engine.cache;
import java.io.IOException;
-import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import org.apache.iotdb.db.query.control.FileReaderManager;
+import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetaData;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadata;
+import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadataIndex;
import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
+import org.apache.iotdb.tsfile.read.common.Path;
/**
* This class is used to read metadata(<code>TsFileMetaData</code> and
@@ -30,7 +37,7 @@ import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
*/
public class TsFileMetadataUtils {
- private TsFileMetadataUtils(){
+ private TsFileMetadataUtils() {
}
@@ -41,9 +48,8 @@ public class TsFileMetadataUtils {
* @return -meta data
*/
public static TsFileMetaData getTsFileMetaData(String filePath) throws IOException {
- try (TsFileSequenceReader reader = new TsFileSequenceReader(filePath)) {
- return reader.readFileMetadata();
- }
+ TsFileSequenceReader reader = FileReaderManager.getInstance().get(filePath, true);
+ return reader.readFileMetadata();
}
/**
@@ -59,14 +65,34 @@ public class TsFileMetadataUtils {
if (!fileMetaData.getDeviceMap().containsKey(deviceId)) {
return null;
} else {
- try (TsFileSequenceReader reader = new TsFileSequenceReader(filePath)) {
- long offset = fileMetaData.getDeviceMap().get(deviceId).getOffset();
- int size = fileMetaData.getDeviceMap().get(deviceId).getLen();
- ByteBuffer data = ByteBuffer.allocate(size);
- reader.readRaw(offset, size, data);
- data.flip();
- return TsDeviceMetadata.deserializeFrom(data);
+ // get the index information of TsDeviceMetadata
+ TsDeviceMetadataIndex index = fileMetaData.getDeviceMetadataIndex(deviceId);
+ TsFileSequenceReader tsFileReader = FileReaderManager.getInstance().get(filePath, true);
+ // read TsDeviceMetadata from file
+ TsDeviceMetadata tsDeviceMetadata = tsFileReader.readTsDeviceMetaData(index);
+ return tsDeviceMetadata;
+ }
+ }
+
+ /**
+ * get all ChunkMetaData List included in all ChunkGroups of this device.
+ */
+ public static ConcurrentHashMap<Path, List<ChunkMetaData>> getChunkMetaDataList(
+ TsDeviceMetadata tsDeviceMetadata) {
+ ConcurrentHashMap<Path, List<ChunkMetaData>> pathToChunkMetaDataList = new ConcurrentHashMap<>();
+ for (ChunkGroupMetaData chunkGroupMetaData : tsDeviceMetadata.getChunkGroupMetaDataList()) {
+ List<ChunkMetaData> chunkMetaDataListInOneChunkGroup = chunkGroupMetaData
+ .getChunkMetaDataList();
+ String deviceId = chunkGroupMetaData.getDeviceID();
+ for (ChunkMetaData chunkMetaData : chunkMetaDataListInOneChunkGroup) {
+ Path path = new Path(deviceId, chunkMetaData.getMeasurementUid());
+ pathToChunkMetaDataList.putIfAbsent(path, new ArrayList<>());
+ chunkMetaData.setVersion(chunkGroupMetaData.getVersion());
+ pathToChunkMetaDataList.get(path).add(chunkMetaData);
}
}
+ return pathToChunkMetaDataList;
}
+
+
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsResource.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsResource.java
new file mode 100644
index 0000000..37c4fa0
--- /dev/null
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsResource.java
@@ -0,0 +1,16 @@
+package org.apache.iotdb.db.engine.cache;
+
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.tsfile.read.common.Path;
+
+public class TsResource{
+ TsFileResource file;
+ Path path;
+ public TsResource(){
+
+ }
+ public TsResource(TsFileResource file, Path path){
+ this.file = file;
+ this.path = path;
+ }
+}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/control/JobFileManager.java b/iotdb/src/main/java/org/apache/iotdb/db/query/control/JobFileManager.java
index 315cc48..f102e44 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/control/JobFileManager.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/control/JobFileManager.java
@@ -76,11 +76,11 @@ public class JobFileManager {
*/
void removeUsedFilesForGivenJob(long jobId) {
for (String filePath : sealedFilePathsMap.get(jobId)) {
- FileReaderManager.getInstance().decreaseFileReaderReference(filePath, false);
+ FileReaderManager.getInstance().decreaseFileReaderReference(filePath, true);
}
sealedFilePathsMap.remove(jobId);
for (String filePath : unsealedFilePathsMap.get(jobId)) {
- FileReaderManager.getInstance().decreaseFileReaderReference(filePath, true);
+ FileReaderManager.getInstance().decreaseFileReaderReference(filePath, false);
}
unsealedFilePathsMap.remove(jobId);
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/factory/SeriesReaderFactoryImpl.java b/iotdb/src/main/java/org/apache/iotdb/db/query/factory/SeriesReaderFactoryImpl.java
index 9876f43..02ec4e4 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/factory/SeriesReaderFactoryImpl.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/factory/SeriesReaderFactoryImpl.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.db.query.factory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import org.apache.iotdb.db.engine.cache.DeviceMetaDataCache;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
@@ -107,8 +108,7 @@ public class SeriesReaderFactoryImpl implements ISeriesReaderFactory {
// get modified chunk metadatas
List<ChunkMetaData> metaDataList;
if (tsFileResource.isClosed()) {
- MetadataQuerierByFileImpl metadataQuerier = new MetadataQuerierByFileImpl(tsFileReader);
- metaDataList = metadataQuerier.getChunkMetaDataList(seriesPath);
+ metaDataList = DeviceMetaDataCache.getInstance().get(tsFileResource.getFile().getPath(), seriesPath);
// mod
List<Modification> pathModifications = context
.getPathModifications(tsFileResource.getModFile(),
@@ -161,8 +161,7 @@ public class SeriesReaderFactoryImpl implements ISeriesReaderFactory {
List<ChunkMetaData> metaDataList;
if (tsFileResource.isClosed()) {
- MetadataQuerierByFileImpl metadataQuerier = new MetadataQuerierByFileImpl(tsFileReader);
- metaDataList = metadataQuerier.getChunkMetaDataList(seriesPath);
+ metaDataList = DeviceMetaDataCache.getInstance().get(tsFileResource.getFile().getPath(), seriesPath);
// mod
List<Modification> pathModifications = context
.getPathModifications(tsFileResource.getModFile(),
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceSeriesReader.java b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceSeriesReader.java
index da4571e..b68ab53 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceSeriesReader.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceSeriesReader.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.db.query.reader.sequence;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
+import org.apache.iotdb.db.engine.cache.DeviceMetaDataCache;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.query.context.QueryContext;
@@ -128,9 +129,8 @@ public class SequenceSeriesReader extends IterateReader {
// to avoid too many opened files
TsFileSequenceReader tsFileReader = FileReaderManager.getInstance()
.get(tsfile.getFile().getPath(), true);
-
- MetadataQuerierByFileImpl metadataQuerier = new MetadataQuerierByFileImpl(tsFileReader);
- List<ChunkMetaData> metaDataList = metadataQuerier.getChunkMetaDataList(seriesPath);
+ List<ChunkMetaData> metaDataList = DeviceMetaDataCache
+ .getInstance().get(tsfile.getFile().getPath(), seriesPath);
List<Modification> pathModifications = context.getPathModifications(tsfile.getModFile(),
seriesPath.getFullPath());
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceSeriesReaderByTimestamp.java b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceSeriesReaderByTimestamp.java
index aeb4bdd..8fdecc7 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceSeriesReaderByTimestamp.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceSeriesReaderByTimestamp.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.query.reader.sequence;
+import org.apache.iotdb.db.engine.cache.DeviceMetaDataCache;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.query.context.QueryContext;
@@ -140,8 +141,8 @@ public class SequenceSeriesReaderByTimestamp implements IReaderByTimeStamp {
TsFileSequenceReader tsFileReader = FileReaderManager.getInstance()
.get(fileNode.getFile().getPath(), true);
- MetadataQuerierByFileImpl metadataQuerier = new MetadataQuerierByFileImpl(tsFileReader);
- List<ChunkMetaData> metaDataList = metadataQuerier.getChunkMetaDataList(seriesPath);
+ List<ChunkMetaData> metaDataList = DeviceMetaDataCache
+ .getInstance().get(fileNode.getFile().getPath(), seriesPath);
List<Modification> pathModifications = context.getPathModifications(fileNode.getModFile(),
seriesPath.getFullPath());