You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by su...@apache.org on 2019/07/15 12:02:11 UTC
[incubator-iotdb] branch feature_query_cache updated (f580cbf ->
f68656a)
This is an automated email from the ASF dual-hosted git repository.
suyue pushed a change to branch feature_query_cache
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.
from f580cbf modify cache
new b49776b temp
new e580a29 add memory control in lru
add 2f5e7ff [ IOTDB-116]Performance statistics module (#195)
add 9960180 remove useless fetchsize in some places (#228)
add fd05a3c [IOTDB-129]Fix statistic bug when restoring incomplete tsfile (#230)
add 09d30d0 fix file name compare bug
add 11d3316 Merge pull request #233 from apache/fix_file_name_compare
add 01e5e1b recover when resource file does not exist while tsfile is complete
add 937d589 Merge pull request #234 from apache/enhance_resource
add fbd0117 change a wait memtable log from debug to info level
add 0afdab3 [IOTDB-91] Improve tsfile-spark-connector to support spark 2.4.3 (#227)
add 7d566ed fix word spelling
add 1e9449b Add UT for DirectoriesStrategy (#197)
add efe7db7 Fix links of nav content in documents
add 9a1f8a1 Merge pull request #229 from samperson1997/document_link
add 8d2f4ac add a function to support appending data at the end of a completed TsFile
add b36a120 remove unused class import which cause an error in UT
add d24cb03 Merge pull request #235 from apache/feature_append_add_on_complete_tsfile
add a300868 [IOTDB-133] Fix mistaken links in documents
add 73f4936 Merge pull request #237 from samperson1997/new_document_link
add a7815e3 [IOTDB-133] Fix mistaken links in documents
add 52c8744 Merge pull request #238 from Genius-pig/master
add 16f0b3b fix bugs in performacne stats module; add new functions in its MBean
add 75fad74 add stopCalculatingStats in MeasurementMBean
add 2f0f4f0 modify log info in performance stats
add fc05734 Merge pull request #239 from apache/fix_performance_stats
add 57c49f6 add pathExist() check after obtain lock when trying to add path to metadata tree
add 8826fda fix typo in comments
add fd815e1 fix WriteLogNodeManagerTest failure (Travis #2557)
add e734368 fix getting timeseries list error in grafana connector
new c37b8d0 merge master
new f68656a memory control in cache
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
docs/Community-Powered By.md | 8 +-
docs/Development.md | 55 +--
docs/Documentation-CHN/QuickStart.md | 17 +
.../4-Deployment and Management/2-Configuration.md | 12 +-
.../4-Performance Monitor.md | 78 ++++
.../{4-System log.md => 5-System log.md} | 0
.../{5-Data Management.md => 6-Data Management.md} | 0
...e.md => 7-Build and use IoTDB by Dockerfile.md} | 0
.../UserGuideV0.7.0/7-Tools-Cli.md | 8 +
.../UserGuideV0.7.0/7-Tools-Grafana.md | 17 +
docs/Documentation/Frequently asked questions.md | 20 +-
.../OtherMaterial-ReleaseNotesV0.7.0.md | 16 +-
docs/Documentation/OtherMaterial-Sample Data.md | 8 +-
docs/Documentation/QuickStart.md | 22 +-
.../4-Deployment and Management/2-Configuration.md | 14 +-
.../4-Performance Monitor.md | 79 ++++
.../{4-System log.md => 5-System log.md} | 0
.../{5-Data Management.md => 6-Data Management.md} | 0
...e.md => 7-Build and use IoTDB by Dockerfile.md} | 0
docs/Documentation/UserGuideV0.7.0/7-Tools-Cli.md | 8 +-
.../UserGuideV0.7.0/7-Tools-Grafana.md | 25 +-
.../UserGuideV0.7.0/7-Tools-Hadoop.md | 3 +-
docs/Documentation/UserGuideV0.7.0/7-Tools-Sync.md | 20 +-
.../Documentation/UserGuideV0.7.0/7-Tools-spark.md | 59 +--
.../iotdb/web/grafana/dao/impl/BasicDaoImpl.java | 3 +-
iotdb/iotdb/conf/iotdb-engine.properties | 12 +
iotdb/iotdb/conf/logback.xml | 20 +
iotdb/pom.xml | 25 ++
.../org/apache/iotdb/db/concurrent/ThreadName.java | 3 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 38 ++
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 10 +
.../directories/strategy/DirectoryStrategy.java | 31 +-
.../strategy/MaxDiskUsableSpaceFirstStrategy.java | 8 +-
.../MinFolderOccupiedSpaceFirstStrategy.java | 35 +-
.../directories/strategy/SequenceStrategy.java | 12 +-
.../db/cost/statistic/ConcurrentCircularArray.java | 69 ++++
.../iotdb/db/cost/statistic/Measurement.java | 458 +++++++++++++++++++++
.../iotdb/db/cost/statistic/MeasurementMBean.java | 72 ++++
.../apache/iotdb/db/cost/statistic/Operation.java | 21 +-
.../org/apache/iotdb/db/engine/StorageEngine.java | 38 +-
.../iotdb/db/engine/cache/DeviceMetaDataCache.java | 49 +--
.../iotdb/db/engine/cache/LruLinkedHashMap.java | 41 +-
.../iotdb/db/engine/cache/TsFileMetaDataCache.java | 5 +-
.../iotdb/db/engine/cache/TsFileMetadataUtils.java | 17 +-
.../engine/storagegroup/StorageGroupProcessor.java | 35 +-
.../db/engine/storagegroup/TsFileProcessor.java | 6 +-
.../db/engine/storagegroup/TsFileResource.java | 4 +
.../org/apache/iotdb/db/metadata/MManager.java | 5 +
.../org/apache/iotdb/db/qp/QueryProcessor.java | 3 +-
.../qp/executor/AbstractQueryProcessExecutor.java | 13 -
.../db/qp/executor/IQueryProcessExecutor.java | 4 -
.../iotdb/db/qp/strategy/LogicalGenerator.java | 6 +-
.../sequence/SequenceSeriesReaderByTimestamp.java | 3 +-
.../org/apache/iotdb/db/rescon/MemTablePool.java | 2 +-
.../java/org/apache/iotdb/db/service/IoTDB.java | 2 +
.../org/apache/iotdb/db/service/ServiceType.java | 1 +
.../org/apache/iotdb/db/service/TSServiceImpl.java | 37 +-
.../org/apache/iotdb/db/utils/CommonUtils.java | 20 +
.../writelog/recover/TsFileRecoverPerformer.java | 35 +-
.../strategy/DirectoryStrategyTest.java | 177 ++++++++
.../db/cost/statistic/PerformanceStatTest.java | 107 +++++
.../db/engine/memtable/MemTableFlushTaskTest.java | 6 +-
.../engine/storagegroup/TsFileProcessorTest.java | 84 +++-
.../iotdb/db/qp/plan/LogicalPlanSmallTest.java | 10 +-
.../apache/iotdb/db/qp/plan/PhysicalPlanTest.java | 10 +-
.../apache/iotdb/db/qp/utils/MemIntQpExecutor.java | 4 -
.../apache/iotdb/db/utils/EnvironmentUtils.java | 2 +-
.../iotdb/db/writelog/WriteLogNodeManagerTest.java | 6 +-
.../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 14 +-
pom.xml | 2 +-
spark/README.md | 2 +-
.../scala/org/apache/iotdb/tsfile/Converter.scala | 24 +-
.../apache/iotdb/tsfile/TsFileOutputWriter.scala | 6 +-
.../apache/iotdb/tsfile/TsFileWriterFactory.scala | 5 +-
.../org/apache/iotdb/tsfile/ConverterTest.scala | 7 +-
.../iotdb/tsfile/common/conf/TSFileDescriptor.java | 6 +-
.../{SystemConstant.java => TsFileConstant.java} | 4 +-
.../iotdb/tsfile/file/metadata/ChunkMetaData.java | 26 +-
.../iotdb/tsfile/file/metadata/TsDigest.java | 24 ++
.../file/metadata/statistics/Statistics.java | 14 +-
.../iotdb/tsfile/read/TsFileSequenceReader.java | 19 +-
.../org/apache/iotdb/tsfile/read/common/Path.java | 30 +-
.../java/org/apache/iotdb/tsfile/utils/Binary.java | 2 +-
.../write/writer/RestorableTsFileIOWriter.java | 57 ++-
.../tsfile/file/metadata/utils/TestHelper.java | 5 +-
.../write/writer/RestorableTsFileIOWriterTest.java | 22 +
86 files changed, 1895 insertions(+), 392 deletions(-)
create mode 100644 docs/Documentation-CHN/UserGuideV0.7.0/4-Deployment and Management/4-Performance Monitor.md
rename docs/Documentation-CHN/UserGuideV0.7.0/4-Deployment and Management/{4-System log.md => 5-System log.md} (100%)
rename docs/Documentation-CHN/UserGuideV0.7.0/4-Deployment and Management/{5-Data Management.md => 6-Data Management.md} (100%)
rename docs/Documentation-CHN/UserGuideV0.7.0/4-Deployment and Management/{6-Build and use IoTDB by Dockerfile.md => 7-Build and use IoTDB by Dockerfile.md} (100%)
create mode 100644 docs/Documentation/UserGuideV0.7.0/4-Deployment and Management/4-Performance Monitor.md
rename docs/Documentation/UserGuideV0.7.0/4-Deployment and Management/{4-System log.md => 5-System log.md} (100%)
rename docs/Documentation/UserGuideV0.7.0/4-Deployment and Management/{5-Data Management.md => 6-Data Management.md} (100%)
rename docs/Documentation/UserGuideV0.7.0/4-Deployment and Management/{6-Build and use IoTDB by Dockerfile.md => 7-Build and use IoTDB by Dockerfile.md} (100%)
create mode 100644 iotdb/src/main/java/org/apache/iotdb/db/cost/statistic/ConcurrentCircularArray.java
create mode 100644 iotdb/src/main/java/org/apache/iotdb/db/cost/statistic/Measurement.java
create mode 100644 iotdb/src/main/java/org/apache/iotdb/db/cost/statistic/MeasurementMBean.java
copy tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/factory/FilterType.java => iotdb/src/main/java/org/apache/iotdb/db/cost/statistic/Operation.java (76%)
create mode 100644 iotdb/src/test/java/org/apache/iotdb/db/conf/directories/strategy/DirectoryStrategyTest.java
create mode 100644 iotdb/src/test/java/org/apache/iotdb/db/cost/statistic/PerformanceStatTest.java
rename tsfile/src/main/java/org/apache/iotdb/tsfile/common/constant/{SystemConstant.java => TsFileConstant.java} (93%)
[incubator-iotdb] 02/04: add memory control in lru
Posted by su...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
suyue pushed a commit to branch feature_query_cache
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit e580a290979e7ae7ce1319ffb3667a856895e8c5
Author: suyue <23...@qq.com>
AuthorDate: Mon Jul 15 16:23:41 2019 +0800
add memory control in lru
---
iotdb/pom.xml | 5 ++++
.../iotdb/db/engine/cache/LruLinkedHashMap.java | 35 +++++++++++++++++-----
2 files changed, 33 insertions(+), 7 deletions(-)
diff --git a/iotdb/pom.xml b/iotdb/pom.xml
index eba79cc..32d7a5e 100644
--- a/iotdb/pom.xml
+++ b/iotdb/pom.xml
@@ -70,6 +70,11 @@
<artifactId>commons-lang3</artifactId>
<version>${common.lang3.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-core</artifactId>
+ <version>4.0.0</version>
+ </dependency>
</dependencies>
<build>
<plugins>
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/LruLinkedHashMap.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/LruLinkedHashMap.java
index bd4c74a..6fbc3dd 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/LruLinkedHashMap.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/LruLinkedHashMap.java
@@ -21,6 +21,8 @@ package org.apache.iotdb.db.engine.cache;
import java.util.LinkedHashMap;
import java.util.Map;
+import org.apache.lucene.util.RamUsageEstimator;
+
/**
* This class is a LRU cache. <b>Note: It's not thread safe.</b>
*/
@@ -28,17 +30,36 @@ public class LruLinkedHashMap<K, V> extends LinkedHashMap<K, V> {
private static final long serialVersionUID = 1290160928914532649L;
private static final float LOAD_FACTOR_MAP = 0.75f;
- private int maxCapacity;
+ private static final int initialCapacity = 128;
+ /**
+ * maximum memory threshold.
+ */
+ private int maxMemInB;
+ /**
+ * current used memory.
+ */
+ private int usedMemInB;
+
+ public LruLinkedHashMap(int maxMemInB, boolean isLru) {
+ super(initialCapacity, LOAD_FACTOR_MAP, isLru);
+ this.maxMemInB = maxMemInB;
+ }
- public LruLinkedHashMap(int maxCapacity, boolean isLru) {
- super(maxCapacity, LOAD_FACTOR_MAP, isLru);
- this.maxCapacity = maxCapacity;
+ @Override
+ protected boolean removeEldestEntry(Map.Entry<K, V> eldest) {
+ if(usedMemInB > maxMemInB){
+ usedMemInB -= RamUsageEstimator.sizeOf(eldest);
+ return true;
+ }
+ else {
+ return false;
+ }
}
@Override
- protected boolean removeEldestEntry(
- Map.Entry<K, V> eldest) {
- return size() > maxCapacity;
+ public V put(K key, V value) {
+ usedMemInB += RamUsageEstimator.sizeOf(key)+RamUsageEstimator.sizeOf(value);
+ return super.put(key, value);
}
@Override
[incubator-iotdb] 01/04: temp
Posted by su...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
suyue pushed a commit to branch feature_query_cache
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit b49776b20bd1cabaff571a1d1ee0038d85f666df
Author: suyue <23...@qq.com>
AuthorDate: Thu Jul 11 19:28:32 2019 +0800
temp
---
.../java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java | 3 ++-
.../main/java/org/apache/iotdb/db/engine/cache/LruLinkedHashMap.java | 3 +--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
index 55ad98c..b233f0d 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
@@ -34,7 +34,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * This class is used to cache <code>DeviceMetaDataCache</code> of tsfile in IoTDB.
+ * This class is used to cache <code>List<ChunkMetaData></code> of tsfile in IoTDB.
+ * The caching strategy is LRU.
*/
public class DeviceMetaDataCache {
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/LruLinkedHashMap.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/LruLinkedHashMap.java
index 856eabd..bd4c74a 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/LruLinkedHashMap.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/LruLinkedHashMap.java
@@ -22,8 +22,7 @@ package org.apache.iotdb.db.engine.cache;
import java.util.LinkedHashMap;
import java.util.Map;
/**
- * This class is a map used to cache the <code>RowGroupBlockMetaData</code>. The caching strategy
- * is LRU.
+ * This class is a LRU cache. <b>Note: It's not thread safe.</b>
*/
public class LruLinkedHashMap<K, V> extends LinkedHashMap<K, V> {
[incubator-iotdb] 04/04: memory control in cache
Posted by su...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
suyue pushed a commit to branch feature_query_cache
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit f68656acc5d0aba794778305cdfb0ba60ed47c59
Author: suyue <23...@qq.com>
AuthorDate: Mon Jul 15 20:01:53 2019 +0800
memory control in cache
---
.../org/apache/iotdb/db/engine/StorageEngine.java | 38 ++++++++++------
.../iotdb/db/engine/cache/DeviceMetaDataCache.java | 50 +++++++++++-----------
.../iotdb/db/engine/cache/LruLinkedHashMap.java | 13 +++---
.../iotdb/db/engine/cache/TsFileMetaDataCache.java | 5 ++-
.../iotdb/db/engine/cache/TsFileMetadataUtils.java | 17 +++++---
.../engine/storagegroup/StorageGroupProcessor.java | 30 ++++++++++++-
.../sequence/SequenceSeriesReaderByTimestamp.java | 3 +-
7 files changed, 101 insertions(+), 55 deletions(-)
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
index d27b0a3..33bb84d 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.commons.io.FileUtils;
import org.apache.iotdb.db.conf.IoTDBConfig;
@@ -30,10 +31,10 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
import org.apache.iotdb.db.engine.storagegroup.StorageGroupProcessor;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
-import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.exception.MetadataErrorException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.ProcessorException;
+import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.exception.StorageEngineFailureException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
@@ -57,8 +58,8 @@ public class StorageEngine implements IService {
private volatile boolean readOnly = false;
/**
- * a folder (system/storage_groups/ by default) that persist system info. Each Storage Processor will have a
- * subfolder under the systemDir.
+ * a folder (system/storage_groups/ by default) that persist system info. Each Storage Processor
+ * will have a subfolder under the systemDir.
*/
private final String systemDir;
@@ -87,7 +88,7 @@ public class StorageEngine implements IService {
*/
try {
List<String> storageGroups = MManager.getInstance().getAllStorageGroupNames();
- for (String storageGroup: storageGroups) {
+ for (String storageGroup : storageGroups) {
StorageGroupProcessor processor = new StorageGroupProcessor(systemDir, storageGroup);
logger.info("Storage Group Processor {} is recovered successfully", storageGroup);
processorMap.put(storageGroup, processor);
@@ -136,7 +137,7 @@ public class StorageEngine implements IService {
}
return processor;
} catch (PathErrorException | ProcessorException e) {
- logger.error("Fail to get StorageGroupProcessor {}", storageGroupName, e);
+ logger.error("Fail to get StorageGroupProcessor {}", storageGroupName, e);
throw new StorageEngineException(e);
}
}
@@ -191,13 +192,12 @@ public class StorageEngine implements IService {
/**
- * flush command
- * Sync asyncCloseOneProcessor all file node processors.
+ * flush command Sync asyncCloseOneProcessor all file node processors.
*/
public void syncCloseAllProcessor() {
logger.info("Start closing all storage group processor");
- synchronized (processorMap){
- for(StorageGroupProcessor processor: processorMap.values()){
+ synchronized (processorMap) {
+ for (StorageGroupProcessor processor : processorMap.values()) {
processor.waitForAllCurrentTsFileProcessorsClosed();
}
}
@@ -244,8 +244,8 @@ public class StorageEngine implements IService {
}
/**
- * end query on a given deviceId. If some TsFile has been merged and this query is the
- * last query using it, the TsFile can be deleted safely.
+ * end query on a given deviceId. If some TsFile has been merged and this query is the last query
+ * using it, the TsFile can be deleted safely.
*/
public void endQuery(String deviceId, int token) throws StorageEngineException {
// TODO implement it when developing the merge function
@@ -264,6 +264,15 @@ public class StorageEngine implements IService {
}
/**
+ * returns the top k% sensors most frequently used in queries.
+ */
+ public Set calTopKSensor(String deviceId, String sensorId, double k)
+ throws StorageEngineException {
+ StorageGroupProcessor storageGroupProcessor = getProcessor(deviceId);
+ return storageGroupProcessor.calTopKSensor(sensorId, k);
+ }
+
+ /**
* Append one specified tsfile to the storage group. <b>This method is only provided for
* transmission module</b>
*
@@ -271,7 +280,8 @@ public class StorageEngine implements IService {
* @param appendFile the appended tsfile information
*/
@SuppressWarnings("unused") // reimplement sync module
- public boolean appendFileToStorageGroupProcessor(String storageGroupName, TsFileResource appendFile,
+ public boolean appendFileToStorageGroupProcessor(String storageGroupName,
+ TsFileResource appendFile,
String appendFilePath) throws StorageEngineException {
// TODO reimplement sync module
return true;
@@ -311,8 +321,8 @@ public class StorageEngine implements IService {
}
/**
- * delete all data files (both memory data and file on disk) in a storage group.
- * It is used when there is no timeseries (which are all deleted) in this storage group)
+ * delete all data files (both memory data and file on disk) in a storage group. It is used when
+ * there is no timeseries (which are all deleted) in this storage group)
*/
public void deleteAllDataFilesInOneStorageGroup(String storageGroupName) {
if (processorMap.containsKey(storageGroupName)) {
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
index b233f0d..362168c 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
@@ -20,12 +20,12 @@ package org.apache.iotdb.db.engine.cache;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.LinkedHashMap;
import java.util.List;
-import java.util.Map;
-import java.util.Objects;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.iotdb.db.engine.StorageEngine;
+import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadata;
import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData;
@@ -34,18 +34,20 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * This class is used to cache <code>List<ChunkMetaData></code> of tsfile in IoTDB.
- * The caching strategy is LRU.
+ * This class is used to cache <code>List<ChunkMetaData></code> of tsfile in IoTDB. The caching
+ * strategy is LRU.
*/
public class DeviceMetaDataCache {
private static final Logger logger = LoggerFactory.getLogger(DeviceMetaDataCache.class);
- private static final int CACHE_SIZE = 100;
+ private static StorageEngine storageEngine = StorageEngine.getInstance();
+
+ private static final int MEMORY_THRESHOLD_IN_B = (int) (50*0.3*1024*1024*1024);
/**
* key: the file path. value: key: series path, value: list of chunkMetaData.
*/
- private LinkedHashMap<String, ConcurrentHashMap<Path, List<ChunkMetaData>>> lruCache;
+ private LruLinkedHashMap<String, ConcurrentHashMap<Path, List<ChunkMetaData>>> lruCache;
private AtomicLong cacheHintNum = new AtomicLong();
private AtomicLong cacheRequestNum = new AtomicLong();
@@ -63,7 +65,7 @@ public class DeviceMetaDataCache {
*/
public List<ChunkMetaData> get(String filePath, Path seriesPath)
throws IOException {
- String jointPath = filePath+"."+seriesPath.getDevice();
+ String jointPath = filePath + "." + seriesPath.getDevice();
Object jointPathObject = jointPath.intern();
synchronized (lruCache) {
cacheRequestNum.incrementAndGet();
@@ -93,34 +95,34 @@ public class DeviceMetaDataCache {
.getTsRowGroupBlockMetaData(filePath, seriesPath.getDevice(),
fileMetaData);
ConcurrentHashMap<Path, List<ChunkMetaData>> chunkMetaData = TsFileMetadataUtils
- .getChunkMetaDataList(blockMetaData);
+ .getChunkMetaDataList(calHotSensorSet(seriesPath), blockMetaData);
synchronized (lruCache) {
lruCache.putIfAbsent(filePath, new ConcurrentHashMap<>());
lruCache.get(filePath).putAll(chunkMetaData);
- if (lruCache.get(filePath).containsKey(seriesPath)) {
- return new ArrayList<>(lruCache.get(filePath).get(seriesPath));
+ if (chunkMetaData.containsKey(seriesPath)) {
+ return chunkMetaData.get(seriesPath);
}
return new ArrayList<>();
}
}
}
- /**
- * the num of chunkMetaData cached in the class.
- * @return num of chunkMetaData cached in the LRUCache
- */
- public int calChunkMetaDataNum() {
- int cnt = 0;
- synchronized (lruCache) {
- for (ConcurrentHashMap<Path, List<ChunkMetaData>> map : lruCache.values()) {
- for(List<ChunkMetaData> metaDataList: map.values()){
- cnt+=metaDataList.size();
- }
+ private Set<String> calHotSensorSet(Path seriesPath) throws IOException {
+ double porportion = lruCache.getUsedMemoryProportion();
+ if (porportion < 0.6) {
+ return null;
+ } else {
+ try {
+ return storageEngine.calTopKSensor(seriesPath.getDevice(), seriesPath.getMeasurement(), 0.1);
+ }
+ catch (Exception e){
+ throw new IOException(e);
}
+
}
- return cnt;
}
+
/**
* clear LRUCache.
*/
@@ -136,6 +138,6 @@ public class DeviceMetaDataCache {
private static class RowGroupBlockMetaDataCacheSingleton {
private static final DeviceMetaDataCache INSTANCE = new
- DeviceMetaDataCache(CACHE_SIZE);
+ DeviceMetaDataCache(MEMORY_THRESHOLD_IN_B);
}
}
\ No newline at end of file
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/LruLinkedHashMap.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/LruLinkedHashMap.java
index 6fbc3dd..10c622b 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/LruLinkedHashMap.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/LruLinkedHashMap.java
@@ -24,7 +24,7 @@ import java.util.Map;
import org.apache.lucene.util.RamUsageEstimator;
/**
- * This class is a LRU cache. <b>Note: It's not thread safe.</b>
+ * This class is a LRU cache. <b>Note: It's not thread safe.</b>
*/
public class LruLinkedHashMap<K, V> extends LinkedHashMap<K, V> {
@@ -47,21 +47,24 @@ public class LruLinkedHashMap<K, V> extends LinkedHashMap<K, V> {
@Override
protected boolean removeEldestEntry(Map.Entry<K, V> eldest) {
- if(usedMemInB > maxMemInB){
+ if (usedMemInB > maxMemInB) {
usedMemInB -= RamUsageEstimator.sizeOf(eldest);
return true;
- }
- else {
+ } else {
return false;
}
}
@Override
public V put(K key, V value) {
- usedMemInB += RamUsageEstimator.sizeOf(key)+RamUsageEstimator.sizeOf(value);
+ usedMemInB += RamUsageEstimator.sizeOf(key) + RamUsageEstimator.sizeOf(value);
return super.put(key, value);
}
+ public double getUsedMemoryProportion() {
+ return usedMemInB * 1.0 / maxMemInB;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java
index 6b431f0..98613cc 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java
@@ -31,7 +31,8 @@ public class TsFileMetaDataCache {
private static final Logger logger = LoggerFactory.getLogger(TsFileMetaDataCache.class);
- private static final int CACHE_SIZE = 100;
+
+ private static final int MEMORY_THRESHOLD_IN_B = (int) (50*0.3*1024*1024*1024);
/**
* key: The file seriesPath of tsfile.
*/
@@ -40,7 +41,7 @@ public class TsFileMetaDataCache {
private AtomicLong cacheRequestNum = new AtomicLong();
private TsFileMetaDataCache() {
- cache = new LruLinkedHashMap<>(CACHE_SIZE, true);
+ cache = new LruLinkedHashMap<>(MEMORY_THRESHOLD_IN_B, true);
}
public static TsFileMetaDataCache getInstance() {
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java
index 49c42b0..82d1322 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.db.engine.cache;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetaData;
@@ -75,24 +76,26 @@ public class TsFileMetadataUtils {
}
/**
- * get all ChunkMetaData List included in all ChunkGroups of this device.
+ * get ChunkMetaData List of sensors in sensorSet included in all ChunkGroups of this device. If
+ * sensorSet is null, then return metadata of all sensor included in this device.
*/
public static ConcurrentHashMap<Path, List<ChunkMetaData>> getChunkMetaDataList(
- TsDeviceMetadata tsDeviceMetadata) {
+ Set<String> sensorSet, TsDeviceMetadata tsDeviceMetadata) {
ConcurrentHashMap<Path, List<ChunkMetaData>> pathToChunkMetaDataList = new ConcurrentHashMap<>();
for (ChunkGroupMetaData chunkGroupMetaData : tsDeviceMetadata.getChunkGroupMetaDataList()) {
List<ChunkMetaData> chunkMetaDataListInOneChunkGroup = chunkGroupMetaData
.getChunkMetaDataList();
String deviceId = chunkGroupMetaData.getDeviceID();
for (ChunkMetaData chunkMetaData : chunkMetaDataListInOneChunkGroup) {
- Path path = new Path(deviceId, chunkMetaData.getMeasurementUid());
- pathToChunkMetaDataList.putIfAbsent(path, new ArrayList<>());
- chunkMetaData.setVersion(chunkGroupMetaData.getVersion());
- pathToChunkMetaDataList.get(path).add(chunkMetaData);
+ if (sensorSet == null || sensorSet.contains(chunkMetaData.getMeasurementUid())) {
+ Path path = new Path(deviceId, chunkMetaData.getMeasurementUid());
+ pathToChunkMetaDataList.putIfAbsent(path, new ArrayList<>());
+ chunkMetaData.setVersion(chunkGroupMetaData.getVersion());
+ pathToChunkMetaDataList.get(path).add(chunkMetaData);
+ }
}
}
return pathToChunkMetaDataList;
}
-
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index a3d0e9d..f6f9990 100755
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -26,14 +26,16 @@ import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Set;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.commons.io.FileUtils;
-import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.directories.DirectoryManager;
import org.apache.iotdb.db.engine.StorageEngine;
@@ -44,8 +46,8 @@ import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
import org.apache.iotdb.db.engine.version.SimpleFileVersionController;
import org.apache.iotdb.db.engine.version.VersionController;
import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
-import org.apache.iotdb.db.exception.StorageGroupProcessorException;
import org.apache.iotdb.db.exception.ProcessorException;
+import org.apache.iotdb.db.exception.StorageGroupProcessorException;
import org.apache.iotdb.db.exception.TsFileProcessorException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.qp.physical.crud.DeletePlan;
@@ -148,6 +150,11 @@ public class StorageGroupProcessor {
*/
private ModificationFile mergingModification;
+ /**
+ * This linked hash set records the access order of sensors used by query.
+ */
+ private LinkedHashSet<String> lruForSensorUsedInQuery = new LinkedHashSet<>();
+
public StorageGroupProcessor(String systemInfoDir, String storageGroupName)
throws ProcessorException {
@@ -445,6 +452,9 @@ public class StorageGroupProcessor {
// TODO need a read lock, please consider the concurrency with flush manager threads.
public QueryDataSource query(String deviceId, String measurementId, QueryContext context) {
insertLock.readLock().lock();
+ synchronized (lruForSensorUsedInQuery){
+ lruForSensorUsedInQuery.add(measurementId);
+ }
try {
List<TsFileResource> seqResources = getFileReSourceListForQuery(sequenceFileList,
deviceId, measurementId, context);
@@ -456,6 +466,22 @@ public class StorageGroupProcessor {
}
}
+ /**
+ * returns the top k% sensors most frequently used in queries.
+ */
+ public Set calTopKSensor(String sensorId, double k) {
+ int num = (int) (lruForSensorUsedInQuery.size() * k);
+ Set<String> sensorSet = new HashSet<>(num);
+ sensorSet.add(sensorId);
+
+ for (String sensor : lruForSensorUsedInQuery) {
+ if (num-- >= 0) {
+ sensorSet.add(sensor);
+ }
+ }
+ return sensorSet;
+ }
+
private void writeLock() {
insertLock.writeLock().lock();
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceSeriesReaderByTimestamp.java b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceSeriesReaderByTimestamp.java
index dee48a6..55bbb14 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceSeriesReaderByTimestamp.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceSeriesReaderByTimestamp.java
@@ -23,6 +23,7 @@ package org.apache.iotdb.db.query.reader.sequence;
import org.apache.iotdb.db.engine.cache.DeviceMetaDataCache;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.engine.modification.Modification;
+import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.db.query.reader.IReaderByTimeStamp;
@@ -135,7 +136,7 @@ public class SequenceSeriesReaderByTimestamp implements IReaderByTimeStamp {
}
private void initSealedTsFileReader(TsFileResource fileNode, QueryContext context)
- throws IOException {
+ throws IOException {
// to avoid too many opened files
TsFileSequenceReader tsFileReader = FileReaderManager.getInstance()
[incubator-iotdb] 03/04: merge master
Posted by su...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
suyue pushed a commit to branch feature_query_cache
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit c37b8d0a81757de950460dd949fc868b902b2591
Merge: e580a29 e734368
Author: suyue <23...@qq.com>
AuthorDate: Mon Jul 15 16:31:16 2019 +0800
merge master
docs/Community-Powered By.md | 8 +-
docs/Development.md | 55 +--
docs/Documentation-CHN/QuickStart.md | 17 +
.../4-Deployment and Management/2-Configuration.md | 12 +-
.../4-Performance Monitor.md | 78 ++++
.../{4-System log.md => 5-System log.md} | 0
.../{5-Data Management.md => 6-Data Management.md} | 0
...e.md => 7-Build and use IoTDB by Dockerfile.md} | 0
.../UserGuideV0.7.0/7-Tools-Cli.md | 8 +
.../UserGuideV0.7.0/7-Tools-Grafana.md | 17 +
docs/Documentation/Frequently asked questions.md | 20 +-
.../OtherMaterial-ReleaseNotesV0.7.0.md | 16 +-
docs/Documentation/OtherMaterial-Sample Data.md | 8 +-
docs/Documentation/QuickStart.md | 22 +-
.../4-Deployment and Management/2-Configuration.md | 14 +-
.../4-Performance Monitor.md | 79 ++++
.../{4-System log.md => 5-System log.md} | 0
.../{5-Data Management.md => 6-Data Management.md} | 0
...e.md => 7-Build and use IoTDB by Dockerfile.md} | 0
docs/Documentation/UserGuideV0.7.0/7-Tools-Cli.md | 8 +-
.../UserGuideV0.7.0/7-Tools-Grafana.md | 25 +-
.../UserGuideV0.7.0/7-Tools-Hadoop.md | 3 +-
docs/Documentation/UserGuideV0.7.0/7-Tools-Sync.md | 20 +-
.../Documentation/UserGuideV0.7.0/7-Tools-spark.md | 59 +--
.../iotdb/web/grafana/dao/impl/BasicDaoImpl.java | 3 +-
iotdb/iotdb/conf/iotdb-engine.properties | 12 +
iotdb/iotdb/conf/logback.xml | 20 +
iotdb/pom.xml | 20 +
.../org/apache/iotdb/db/concurrent/ThreadName.java | 3 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 38 ++
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 10 +
.../directories/strategy/DirectoryStrategy.java | 31 +-
.../strategy/MaxDiskUsableSpaceFirstStrategy.java | 8 +-
.../MinFolderOccupiedSpaceFirstStrategy.java | 35 +-
.../directories/strategy/SequenceStrategy.java | 12 +-
.../db/cost/statistic/ConcurrentCircularArray.java | 69 ++++
.../iotdb/db/cost/statistic/Measurement.java | 458 +++++++++++++++++++++
.../iotdb/db/cost/statistic/MeasurementMBean.java | 72 ++++
.../statistic/Operation.java} | 28 +-
.../engine/storagegroup/StorageGroupProcessor.java | 7 +-
.../db/engine/storagegroup/TsFileProcessor.java | 6 +-
.../db/engine/storagegroup/TsFileResource.java | 4 +
.../org/apache/iotdb/db/metadata/MManager.java | 5 +
.../org/apache/iotdb/db/qp/QueryProcessor.java | 3 +-
.../qp/executor/AbstractQueryProcessExecutor.java | 13 -
.../db/qp/executor/IQueryProcessExecutor.java | 4 -
.../iotdb/db/qp/strategy/LogicalGenerator.java | 6 +-
.../org/apache/iotdb/db/rescon/MemTablePool.java | 2 +-
.../java/org/apache/iotdb/db/service/IoTDB.java | 2 +
.../org/apache/iotdb/db/service/ServiceType.java | 1 +
.../org/apache/iotdb/db/service/TSServiceImpl.java | 37 +-
.../org/apache/iotdb/db/utils/CommonUtils.java | 20 +
.../writelog/recover/TsFileRecoverPerformer.java | 35 +-
.../strategy/DirectoryStrategyTest.java | 177 ++++++++
.../db/cost/statistic/PerformanceStatTest.java | 107 +++++
.../db/engine/memtable/MemTableFlushTaskTest.java | 6 +-
.../engine/storagegroup/TsFileProcessorTest.java | 84 +++-
.../iotdb/db/qp/plan/LogicalPlanSmallTest.java | 10 +-
.../apache/iotdb/db/qp/plan/PhysicalPlanTest.java | 10 +-
.../apache/iotdb/db/qp/utils/MemIntQpExecutor.java | 4 -
.../apache/iotdb/db/utils/EnvironmentUtils.java | 2 +-
.../iotdb/db/writelog/WriteLogNodeManagerTest.java | 6 +-
.../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 14 +-
pom.xml | 2 +-
spark/README.md | 2 +-
.../scala/org/apache/iotdb/tsfile/Converter.scala | 24 +-
.../apache/iotdb/tsfile/TsFileOutputWriter.scala | 6 +-
.../apache/iotdb/tsfile/TsFileWriterFactory.scala | 5 +-
.../org/apache/iotdb/tsfile/ConverterTest.scala | 7 +-
.../iotdb/tsfile/common/conf/TSFileDescriptor.java | 6 +-
.../{SystemConstant.java => TsFileConstant.java} | 4 +-
.../iotdb/tsfile/file/metadata/ChunkMetaData.java | 26 +-
.../iotdb/tsfile/file/metadata/TsDigest.java | 24 ++
.../file/metadata/statistics/Statistics.java | 14 +-
.../iotdb/tsfile/read/TsFileSequenceReader.java | 19 +-
.../org/apache/iotdb/tsfile/read/common/Path.java | 30 +-
.../java/org/apache/iotdb/tsfile/utils/Binary.java | 2 +-
.../write/writer/RestorableTsFileIOWriter.java | 57 ++-
.../tsfile/file/metadata/utils/TestHelper.java | 5 +-
.../write/writer/RestorableTsFileIOWriterTest.java | 22 +
80 files changed, 1768 insertions(+), 340 deletions(-)
diff --cc iotdb/pom.xml
index 32d7a5e,44fb7d2..6965929
--- a/iotdb/pom.xml
+++ b/iotdb/pom.xml
@@@ -70,11 -70,25 +70,31 @@@
<artifactId>commons-lang3</artifactId>
<version>${common.lang3.version}</version>
</dependency>
++
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-core</artifactId>
+ <version>4.0.0</version>
+ </dependency>
+ <!-- for mocked test-->
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-core</artifactId>
+ <version>2.0.2</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ <version>2.0.2</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-mockito2</artifactId>
+ <version>2.0.2</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
<plugins>