You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by su...@apache.org on 2019/07/17 02:44:56 UTC
[incubator-iotdb] branch feature_query_cache updated (f68656a ->
e63b769)
This is an automated email from the ASF dual-hosted git repository.
suyue pushed a change to branch feature_query_cache
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.
from f68656a memory control in cache
new cec8ae5 memory control of cache
add 4c67e79 refactor readonly mode check
add 2bf13d2 add read-only log
add b018288 [IOTDB-134]Add default parameters for client (#244)
add 2678988 [IOTDB-130]Dynamic parameters adapter (#232)
add 4fb2818 fix conflict after merging master
add 7e04cd6 Merge pull request #245 from apache/fix_create_timeseries_check
add 3e95348 [IOTDB-73]Add new encoding method for regular timestamp column
add 78f0ac5 Delete TSRecordBatch.java
add 43271bb Update documentation
add d1522e7 Code reformat
add 19c56aa Merge pull request #231 from jack870131/missing_point_encoding
add 0c37235 [release preparation] Updated scm information in pom
add 1884dd2 [maven-release-plugin] prepare branch rel/0.8
add 88f8e5b [maven-release-plugin] prepare for next development iteration
new e63b769 merge master
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../3-Encoding.md | 10 +-
.../1-IoTDB Query Statement.md | 2 +-
.../3-Encoding.md | 10 +-
.../1-IoTDB Query Statement.md | 2 +-
example/kafka/pom.xml | 2 +-
example/pom.xml | 2 +-
example/rocketmq/pom.xml | 2 +-
grafana/pom.xml | 6 +-
iotdb-cli/cli/bin/start-client.bat | 6 +-
iotdb-cli/cli/bin/start-client.sh | 9 +-
iotdb-cli/pom.xml | 4 +-
iotdb/iotdb/conf/iotdb-engine.properties | 23 +-
iotdb/pom.xml | 8 +-
.../iotdb/db/auth/role/LocalFileRoleAccessor.java | 10 +-
.../iotdb/db/auth/user/LocalFileUserAccessor.java | 7 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 79 +-
.../org/apache/iotdb/db/conf/IoTDBConstant.java | 1 +
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 51 +-
.../iotdb/db/conf/adapter/CompressionRatio.java | 184 ++++
.../adapter/IDynamicAdapter.java} | 34 +-
.../db/conf/adapter/IoTDBConfigDynamicAdapter.java | 275 ++++++
.../db/conf/directories/DirectoryManager.java | 3 +
.../iotdb/db/cost/statistic/Measurement.java | 29 +-
.../org/apache/iotdb/db/engine/StorageEngine.java | 38 +-
.../iotdb/db/engine/cache/DeviceMetaDataCache.java | 92 +-
.../iotdb/db/engine/cache/LruLinkedHashMap.java | 25 +-
.../iotdb/db/engine/cache/RamUsageEstimator.java | 1019 ++++++++++++++++++++
.../iotdb/db/engine/cache/TsFileMetaDataCache.java | 44 +-
.../iotdb/db/engine/cache/TsFileMetadataUtils.java | 12 +-
.../iotdb/db/engine/memtable/ChunkBufferPool.java | 4 +-
.../db/engine/querycontext/ReadOnlyMemChunk.java | 2 +-
.../iotdb/db/engine/storagegroup/FlushManager.java | 2 +-
.../engine/storagegroup/StorageGroupProcessor.java | 47 +-
.../db/engine/storagegroup/TsFileProcessor.java | 42 +-
...Exception.java => ConfigAdjusterException.java} | 11 +-
.../org/apache/iotdb/db/metadata/MManager.java | 91 +-
.../iotdb/db/qp/executor/QueryProcessExecutor.java | 62 +-
.../org/apache/iotdb/db/rescon/MemTablePool.java | 23 +-
.../java/org/apache/iotdb/db/service/IoTDB.java | 17 +-
.../org/apache/iotdb/db/service/JDBCService.java | 2 +-
.../org/apache/iotdb/db/service/TSServiceImpl.java | 73 +-
.../writelog/manager/MultiFileLogNodeManager.java | 3 +-
.../db/writelog/node/ExclusiveWriteLogNode.java | 12 +-
.../db/conf/adapter/CompressionRatioTest.java | 115 +++
.../adapter/IoTDBConfigDynamicAdapterTest.java | 148 +++
.../db/cost/statistic/PerformanceStatTest.java | 2 +-
.../iotdb/db/engine/MetadataManagerHelper.java | 1 +
.../iotdb/db/engine/memtable/MemTablePoolTest.java | 20 +-
.../engine/modification/DeletionFileNodeTest.java | 1 +
.../db/engine/modification/DeletionQueryTest.java | 3 +-
.../engine/modification/ModificationFileTest.java | 1 +
.../integration/IoTDBAggregationSmallDataIT.java | 2 +-
.../db/integration/IoTDBEngineTimeGeneratorIT.java | 3 +
.../iotdb/db/integration/IoTDBLargeDataIT.java | 3 +
.../iotdb/db/integration/IoTDBMultiSeriesIT.java | 3 +
.../db/integration/IoTDBSequenceDataQueryIT.java | 3 +
.../iotdb/db/integration/IoTDBSeriesReaderIT.java | 3 +
.../org/apache/iotdb/db/metadata/MGraphTest.java | 1 +
.../iotdb/db/metadata/MManagerAdvancedTest.java | 2 +
.../iotdb/db/metadata/MManagerBasicTest.java | 1 +
.../iotdb/db/metadata/MManagerImproveTest.java | 1 +
.../org/apache/iotdb/db/metadata/MTreeTest.java | 1 +
.../org/apache/iotdb/db/metadata/MetadataTest.java | 1 +
.../org/apache/iotdb/db/qp/QueryProcessorTest.java | 4 +
.../dataset/EngineDataSetWithValueFilterTest.java | 5 +
.../iotdb/db/query/reader/ReaderTestHelper.java | 7 +-
.../apache/iotdb/db/utils/EnvironmentUtils.java | 19 +
.../iotdb/db/writelog/IoTDBLogFileSizeTest.java | 3 +
jdbc/pom.xml | 8 +-
pom.xml | 14 +-
service-rpc/pom.xml | 4 +-
spark/pom.xml | 2 +-
tsfile/pom.xml | 2 +-
.../iotdb/tsfile/encoding/decoder/Decoder.java | 10 +
...aBinaryDecoder.java => RegularDataDecoder.java} | 160 +--
.../encoding/encoder/RegularDataEncoder.java | 361 +++++++
.../tsfile/encoding/encoder/TSEncodingBuilder.java | 27 +-
.../tsfile/file/metadata/enums/TSEncoding.java | 6 +-
.../write/writer/RestorableTsFileIOWriter.java | 3 +-
.../decoder/delta/DeltaBinaryEncoderLongTest.java | 32 +-
.../regular/RegularDataEncoderIntegerTest.java | 145 +++
.../regular/RegularDataEncoderLongTest.java | 223 +++++
.../iotdb/tsfile/write/TsFileReadWriteTest.java | 24 +-
83 files changed, 3309 insertions(+), 445 deletions(-)
create mode 100644 iotdb/src/main/java/org/apache/iotdb/db/conf/adapter/CompressionRatio.java
copy iotdb/src/main/java/org/apache/iotdb/db/{writelog/manager/WriteLogNodeManager.java => conf/adapter/IDynamicAdapter.java} (51%)
create mode 100644 iotdb/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java
create mode 100644 iotdb/src/main/java/org/apache/iotdb/db/engine/cache/RamUsageEstimator.java
copy iotdb/src/main/java/org/apache/iotdb/db/exception/{FlushRunTimeException.java => ConfigAdjusterException.java} (77%)
create mode 100644 iotdb/src/test/java/org/apache/iotdb/db/conf/adapter/CompressionRatioTest.java
create mode 100644 iotdb/src/test/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapterTest.java
copy tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/{DeltaBinaryDecoder.java => RegularDataDecoder.java} (60%)
create mode 100644 tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/RegularDataEncoder.java
create mode 100644 tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/regular/RegularDataEncoderIntegerTest.java
create mode 100644 tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/regular/RegularDataEncoderLongTest.java
[incubator-iotdb] 01/02: memory control of cache
Posted by su...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
suyue pushed a commit to branch feature_query_cache
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit cec8ae5c2a55bf203b219f9919f7b9bd18a24390
Author: suyue <23...@qq.com>
AuthorDate: Wed Jul 17 09:54:10 2019 +0800
memory control of cache
---
iotdb/pom.xml | 6 -
.../iotdb/db/engine/cache/DeviceMetaDataCache.java | 80 +-
.../iotdb/db/engine/cache/LruLinkedHashMap.java | 15 +-
.../iotdb/db/engine/cache/RamUsageEstimator.java | 1019 ++++++++++++++++++++
.../iotdb/db/engine/cache/TsFileMetaDataCache.java | 39 +-
.../iotdb/db/engine/cache/TsFileMetadataUtils.java | 2 +-
.../engine/storagegroup/StorageGroupProcessor.java | 6 +-
7 files changed, 1124 insertions(+), 43 deletions(-)
diff --git a/iotdb/pom.xml b/iotdb/pom.xml
index 6965929..44fb7d2 100644
--- a/iotdb/pom.xml
+++ b/iotdb/pom.xml
@@ -70,12 +70,6 @@
<artifactId>commons-lang3</artifactId>
<version>${common.lang3.version}</version>
</dependency>
-
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-core</artifactId>
- <version>4.0.0</version>
- </dependency>
<!-- for mocked test-->
<dependency>
<groupId>org.powermock</groupId>
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
index 362168c..5a82116 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
@@ -25,7 +25,6 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.iotdb.db.engine.StorageEngine;
-import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadata;
import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData;
@@ -43,17 +42,32 @@ public class DeviceMetaDataCache {
private static StorageEngine storageEngine = StorageEngine.getInstance();
- private static final int MEMORY_THRESHOLD_IN_B = (int) (50*0.3*1024*1024*1024);
+ private static final int MEMORY_THRESHOLD_IN_B = (int) (50 * 0.3 * 1024 * 1024 * 1024);
/**
- * key: the file path. value: key: series path, value: list of chunkMetaData.
+ * key: file path dot deviceId dot sensorId.
+ * <p>
+ * value: chunkMetaData list of one timeseries in the file.
*/
- private LruLinkedHashMap<String, ConcurrentHashMap<Path, List<ChunkMetaData>>> lruCache;
+ private LruLinkedHashMap<String, List<ChunkMetaData>> lruCache;
private AtomicLong cacheHintNum = new AtomicLong();
private AtomicLong cacheRequestNum = new AtomicLong();
- private DeviceMetaDataCache(int cacheSize) {
- lruCache = new LruLinkedHashMap(cacheSize, true);
+ /**
+ * approximate estimate of chunkMetaData size
+ */
+ private long chunkMetaDataSize = 0;
+
+ private DeviceMetaDataCache(int memoryThreshold) {
+ lruCache = new LruLinkedHashMap<String, List<ChunkMetaData>>(memoryThreshold, true) {
+ @Override
+ protected long calEntrySize(String key, List<ChunkMetaData> value) {
+ if (chunkMetaDataSize == 0 && value.size() > 0) {
+ chunkMetaDataSize = RamUsageEstimator.sizeOf(value.get(0));
+ }
+ return value.size() * chunkMetaDataSize + key.length();
+ }
+ };
}
public static DeviceMetaDataCache getInstance() {
@@ -65,11 +79,14 @@ public class DeviceMetaDataCache {
*/
public List<ChunkMetaData> get(String filePath, Path seriesPath)
throws IOException {
- String jointPath = filePath + "." + seriesPath.getDevice();
- Object jointPathObject = jointPath.intern();
+ StringBuilder builder = new StringBuilder(filePath).append(".").append(seriesPath.getDevice());
+ String devicePathStr = builder.toString();
+ String key = builder.append(".").append(seriesPath.getMeasurement()).toString();
+ Object devicePathObject = devicePathStr.intern();
+
synchronized (lruCache) {
cacheRequestNum.incrementAndGet();
- if (lruCache.containsKey(filePath) && lruCache.get(filePath).containsKey(seriesPath)) {
+ if (lruCache.containsKey(key)) {
cacheHintNum.incrementAndGet();
if (logger.isDebugEnabled()) {
logger.debug(
@@ -77,13 +94,14 @@ public class DeviceMetaDataCache {
+ "the number of hints for cache is {}",
cacheRequestNum.get(), cacheHintNum.get());
}
- return new ArrayList<>(lruCache.get(filePath).get(seriesPath));
+ return new ArrayList<>(lruCache.get(key));
}
}
- synchronized (jointPathObject) {
+ synchronized (devicePathObject) {
synchronized (lruCache) {
- if (lruCache.containsKey(filePath) && lruCache.get(filePath).containsKey(seriesPath)) {
- return new ArrayList<>(lruCache.get(filePath).get(seriesPath));
+ if (lruCache.containsKey(key)) {
+ cacheHintNum.incrementAndGet();
+ return new ArrayList<>(lruCache.get(key));
}
}
if (logger.isDebugEnabled()) {
@@ -97,32 +115,46 @@ public class DeviceMetaDataCache {
ConcurrentHashMap<Path, List<ChunkMetaData>> chunkMetaData = TsFileMetadataUtils
.getChunkMetaDataList(calHotSensorSet(seriesPath), blockMetaData);
synchronized (lruCache) {
- lruCache.putIfAbsent(filePath, new ConcurrentHashMap<>());
- lruCache.get(filePath).putAll(chunkMetaData);
+ chunkMetaData.forEach((path, chunkMetaDataList) -> {
+ String k = devicePathStr + "." + path.getMeasurement();
+ if (!lruCache.containsKey(k)) {
+ lruCache.put(k, chunkMetaDataList);
+ }
+ });
if (chunkMetaData.containsKey(seriesPath)) {
- return chunkMetaData.get(seriesPath);
+ return new ArrayList<>(chunkMetaData.get(seriesPath));
}
return new ArrayList<>();
}
}
}
+ /**
+ * calculate the most frequently query sensors set.
+ *
+ * @param seriesPath the series to be queried in a query statements.
+ */
private Set<String> calHotSensorSet(Path seriesPath) throws IOException {
- double porportion = lruCache.getUsedMemoryProportion();
- if (porportion < 0.6) {
+ double usedMemProportion = lruCache.getUsedMemoryProportion();
+
+ if (usedMemProportion < 0.6) {
return null;
} else {
- try {
- return storageEngine.calTopKSensor(seriesPath.getDevice(), seriesPath.getMeasurement(), 0.1);
+ double hotSensorProportion;
+ if (usedMemProportion < 0.8) {
+ hotSensorProportion = 0.1;
+ } else {
+ hotSensorProportion = 0.05;
}
- catch (Exception e){
+ try {
+ return storageEngine
+ .calTopKSensor(seriesPath.getDevice(), seriesPath.getMeasurement(), hotSensorProportion);
+ } catch (Exception e) {
throw new IOException(e);
}
-
}
}
-
/**
* clear LRUCache.
*/
@@ -133,7 +165,7 @@ public class DeviceMetaDataCache {
}
/**
- * the default LRU cache size is 100. The singleton pattern.
+ * singleton pattern.
*/
private static class RowGroupBlockMetaDataCacheSingleton {
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/LruLinkedHashMap.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/LruLinkedHashMap.java
index 10c622b..05dd961 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/LruLinkedHashMap.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/LruLinkedHashMap.java
@@ -21,12 +21,11 @@ package org.apache.iotdb.db.engine.cache;
import java.util.LinkedHashMap;
import java.util.Map;
-import org.apache.lucene.util.RamUsageEstimator;
/**
* This class is a LRU cache. <b>Note: It's not thread safe.</b>
*/
-public class LruLinkedHashMap<K, V> extends LinkedHashMap<K, V> {
+public abstract class LruLinkedHashMap<K, V> extends LinkedHashMap<K, V> {
private static final long serialVersionUID = 1290160928914532649L;
private static final float LOAD_FACTOR_MAP = 0.75f;
@@ -48,7 +47,7 @@ public class LruLinkedHashMap<K, V> extends LinkedHashMap<K, V> {
@Override
protected boolean removeEldestEntry(Map.Entry<K, V> eldest) {
if (usedMemInB > maxMemInB) {
- usedMemInB -= RamUsageEstimator.sizeOf(eldest);
+ usedMemInB -= calEntrySize(eldest.getKey(), eldest.getValue());
return true;
} else {
return false;
@@ -57,10 +56,18 @@ public class LruLinkedHashMap<K, V> extends LinkedHashMap<K, V> {
@Override
public V put(K key, V value) {
- usedMemInB += RamUsageEstimator.sizeOf(key) + RamUsageEstimator.sizeOf(value);
+ usedMemInB += calEntrySize(key, value);
return super.put(key, value);
}
+ /**
+ * approximate estimate addition size of key and value.
+ */
+ protected abstract long calEntrySize(K key, V value);
+
+ /**
+ * calculate the proportion of used memory.
+ */
public double getUsedMemoryProportion() {
return usedMemInB * 1.0 / maxMemInB;
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/RamUsageEstimator.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/RamUsageEstimator.java
new file mode 100644
index 0000000..8ad7d63
--- /dev/null
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/RamUsageEstimator.java
@@ -0,0 +1,1019 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.engine.cache;
+
+import java.lang.management.ManagementFactory;
+import java.lang.reflect.Array;
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.text.DecimalFormat;
+import java.text.DecimalFormatSymbols;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.IdentityHashMap;
+import java.util.Iterator;
+import java.util.Locale;
+import java.util.Map;
+import java.util.NoSuchElementException;
+
+/**
+ * This class is copy from apache lucene, version 4.6.1. Estimates the size (memory representation)
+ * of Java objects. https://github.com/apache/lucene-solr/blob/releases/lucene-solr/4.6.1/lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java
+ *
+ * @lucene.internal
+ * @see #sizeOf(Object)
+ * @see #shallowSizeOf(Object)
+ * @see #shallowSizeOfInstance(Class)
+ */
+public final class RamUsageEstimator {
+
+ /**
+ * JVM diagnostic features.
+ */
+ public static enum JvmFeature {
+ OBJECT_REFERENCE_SIZE("Object reference size estimated using array index scale"),
+ ARRAY_HEADER_SIZE("Array header size estimated using array based offset"),
+ FIELD_OFFSETS("Shallow instance size based on field offsets"),
+ OBJECT_ALIGNMENT("Object alignment retrieved from HotSpotDiagnostic MX bean");
+
+ public final String description;
+
+ private JvmFeature(String description) {
+ this.description = description;
+ }
+
+ @Override
+ public String toString() {
+ return super.name() + " (" + description + ")";
+ }
+ }
+
+ /**
+ * JVM info string for debugging and reports.
+ */
+ public final static String JVM_INFO_STRING;
+
+ /**
+ * One kilobyte bytes.
+ */
+ public static final long ONE_KB = 1024;
+
+ /**
+ * One megabyte bytes.
+ */
+ public static final long ONE_MB = ONE_KB * ONE_KB;
+
+ /**
+ * One gigabyte bytes.
+ */
+ public static final long ONE_GB = ONE_KB * ONE_MB;
+
+ /**
+ * No instantiation.
+ */
+ private RamUsageEstimator() {
+ }
+
+ public final static int NUM_BYTES_BOOLEAN = 1;
+ public final static int NUM_BYTES_BYTE = 1;
+ public final static int NUM_BYTES_CHAR = 2;
+ public final static int NUM_BYTES_SHORT = 2;
+ public final static int NUM_BYTES_INT = 4;
+ public final static int NUM_BYTES_FLOAT = 4;
+ public final static int NUM_BYTES_LONG = 8;
+ public final static int NUM_BYTES_DOUBLE = 8;
+
+ /**
+ * Number of bytes this jvm uses to represent an object reference.
+ */
+ public final static int NUM_BYTES_OBJECT_REF;
+
+ /**
+ * Number of bytes to represent an object header (no fields, no alignments).
+ */
+ public final static int NUM_BYTES_OBJECT_HEADER;
+
+ /**
+ * Number of bytes to represent an array header (no content, but with alignments).
+ */
+ public final static int NUM_BYTES_ARRAY_HEADER;
+
+ /**
+ * A constant specifying the object alignment boundary inside the JVM. Objects will always take a
+ * full multiple of this constant, possibly wasting some space.
+ */
+ public final static int NUM_BYTES_OBJECT_ALIGNMENT;
+
+ /**
+ * Sizes of primitive classes.
+ */
+ private static final Map<Class<?>, Integer> primitiveSizes;
+
+ static {
+ primitiveSizes = new IdentityHashMap<Class<?>, Integer>();
+ primitiveSizes.put(boolean.class, Integer.valueOf(NUM_BYTES_BOOLEAN));
+ primitiveSizes.put(byte.class, Integer.valueOf(NUM_BYTES_BYTE));
+ primitiveSizes.put(char.class, Integer.valueOf(NUM_BYTES_CHAR));
+ primitiveSizes.put(short.class, Integer.valueOf(NUM_BYTES_SHORT));
+ primitiveSizes.put(int.class, Integer.valueOf(NUM_BYTES_INT));
+ primitiveSizes.put(float.class, Integer.valueOf(NUM_BYTES_FLOAT));
+ primitiveSizes.put(double.class, Integer.valueOf(NUM_BYTES_DOUBLE));
+ primitiveSizes.put(long.class, Integer.valueOf(NUM_BYTES_LONG));
+ }
+
+ /**
+ * A handle to <code>sun.misc.Unsafe</code>.
+ */
+ private final static Object theUnsafe;
+
+ /**
+ * A handle to <code>sun.misc.Unsafe#fieldOffset(Field)</code>.
+ */
+ private final static Method objectFieldOffsetMethod;
+
+ /**
+ * All the supported "internal" JVM features detected at clinit.
+ */
+ private final static EnumSet<JvmFeature> supportedFeatures;
+
+ /**
+ * Initialize constants and try to collect information about the JVM internals.
+ */
+ static {
+ // Initialize empirically measured defaults. We'll modify them to the current
+ // JVM settings later on if possible.
+ int referenceSize = Constants.JRE_IS_64BIT ? 8 : 4;
+ int objectHeader = Constants.JRE_IS_64BIT ? 16 : 8;
+ // The following is objectHeader + NUM_BYTES_INT, but aligned (object alignment)
+ // so on 64 bit JVMs it'll be align(16 + 4, @8) = 24.
+ int arrayHeader = Constants.JRE_IS_64BIT ? 24 : 12;
+
+ supportedFeatures = EnumSet.noneOf(JvmFeature.class);
+
+ Class<?> unsafeClass = null;
+ Object tempTheUnsafe = null;
+ try {
+ unsafeClass = Class.forName("sun.misc.Unsafe");
+ final Field unsafeField = unsafeClass.getDeclaredField("theUnsafe");
+ unsafeField.setAccessible(true);
+ tempTheUnsafe = unsafeField.get(null);
+ } catch (Exception e) {
+ // Ignore.
+ }
+ theUnsafe = tempTheUnsafe;
+
+ // get object reference size by getting scale factor of Object[] arrays:
+ try {
+ final Method arrayIndexScaleM = unsafeClass.getMethod("arrayIndexScale", Class.class);
+ referenceSize = ((Number) arrayIndexScaleM.invoke(theUnsafe, Object[].class)).intValue();
+ supportedFeatures.add(JvmFeature.OBJECT_REFERENCE_SIZE);
+ } catch (Exception e) {
+ // ignore.
+ }
+
+ // "best guess" based on reference size. We will attempt to modify
+ // these to exact values if there is supported infrastructure.
+ objectHeader = Constants.JRE_IS_64BIT ? (8 + referenceSize) : 8;
+ arrayHeader = Constants.JRE_IS_64BIT ? (8 + 2 * referenceSize) : 12;
+
+ // get the object header size:
+ // - first try out if the field offsets are not scaled (see warning in Unsafe docs)
+ // - get the object header size by getting the field offset of the first field of a dummy object
+ // If the scaling is byte-wise and unsafe is available, enable dynamic size measurement for
+ // estimateRamUsage().
+ Method tempObjectFieldOffsetMethod = null;
+ try {
+ final Method objectFieldOffsetM = unsafeClass.getMethod("objectFieldOffset", Field.class);
+ final Field dummy1Field = DummyTwoLongObject.class.getDeclaredField("dummy1");
+ final int ofs1 = ((Number) objectFieldOffsetM.invoke(theUnsafe, dummy1Field)).intValue();
+ final Field dummy2Field = DummyTwoLongObject.class.getDeclaredField("dummy2");
+ final int ofs2 = ((Number) objectFieldOffsetM.invoke(theUnsafe, dummy2Field)).intValue();
+ if (Math.abs(ofs2 - ofs1) == NUM_BYTES_LONG) {
+ final Field baseField = DummyOneFieldObject.class.getDeclaredField("base");
+ objectHeader = ((Number) objectFieldOffsetM.invoke(theUnsafe, baseField)).intValue();
+ supportedFeatures.add(JvmFeature.FIELD_OFFSETS);
+ tempObjectFieldOffsetMethod = objectFieldOffsetM;
+ }
+ } catch (Exception e) {
+ // Ignore.
+ }
+ objectFieldOffsetMethod = tempObjectFieldOffsetMethod;
+
+ // Get the array header size by retrieving the array base offset
+ // (offset of the first element of an array).
+ try {
+ final Method arrayBaseOffsetM = unsafeClass.getMethod("arrayBaseOffset", Class.class);
+ // we calculate that only for byte[] arrays, it's actually the same for all types:
+ arrayHeader = ((Number) arrayBaseOffsetM.invoke(theUnsafe, byte[].class)).intValue();
+ supportedFeatures.add(JvmFeature.ARRAY_HEADER_SIZE);
+ } catch (Exception e) {
+ // Ignore.
+ }
+
+ NUM_BYTES_OBJECT_REF = referenceSize;
+ NUM_BYTES_OBJECT_HEADER = objectHeader;
+ NUM_BYTES_ARRAY_HEADER = arrayHeader;
+
+ // Try to get the object alignment (the default seems to be 8 on Hotspot,
+ // regardless of the architecture).
+ int objectAlignment = 8;
+ try {
+ final Class<?> beanClazz = Class.forName("com.sun.management.HotSpotDiagnosticMXBean");
+ // Try to get the diagnostic mxbean without calling {@link ManagementFactory#getPlatformMBeanServer()}
+ // which starts AWT thread (and shows junk in the dock) on a Mac:
+ Object hotSpotBean;
+ // Java 7+, HotSpot
+ try {
+ hotSpotBean = ManagementFactory.class
+ .getMethod("getPlatformMXBean", Class.class)
+ .invoke(null, beanClazz);
+ } catch (Exception e1) {
+ // Java 6, HotSpot
+ try {
+ Class<?> sunMF = Class.forName("sun.management.ManagementFactory");
+ hotSpotBean = sunMF.getMethod("getDiagnosticMXBean").invoke(null);
+ } catch (Exception e2) {
+ // Last resort option is an attempt to get it from ManagementFactory's server anyway (may start AWT).
+ hotSpotBean = ManagementFactory.newPlatformMXBeanProxy(
+ ManagementFactory.getPlatformMBeanServer(),
+ "com.sun.management:type=HotSpotDiagnostic", beanClazz);
+ }
+ }
+ if (hotSpotBean != null) {
+ final Method getVMOptionMethod = beanClazz.getMethod("getVMOption", String.class);
+ final Object vmOption = getVMOptionMethod.invoke(hotSpotBean, "ObjectAlignmentInBytes");
+ objectAlignment = Integer.parseInt(
+ vmOption.getClass().getMethod("getValue").invoke(vmOption).toString()
+ );
+ supportedFeatures.add(JvmFeature.OBJECT_ALIGNMENT);
+ }
+ } catch (Exception e) {
+ // Ignore.
+ }
+
+ NUM_BYTES_OBJECT_ALIGNMENT = objectAlignment;
+
+ JVM_INFO_STRING = "[JVM: " +
+ Constants.JVM_NAME + ", " + Constants.JVM_VERSION + ", " + Constants.JVM_VENDOR + ", " +
+ Constants.JAVA_VENDOR + ", " + Constants.JAVA_VERSION + "]";
+ }
+
+ /**
+ * Cached information about a given class.
+ */
+ private static final class ClassCache {
+
+ public final long alignedShallowInstanceSize;
+ public final Field[] referenceFields;
+
+ public ClassCache(long alignedShallowInstanceSize, Field[] referenceFields) {
+ this.alignedShallowInstanceSize = alignedShallowInstanceSize;
+ this.referenceFields = referenceFields;
+ }
+ }
+
+ // Object with just one field to determine the object header size by getting the offset of the dummy field:
+ @SuppressWarnings("unused")
+ private static final class DummyOneFieldObject {
+
+ public byte base;
+ }
+
+ // Another test object for checking, if the difference in offsets of dummy1 and dummy2 is 8 bytes.
+ // Only then we can be sure that those are real, unscaled offsets:
+ @SuppressWarnings("unused")
+ private static final class DummyTwoLongObject {
+
+ public long dummy1, dummy2;
+ }
+
+ /**
+ * Returns true, if the current JVM is fully supported by {@code RamUsageEstimator}. If this
+ * method returns {@code false} you are maybe using a 3rd party Java VM that is not supporting
+ * Oracle/Sun private APIs. The memory estimates can be imprecise then (no way of detecting
+ * compressed references, alignments, etc.). Lucene still tries to use sensible defaults.
+ */
+ public static boolean isSupportedJVM() {
+ return supportedFeatures.size() == JvmFeature.values().length;
+ }
+
+ /**
+ * Aligns an object size to be the next multiple of {@link #NUM_BYTES_OBJECT_ALIGNMENT}.
+ */
+ public static long alignObjectSize(long size) {
+ size += (long) NUM_BYTES_OBJECT_ALIGNMENT - 1L;
+ return size - (size % NUM_BYTES_OBJECT_ALIGNMENT);
+ }
+
+ /**
+ * Returns the size in bytes of the byte[] object.
+ */
+ public static long sizeOf(byte[] arr) {
+ return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + arr.length);
+ }
+
+ /**
+ * Returns the size in bytes of the boolean[] object.
+ */
+ public static long sizeOf(boolean[] arr) {
+ return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + arr.length);
+ }
+
+ /**
+ * Returns the size in bytes of the char[] object.
+ */
+ public static long sizeOf(char[] arr) {
+ return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_CHAR * arr.length);
+ }
+
+ /**
+ * Returns the size in bytes of the short[] object.
+ */
+ public static long sizeOf(short[] arr) {
+ return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_SHORT * arr.length);
+ }
+
+ /**
+ * Returns the size in bytes of the int[] object.
+ */
+ public static long sizeOf(int[] arr) {
+ return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_INT * arr.length);
+ }
+
+ /**
+ * Returns the size in bytes of the float[] object.
+ */
+ public static long sizeOf(float[] arr) {
+ return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_FLOAT * arr.length);
+ }
+
+ /**
+ * Returns the size in bytes of the long[] object.
+ */
+ public static long sizeOf(long[] arr) {
+ return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_LONG * arr.length);
+ }
+
+ /**
+ * Returns the size in bytes of the double[] object.
+ */
+ public static long sizeOf(double[] arr) {
+ return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_DOUBLE * arr.length);
+ }
+
+ /**
+ * Estimates the RAM usage by the given object. It will walk the object tree and sum up all
+ * referenced objects.
+ *
+ * <p><b>Resource Usage:</b> This method internally uses a set of
+ * every object seen during traversals so it does allocate memory (it isn't side-effect free).
+ * After the method exits, this memory should be GCed.</p>
+ */
+ public static long sizeOf(Object obj) {
+ return measureObjectSize(obj);
+ }
+
+ /**
+ * Estimates a "shallow" memory usage of the given object. For arrays, this will be the memory
+ * taken by array storage (no subreferences will be followed). For objects, this will be the
+ * memory taken by the fields.
+ *
+ * JVM object alignments are also applied.
+ */
+ public static long shallowSizeOf(Object obj) {
+ if (obj == null) {
+ return 0;
+ }
+ final Class<?> clz = obj.getClass();
+ if (clz.isArray()) {
+ return shallowSizeOfArray(obj);
+ } else {
+ return shallowSizeOfInstance(clz);
+ }
+ }
+
+ /**
+ * Returns the shallow instance size in bytes an instance of the given class would occupy. This
+ * works with all conventional classes and primitive types, but not with arrays (the size then
+ * depends on the number of elements and varies from object to object).
+ *
+ * @throws IllegalArgumentException if {@code clazz} is an array class.
+ * @see #shallowSizeOf(Object)
+ */
+ public static long shallowSizeOfInstance(Class<?> clazz) {
+ if (clazz.isArray()) {
+ throw new IllegalArgumentException("This method does not work with array classes.");
+ }
+ if (clazz.isPrimitive()) {
+ return primitiveSizes.get(clazz);
+ }
+
+ long size = NUM_BYTES_OBJECT_HEADER;
+
+ // Walk type hierarchy
+ for (; clazz != null; clazz = clazz.getSuperclass()) {
+ final Field[] fields = clazz.getDeclaredFields();
+ for (Field f : fields) {
+ if (!Modifier.isStatic(f.getModifiers())) {
+ size = adjustForField(size, f);
+ }
+ }
+ }
+ return alignObjectSize(size);
+ }
+
+ /**
+ * Return shallow size of any <code>array</code>.
+ */
+ private static long shallowSizeOfArray(Object array) {
+ long size = NUM_BYTES_ARRAY_HEADER;
+ final int len = Array.getLength(array);
+ if (len > 0) {
+ Class<?> arrayElementClazz = array.getClass().getComponentType();
+ if (arrayElementClazz.isPrimitive()) {
+ size += (long) len * primitiveSizes.get(arrayElementClazz);
+ } else {
+ size += (long) NUM_BYTES_OBJECT_REF * len;
+ }
+ }
+ return alignObjectSize(size);
+ }
+
+ /*
+ * Non-recursive version of object descend. This consumes more memory than recursive in-depth
+ * traversal but prevents stack overflows on long chains of objects
+ * or complex graphs (a max. recursion depth on my machine was ~5000 objects linked in a chain
+ * so not too much).
+ */
+ private static long measureObjectSize(Object root) {
+ // Objects seen so far.
+ final IdentityHashSet<Object> seen = new IdentityHashSet<Object>();
+ // Class cache with reference Field and precalculated shallow size.
+ final IdentityHashMap<Class<?>, ClassCache> classCache = new IdentityHashMap<Class<?>, ClassCache>();
+ // Stack of objects pending traversal. Recursion caused stack overflows.
+ final ArrayList<Object> stack = new ArrayList<Object>();
+ stack.add(root);
+
+ long totalSize = 0;
+ while (!stack.isEmpty()) {
+ final Object ob = stack.remove(stack.size() - 1);
+
+ if (ob == null || seen.contains(ob)) {
+ continue;
+ }
+ seen.add(ob);
+
+ final Class<?> obClazz = ob.getClass();
+ if (obClazz.isArray()) {
+ /*
+ * Consider an array, possibly of primitive types. Push any of its references to
+ * the processing stack and accumulate this array's shallow size.
+ */
+ long size = NUM_BYTES_ARRAY_HEADER;
+ final int len = Array.getLength(ob);
+ if (len > 0) {
+ Class<?> componentClazz = obClazz.getComponentType();
+ if (componentClazz.isPrimitive()) {
+ size += (long) len * primitiveSizes.get(componentClazz);
+ } else {
+ size += (long) NUM_BYTES_OBJECT_REF * len;
+
+ // Push refs for traversal later.
+ for (int i = len; --i >= 0; ) {
+ final Object o = Array.get(ob, i);
+ if (o != null && !seen.contains(o)) {
+ stack.add(o);
+ }
+ }
+ }
+ }
+ totalSize += alignObjectSize(size);
+ } else {
+ /*
+ * Consider an object. Push any references it has to the processing stack
+ * and accumulate this object's shallow size.
+ */
+ try {
+ ClassCache cachedInfo = classCache.get(obClazz);
+ if (cachedInfo == null) {
+ classCache.put(obClazz, cachedInfo = createCacheEntry(obClazz));
+ }
+
+ for (Field f : cachedInfo.referenceFields) {
+ // Fast path to eliminate redundancies.
+ final Object o = f.get(ob);
+ if (o != null && !seen.contains(o)) {
+ stack.add(o);
+ }
+ }
+
+ totalSize += cachedInfo.alignedShallowInstanceSize;
+ } catch (IllegalAccessException e) {
+ // this should never happen as we enabled setAccessible().
+ throw new RuntimeException("Reflective field access failed?", e);
+ }
+ }
+ }
+
+ // Help the GC (?).
+ seen.clear();
+ stack.clear();
+ classCache.clear();
+
+ return totalSize;
+ }
+
+ /**
+ * Create a cached information about shallow size and reference fields for a given class.
+ */
+ private static ClassCache createCacheEntry(final Class<?> clazz) {
+ ClassCache cachedInfo;
+ long shallowInstanceSize = NUM_BYTES_OBJECT_HEADER;
+ final ArrayList<Field> referenceFields = new ArrayList<Field>(32);
+ for (Class<?> c = clazz; c != null; c = c.getSuperclass()) {
+ final Field[] fields = c.getDeclaredFields();
+ for (final Field f : fields) {
+ if (!Modifier.isStatic(f.getModifiers())) {
+ shallowInstanceSize = adjustForField(shallowInstanceSize, f);
+
+ if (!f.getType().isPrimitive()) {
+ f.setAccessible(true);
+ referenceFields.add(f);
+ }
+ }
+ }
+ }
+
+ cachedInfo = new ClassCache(
+ alignObjectSize(shallowInstanceSize),
+ referenceFields.toArray(new Field[referenceFields.size()]));
+ return cachedInfo;
+ }
+
+ /**
+ * This method returns the maximum representation size of an object. <code>sizeSoFar</code> is the
+ * object's size measured so far. <code>f</code> is the field being probed.
+ *
+ * <p>The returned offset will be the maximum of whatever was measured so far and
+ * <code>f</code> field's offset and representation size (unaligned).
+ */
+ private static long adjustForField(long sizeSoFar, final Field f) {
+ final Class<?> type = f.getType();
+ final int fsize = type.isPrimitive() ? primitiveSizes.get(type) : NUM_BYTES_OBJECT_REF;
+ if (objectFieldOffsetMethod != null) {
+ try {
+ final long offsetPlusSize =
+ ((Number) objectFieldOffsetMethod.invoke(theUnsafe, f)).longValue() + fsize;
+ return Math.max(sizeSoFar, offsetPlusSize);
+ } catch (IllegalAccessException ex) {
+ throw new RuntimeException("Access problem with sun.misc.Unsafe", ex);
+ } catch (InvocationTargetException ite) {
+ final Throwable cause = ite.getCause();
+ if (cause instanceof RuntimeException) {
+ throw (RuntimeException) cause;
+ }
+ if (cause instanceof Error) {
+ throw (Error) cause;
+ }
+ // this should never happen (Unsafe does not declare
+ // checked Exceptions for this method), but who knows?
+ throw new RuntimeException("Call to Unsafe's objectFieldOffset() throwed " +
+ "checked Exception when accessing field " +
+ f.getDeclaringClass().getName() + "#" + f.getName(), cause);
+ }
+ } else {
+ // TODO: No alignments based on field type/ subclass fields alignments?
+ return sizeSoFar + fsize;
+ }
+ }
+
+ /**
+ * Return the set of unsupported JVM features that improve the estimation.
+ */
+ public static EnumSet<JvmFeature> getUnsupportedFeatures() {
+ EnumSet<JvmFeature> unsupported = EnumSet.allOf(JvmFeature.class);
+ unsupported.removeAll(supportedFeatures);
+ return unsupported;
+ }
+
+ /**
+ * Return the set of supported JVM features that improve the estimation.
+ */
+ public static EnumSet<JvmFeature> getSupportedFeatures() {
+ return EnumSet.copyOf(supportedFeatures);
+ }
+
+ /**
+ * Returns <code>size</code> in human-readable units (GB, MB, KB or bytes).
+ */
+ public static String humanReadableUnits(long bytes) {
+ return humanReadableUnits(bytes,
+ new DecimalFormat("0.#", DecimalFormatSymbols.getInstance(Locale.ROOT)));
+ }
+
+ /**
+ * Returns <code>size</code> in human-readable units (GB, MB, KB or bytes).
+ */
+ public static String humanReadableUnits(long bytes, DecimalFormat df) {
+ if (bytes / ONE_GB > 0) {
+ return df.format((float) bytes / ONE_GB) + " GB";
+ } else if (bytes / ONE_MB > 0) {
+ return df.format((float) bytes / ONE_MB) + " MB";
+ } else if (bytes / ONE_KB > 0) {
+ return df.format((float) bytes / ONE_KB) + " KB";
+ } else {
+ return bytes + " bytes";
+ }
+ }
+
+ /**
+ * Return a human-readable size of a given object.
+ *
+ * @see #sizeOf(Object)
+ * @see #humanReadableUnits(long)
+ */
+ public static String humanSizeOf(Object object) {
+ return humanReadableUnits(sizeOf(object));
+ }
+
+ /**
+ * An identity hash set implemented using open addressing. No null keys are allowed.
+ *
+ * TODO: If this is useful outside this class, make it public - needs some work
+ */
+ static final class IdentityHashSet<KType> implements Iterable<KType> {
+
+ /**
+ * Default load factor.
+ */
+ public final static float DEFAULT_LOAD_FACTOR = 0.75f;
+
+ /**
+ * Minimum capacity for the set.
+ */
+ public final static int MIN_CAPACITY = 4;
+
+ /**
+ * All of set entries. Always of power of two length.
+ */
+ public Object[] keys;
+
+ /**
+ * Cached number of assigned slots.
+ */
+ public int assigned;
+
+ /**
+ * The load factor for this set (fraction of allocated or deleted slots before the buffers must
+ * be rehashed or reallocated).
+ */
+ public final float loadFactor;
+
+ /**
+ * Cached capacity threshold at which we must resize the buffers.
+ */
+ private int resizeThreshold;
+
+ /**
+ * Creates a hash set with the default capacity of 16. load factor of {@value
+ * #DEFAULT_LOAD_FACTOR}. `
+ */
+ public IdentityHashSet() {
+ this(16, DEFAULT_LOAD_FACTOR);
+ }
+
+ /**
+ * Creates a hash set with the given capacity, load factor of {@value #DEFAULT_LOAD_FACTOR}.
+ */
+ public IdentityHashSet(int initialCapacity) {
+ this(initialCapacity, DEFAULT_LOAD_FACTOR);
+ }
+
+ /**
+ * Creates a hash set with the given capacity and load factor.
+ */
+ public IdentityHashSet(int initialCapacity, float loadFactor) {
+ initialCapacity = Math.max(MIN_CAPACITY, initialCapacity);
+
+ assert initialCapacity > 0 : "Initial capacity must be between (0, "
+ + Integer.MAX_VALUE + "].";
+ assert loadFactor > 0 && loadFactor < 1 : "Load factor must be between (0, 1).";
+ this.loadFactor = loadFactor;
+ allocateBuffers(roundCapacity(initialCapacity));
+ }
+
+ /**
+ * Adds a reference to the set. Null keys are not allowed.
+ */
+ public boolean add(KType e) {
+ assert e != null : "Null keys not allowed.";
+
+ if (assigned >= resizeThreshold) {
+ expandAndRehash();
+ }
+
+ final int mask = keys.length - 1;
+ int slot = rehash(e) & mask;
+ Object existing;
+ while ((existing = keys[slot]) != null) {
+ if (e == existing) {
+ return false; // already found.
+ }
+ slot = (slot + 1) & mask;
+ }
+ assigned++;
+ keys[slot] = e;
+ return true;
+ }
+
+ /**
+ * Checks if the set contains a given ref.
+ */
+ public boolean contains(KType e) {
+ final int mask = keys.length - 1;
+ int slot = rehash(e) & mask;
+ Object existing;
+ while ((existing = keys[slot]) != null) {
+ if (e == existing) {
+ return true;
+ }
+ slot = (slot + 1) & mask;
+ }
+ return false;
+ }
+
+ /**
+ * Rehash via MurmurHash.
+ *
+ * <p>The implementation is based on the
+ * finalization step from Austin Appleby's
+ * <code>MurmurHash3</code>.
+ *
+ * @see "http://sites.google.com/site/murmurhash/"
+ */
+ private static int rehash(Object o) {
+ int k = System.identityHashCode(o);
+ k ^= k >>> 16;
+ k *= 0x85ebca6b;
+ k ^= k >>> 13;
+ k *= 0xc2b2ae35;
+ k ^= k >>> 16;
+ return k;
+ }
+
+ /**
+ * Expand the internal storage buffers (capacity) or rehash current keys and values if there are
+ * a lot of deleted slots.
+ */
+ private void expandAndRehash() {
+ final Object[] oldKeys = this.keys;
+
+ assert assigned >= resizeThreshold;
+ allocateBuffers(nextCapacity(keys.length));
+
+ /*
+ * Rehash all assigned slots from the old hash table.
+ */
+ final int mask = keys.length - 1;
+ for (int i = 0; i < oldKeys.length; i++) {
+ final Object key = oldKeys[i];
+ if (key != null) {
+ int slot = rehash(key) & mask;
+ while (keys[slot] != null) {
+ slot = (slot + 1) & mask;
+ }
+ keys[slot] = key;
+ }
+ }
+ Arrays.fill(oldKeys, null);
+ }
+
+ /**
+ * Allocate internal buffers for a given capacity.
+ *
+ * @param capacity New capacity (must be a power of two).
+ */
+ private void allocateBuffers(int capacity) {
+ this.keys = new Object[capacity];
+ this.resizeThreshold = (int) (capacity * DEFAULT_LOAD_FACTOR);
+ }
+
+ /**
+ * Return the next possible capacity, counting from the current buffers' size.
+ */
+ protected int nextCapacity(int current) {
+ assert current > 0 && Long.bitCount(current) == 1 : "Capacity must be a power of two.";
+ assert ((current << 1) > 0) : "Maximum capacity exceeded ("
+ + (0x80000000 >>> 1) + ").";
+
+ if (current < MIN_CAPACITY / 2) {
+ current = MIN_CAPACITY / 2;
+ }
+ return current << 1;
+ }
+
+ /**
+ * Round the capacity to the next allowed value.
+ */
+ protected int roundCapacity(int requestedCapacity) {
+ // Maximum positive integer that is a power of two.
+ if (requestedCapacity > (0x80000000 >>> 1)) {
+ return (0x80000000 >>> 1);
+ }
+
+ int capacity = MIN_CAPACITY;
+ while (capacity < requestedCapacity) {
+ capacity <<= 1;
+ }
+
+ return capacity;
+ }
+
+ public void clear() {
+ assigned = 0;
+ Arrays.fill(keys, null);
+ }
+
+ public int size() {
+ return assigned;
+ }
+
+ public boolean isEmpty() {
+ return size() == 0;
+ }
+
+ @Override
+ public Iterator<KType> iterator() {
+ return new Iterator<KType>() {
+ int pos = -1;
+ Object nextElement = fetchNext();
+
+ @Override
+ public boolean hasNext() {
+ return nextElement != null;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public KType next() {
+ Object r = this.nextElement;
+ if (r == null) {
+ throw new NoSuchElementException();
+ }
+ this.nextElement = fetchNext();
+ return (KType) r;
+ }
+
+ private Object fetchNext() {
+ pos++;
+ while (pos < keys.length && keys[pos] == null) {
+ pos++;
+ }
+
+ return (pos >= keys.length ? null : keys[pos]);
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+ }
+
+
+ /**
+ * Some useful constants.
+ **/
+
+ static class Constants {
+
+ private Constants() {
+ } // can't construct
+
+ /**
+ * JVM vendor info.
+ */
+ public static final String JVM_VENDOR = System.getProperty("java.vm.vendor");
+ public static final String JVM_VERSION = System.getProperty("java.vm.version");
+ public static final String JVM_NAME = System.getProperty("java.vm.name");
+
+ /**
+ * The value of <tt>System.getProperty("java.version")</tt>.
+ **/
+ public static final String JAVA_VERSION = System.getProperty("java.version");
+
+ /**
+ * The value of <tt>System.getProperty("os.name")</tt>.
+ **/
+ public static final String OS_NAME = System.getProperty("os.name");
+ /**
+ * True iff running on Linux.
+ */
+ public static final boolean LINUX = OS_NAME.startsWith("Linux");
+ /**
+ * True iff running on Windows.
+ */
+ public static final boolean WINDOWS = OS_NAME.startsWith("Windows");
+ /**
+ * True iff running on SunOS.
+ */
+ public static final boolean SUN_OS = OS_NAME.startsWith("SunOS");
+ /**
+ * True iff running on Mac OS X
+ */
+ public static final boolean MAC_OS_X = OS_NAME.startsWith("Mac OS X");
+ /**
+ * True iff running on FreeBSD
+ */
+ public static final boolean FREE_BSD = OS_NAME.startsWith("FreeBSD");
+
+ public static final String OS_ARCH = System.getProperty("os.arch");
+ public static final String OS_VERSION = System.getProperty("os.version");
+ public static final String JAVA_VENDOR = System.getProperty("java.vendor");
+
+ /**
+ * @deprecated With Lucene 4.0, we are always on Java 6
+ */
+ @Deprecated
+ public static final boolean JRE_IS_MINIMUM_JAVA6 =
+ new Boolean(true).booleanValue(); // prevent inlining in foreign class files
+
+ public static final boolean JRE_IS_MINIMUM_JAVA7;
+ public static final boolean JRE_IS_MINIMUM_JAVA8;
+
+ /**
+ * True iff running on a 64bit JVM
+ */
+ public static final boolean JRE_IS_64BIT;
+
+ static {
+ boolean is64Bit = false;
+ try {
+ final Class<?> unsafeClass = Class.forName("sun.misc.Unsafe");
+ final Field unsafeField = unsafeClass.getDeclaredField("theUnsafe");
+ unsafeField.setAccessible(true);
+ final Object unsafe = unsafeField.get(null);
+ final int addressSize = ((Number) unsafeClass.getMethod("addressSize")
+ .invoke(unsafe)).intValue();
+ //System.out.println("Address size: " + addressSize);
+ is64Bit = addressSize >= 8;
+ } catch (Exception e) {
+ final String x = System.getProperty("sun.arch.data.model");
+ if (x != null) {
+ is64Bit = x.indexOf("64") != -1;
+ } else {
+ if (OS_ARCH != null && OS_ARCH.indexOf("64") != -1) {
+ is64Bit = true;
+ } else {
+ is64Bit = false;
+ }
+ }
+ }
+ JRE_IS_64BIT = is64Bit;
+
+ // this method only exists in Java 7:
+ boolean v7 = true;
+ try {
+ Throwable.class.getMethod("getSuppressed");
+ } catch (NoSuchMethodException nsme) {
+ v7 = false;
+ }
+ JRE_IS_MINIMUM_JAVA7 = v7;
+
+ if (JRE_IS_MINIMUM_JAVA7) {
+ // this method only exists in Java 8:
+ boolean v8 = true;
+ try {
+ Collections.class.getMethod("emptySortedSet");
+ } catch (NoSuchMethodException nsme) {
+ v8 = false;
+ }
+ JRE_IS_MINIMUM_JAVA8 = v8;
+ } else {
+ JRE_IS_MINIMUM_JAVA8 = false;
+ }
+ }
+
+ }
+}
\ No newline at end of file
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java
index 98613cc..4c93376 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java
@@ -32,16 +32,45 @@ public class TsFileMetaDataCache {
private static final Logger logger = LoggerFactory.getLogger(TsFileMetaDataCache.class);
- private static final int MEMORY_THRESHOLD_IN_B = (int) (50*0.3*1024*1024*1024);
+ private static final int MEMORY_THRESHOLD_IN_B = (int) (50 * 0.3 * 1024 * 1024 * 1024);
/**
- * key: The file seriesPath of tsfile.
+ * key: The file seriesPath of tsfile. value: TsFileMetaData
*/
private LruLinkedHashMap<String, TsFileMetaData> cache;
private AtomicLong cacheHintNum = new AtomicLong();
private AtomicLong cacheRequestNum = new AtomicLong();
+ /**
+ * estimate size of a deviceIndexMap entry in TsFileMetaData.
+ */
+ private long deviceIndexMapEntrySize = 0;
+ /**
+ * estimate size of measurementSchema entry in TsFileMetaData.
+ */
+ private long measurementSchemaEntrySize = 0;
+ /**
+ * estimate size of version and CreateBy in TsFileMetaData.
+ */
+ private long versionAndCreatebySize = 10;
+
private TsFileMetaDataCache() {
- cache = new LruLinkedHashMap<>(MEMORY_THRESHOLD_IN_B, true);
+ cache = new LruLinkedHashMap<String, TsFileMetaData>(MEMORY_THRESHOLD_IN_B, true) {
+ @Override
+ protected long calEntrySize(String key, TsFileMetaData value) {
+ if (deviceIndexMapEntrySize == 0 && value.getDeviceMap().size() > 0) {
+ deviceIndexMapEntrySize = RamUsageEstimator
+ .sizeOf(value.getDeviceMap().entrySet().iterator().next());
+ }
+ if (measurementSchemaEntrySize == 0 && value.getMeasurementSchema().size() > 0) {
+ measurementSchemaEntrySize = RamUsageEstimator
+ .sizeOf(value.getMeasurementSchema().entrySet().iterator().next());
+ }
+ long valueSize = value.getDeviceMap().size() * deviceIndexMapEntrySize
+ + measurementSchemaEntrySize * value.getMeasurementSchema().size()
+ + versionAndCreatebySize;
+ return key.length() + valueSize;
+ }
+ };
}
public static TsFileMetaDataCache getInstance() {
@@ -49,7 +78,7 @@ public class TsFileMetaDataCache {
}
/**
- * get the TsFileMetaData for the given path.
+ * get the TsFileMetaData for given path.
*
* @param path -given path
*/
@@ -100,7 +129,7 @@ public class TsFileMetaDataCache {
}
}
- /*
+ /**
* Singleton pattern
*/
private static class TsFileMetaDataCacheHolder {
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java
index 82d1322..9b012c2 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java
@@ -58,7 +58,7 @@ public class TsFileMetadataUtils {
*
* @param filePath -file path
* @param deviceId -device id
- * @param fileMetaData -ts file meta data
+ * @param fileMetaData -tsfile meta data
* @return -device meta data
*/
public static TsDeviceMetadata getTsRowGroupBlockMetaData(String filePath, String deviceId,
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index f6f9990..f95d40a 100755
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -470,15 +470,15 @@ public class StorageGroupProcessor {
* returns the top k% sensors most frequently used in queries.
*/
public Set calTopKSensor(String sensorId, double k) {
- int num = (int) (lruForSensorUsedInQuery.size() * k);
+ int num = (int) (lruForSensorUsedInQuery.size() * k) + 1;
Set<String> sensorSet = new HashSet<>(num);
- sensorSet.add(sensorId);
for (String sensor : lruForSensorUsedInQuery) {
- if (num-- >= 0) {
+ if (--num > 0) {
sensorSet.add(sensor);
}
}
+ sensorSet.add(sensorId);
return sensorSet;
}
[incubator-iotdb] 02/02: merge master
Posted by su...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
suyue pushed a commit to branch feature_query_cache
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit e63b769d06bb4e001bdc80e56b45d9989ffb47f8
Merge: cec8ae5 88f8e5b
Author: suyue <23...@qq.com>
AuthorDate: Wed Jul 17 10:44:41 2019 +0800
merge master
.../3-Encoding.md | 10 +-
.../1-IoTDB Query Statement.md | 2 +-
.../3-Encoding.md | 10 +-
.../1-IoTDB Query Statement.md | 2 +-
example/kafka/pom.xml | 2 +-
example/pom.xml | 2 +-
example/rocketmq/pom.xml | 2 +-
grafana/pom.xml | 6 +-
iotdb-cli/cli/bin/start-client.bat | 6 +-
iotdb-cli/cli/bin/start-client.sh | 9 +-
iotdb-cli/pom.xml | 4 +-
iotdb/iotdb/conf/iotdb-engine.properties | 23 +-
iotdb/pom.xml | 2 +-
.../iotdb/db/auth/role/LocalFileRoleAccessor.java | 10 +-
.../iotdb/db/auth/user/LocalFileUserAccessor.java | 7 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 79 ++++-
.../org/apache/iotdb/db/conf/IoTDBConstant.java | 1 +
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 51 ++-
.../iotdb/db/conf/adapter/CompressionRatio.java | 184 +++++++++++
.../iotdb/db/conf/adapter/IDynamicAdapter.java | 47 +++
.../db/conf/adapter/IoTDBConfigDynamicAdapter.java | 275 ++++++++++++++++
.../db/conf/directories/DirectoryManager.java | 3 +
.../iotdb/db/cost/statistic/Measurement.java | 29 +-
.../org/apache/iotdb/db/engine/StorageEngine.java | 38 +--
.../iotdb/db/engine/cache/DeviceMetaDataCache.java | 20 +-
.../iotdb/db/engine/cache/LruLinkedHashMap.java | 10 +-
.../iotdb/db/engine/cache/TsFileMetaDataCache.java | 7 +-
.../iotdb/db/engine/cache/TsFileMetadataUtils.java | 10 +-
.../iotdb/db/engine/memtable/ChunkBufferPool.java | 4 +-
.../db/engine/querycontext/ReadOnlyMemChunk.java | 2 +-
.../iotdb/db/engine/storagegroup/FlushManager.java | 2 +-
.../engine/storagegroup/StorageGroupProcessor.java | 41 ++-
.../db/engine/storagegroup/TsFileProcessor.java | 42 ++-
.../db/exception/ConfigAdjusterException.java | 37 +++
.../org/apache/iotdb/db/metadata/MManager.java | 91 ++++--
.../iotdb/db/qp/executor/QueryProcessExecutor.java | 62 ++--
.../org/apache/iotdb/db/rescon/MemTablePool.java | 23 +-
.../java/org/apache/iotdb/db/service/IoTDB.java | 17 +-
.../org/apache/iotdb/db/service/JDBCService.java | 2 +-
.../org/apache/iotdb/db/service/TSServiceImpl.java | 73 ++---
.../writelog/manager/MultiFileLogNodeManager.java | 3 +-
.../db/writelog/node/ExclusiveWriteLogNode.java | 12 +-
.../db/conf/adapter/CompressionRatioTest.java | 115 +++++++
.../adapter/IoTDBConfigDynamicAdapterTest.java | 148 +++++++++
.../db/cost/statistic/PerformanceStatTest.java | 2 +-
.../iotdb/db/engine/MetadataManagerHelper.java | 1 +
.../iotdb/db/engine/memtable/MemTablePoolTest.java | 20 +-
.../engine/modification/DeletionFileNodeTest.java | 1 +
.../db/engine/modification/DeletionQueryTest.java | 3 +-
.../engine/modification/ModificationFileTest.java | 1 +
.../integration/IoTDBAggregationSmallDataIT.java | 2 +-
.../db/integration/IoTDBEngineTimeGeneratorIT.java | 3 +
.../iotdb/db/integration/IoTDBLargeDataIT.java | 3 +
.../iotdb/db/integration/IoTDBMultiSeriesIT.java | 3 +
.../db/integration/IoTDBSequenceDataQueryIT.java | 3 +
.../iotdb/db/integration/IoTDBSeriesReaderIT.java | 3 +
.../org/apache/iotdb/db/metadata/MGraphTest.java | 1 +
.../iotdb/db/metadata/MManagerAdvancedTest.java | 2 +
.../iotdb/db/metadata/MManagerBasicTest.java | 1 +
.../iotdb/db/metadata/MManagerImproveTest.java | 1 +
.../org/apache/iotdb/db/metadata/MTreeTest.java | 1 +
.../org/apache/iotdb/db/metadata/MetadataTest.java | 1 +
.../org/apache/iotdb/db/qp/QueryProcessorTest.java | 4 +
.../dataset/EngineDataSetWithValueFilterTest.java | 5 +
.../iotdb/db/query/reader/ReaderTestHelper.java | 7 +-
.../apache/iotdb/db/utils/EnvironmentUtils.java | 19 ++
.../iotdb/db/writelog/IoTDBLogFileSizeTest.java | 3 +
jdbc/pom.xml | 8 +-
pom.xml | 14 +-
service-rpc/pom.xml | 4 +-
spark/pom.xml | 2 +-
tsfile/pom.xml | 2 +-
.../iotdb/tsfile/encoding/decoder/Decoder.java | 10 +
.../encoding/decoder/RegularDataDecoder.java | 289 +++++++++++++++++
.../encoding/encoder/RegularDataEncoder.java | 361 +++++++++++++++++++++
.../tsfile/encoding/encoder/TSEncodingBuilder.java | 27 +-
.../tsfile/file/metadata/enums/TSEncoding.java | 6 +-
.../write/writer/RestorableTsFileIOWriter.java | 3 +-
.../decoder/delta/DeltaBinaryEncoderLongTest.java | 32 +-
.../regular/RegularDataEncoderIntegerTest.java | 145 +++++++++
.../regular/RegularDataEncoderLongTest.java | 223 +++++++++++++
.../iotdb/tsfile/write/TsFileReadWriteTest.java | 24 +-
82 files changed, 2446 insertions(+), 319 deletions(-)
diff --cc iotdb/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
index 5a82116,994622a..3dd72d0
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
@@@ -19,16 -19,12 +19,19 @@@
package org.apache.iotdb.db.engine.cache;
import java.io.IOException;
-import java.util.LinkedHashMap;
+import java.util.ArrayList;
++import java.util.HashSet;
+import java.util.List;
+ import java.util.Map;
-import java.util.Objects;
+import java.util.Set;
- import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
++import org.apache.iotdb.db.conf.IoTDBConfig;
++import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.StorageEngine;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadata;
import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData;
+import org.apache.iotdb.tsfile.read.common.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@@ -39,35 -34,18 +42,37 @@@
public class DeviceMetaDataCache {
private static final Logger logger = LoggerFactory.getLogger(DeviceMetaDataCache.class);
++ private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
- private static final int CACHE_SIZE = 100;
+ private static StorageEngine storageEngine = StorageEngine.getInstance();
+
- private static final int MEMORY_THRESHOLD_IN_B = (int) (50 * 0.3 * 1024 * 1024 * 1024);
++ private static final long MEMORY_THRESHOLD_IN_B = (long) (0.3 * config
++ .getAllocateMemoryForRead());
/**
- * key: the file path + deviceId.
+ * key: file path dot deviceId dot sensorId.
+ * <p>
+ * value: chunkMetaData list of one timeseries in the file.
*/
- private LinkedHashMap<String, TsDeviceMetadata> lruCache;
+ private LruLinkedHashMap<String, List<ChunkMetaData>> lruCache;
private AtomicLong cacheHintNum = new AtomicLong();
private AtomicLong cacheRequestNum = new AtomicLong();
- private DeviceMetaDataCache(int cacheSize) {
- lruCache = new LruLinkedHashMap(cacheSize, true);
+ /**
+ * approximate estimate of chunkMetaData size
+ */
+ private long chunkMetaDataSize = 0;
+
- private DeviceMetaDataCache(int memoryThreshold) {
++ private DeviceMetaDataCache(long memoryThreshold) {
+ lruCache = new LruLinkedHashMap<String, List<ChunkMetaData>>(memoryThreshold, true) {
+ @Override
+ protected long calEntrySize(String key, List<ChunkMetaData> value) {
- if (chunkMetaDataSize == 0 && value.size() > 0) {
++ if (chunkMetaDataSize == 0 && !value.isEmpty()) {
+ chunkMetaDataSize = RamUsageEstimator.sizeOf(value.get(0));
+ }
+ return value.size() * chunkMetaDataSize + key.length();
+ }
+ };
}
public static DeviceMetaDataCache getInstance() {
@@@ -108,49 -84,12 +113,50 @@@
logger.debug("Cache didn't hint: the number of requests for cache is {}",
cacheRequestNum.get());
}
+ TsFileMetaData fileMetaData = TsFileMetaDataCache.getInstance().get(filePath);
TsDeviceMetadata blockMetaData = TsFileMetadataUtils
- .getTsRowGroupBlockMetaData(filePath, deviceId,
+ .getTsRowGroupBlockMetaData(filePath, seriesPath.getDevice(),
fileMetaData);
- ConcurrentHashMap<Path, List<ChunkMetaData>> chunkMetaData = TsFileMetadataUtils
++ Map<Path, List<ChunkMetaData>> chunkMetaData = TsFileMetadataUtils
+ .getChunkMetaDataList(calHotSensorSet(seriesPath), blockMetaData);
synchronized (lruCache) {
- lruCache.put(jointPath, blockMetaData);
- return lruCache.get(jointPath);
+ chunkMetaData.forEach((path, chunkMetaDataList) -> {
+ String k = devicePathStr + "." + path.getMeasurement();
+ if (!lruCache.containsKey(k)) {
+ lruCache.put(k, chunkMetaDataList);
+ }
+ });
+ if (chunkMetaData.containsKey(seriesPath)) {
+ return new ArrayList<>(chunkMetaData.get(seriesPath));
+ }
+ return new ArrayList<>();
+ }
+ }
+ }
+
+ /**
+ * calculate the most frequently query sensors set.
+ *
+ * @param seriesPath the series to be queried in a query statements.
+ */
+ private Set<String> calHotSensorSet(Path seriesPath) throws IOException {
+ double usedMemProportion = lruCache.getUsedMemoryProportion();
+
+ if (usedMemProportion < 0.6) {
- return null;
++ return new HashSet<>();
+ } else {
+ double hotSensorProportion;
+ if (usedMemProportion < 0.8) {
+ hotSensorProportion = 0.1;
+ } else {
+ hotSensorProportion = 0.05;
+ }
+ try {
+ return storageEngine
- .calTopKSensor(seriesPath.getDevice(), seriesPath.getMeasurement(), hotSensorProportion);
++ .calTopKSensor(seriesPath.getDevice(), seriesPath.getMeasurement(),
++ hotSensorProportion);
+ } catch (Exception e) {
+ throw new IOException(e);
}
}
}
diff --cc iotdb/src/main/java/org/apache/iotdb/db/engine/cache/LruLinkedHashMap.java
index 05dd961,0000000..8706a20
mode 100644,000000..100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/LruLinkedHashMap.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/LruLinkedHashMap.java
@@@ -1,90 -1,0 +1,90 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.cache;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * This class is a LRU cache. <b>Note: It's not thread safe.</b>
+ */
+public abstract class LruLinkedHashMap<K, V> extends LinkedHashMap<K, V> {
+
+ private static final long serialVersionUID = 1290160928914532649L;
+ private static final float LOAD_FACTOR_MAP = 0.75f;
- private static final int initialCapacity = 128;
++ private static final int INITIAL_CAPACITY = 128;
+ /**
+ * maximum memory threshold.
+ */
- private int maxMemInB;
++ private long maxMemInB;
+ /**
+ * current used memory.
+ */
- private int usedMemInB;
++ private long usedMemInB;
+
- public LruLinkedHashMap(int maxMemInB, boolean isLru) {
- super(initialCapacity, LOAD_FACTOR_MAP, isLru);
++ public LruLinkedHashMap(long maxMemInB, boolean isLru) {
++ super(INITIAL_CAPACITY, LOAD_FACTOR_MAP, isLru);
+ this.maxMemInB = maxMemInB;
+ }
+
+ @Override
+ protected boolean removeEldestEntry(Map.Entry<K, V> eldest) {
+ if (usedMemInB > maxMemInB) {
+ usedMemInB -= calEntrySize(eldest.getKey(), eldest.getValue());
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public V put(K key, V value) {
+ usedMemInB += calEntrySize(key, value);
+ return super.put(key, value);
+ }
+
+ /**
+ * approximate estimate addition size of key and value.
+ */
+ protected abstract long calEntrySize(K key, V value);
+
+ /**
+ * calculate the proportion of used memory.
+ */
+ public double getUsedMemoryProportion() {
+ return usedMemInB * 1.0 / maxMemInB;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ return super.equals(o);
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+}
diff --cc iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java
index 4c93376,cff1a06..c27c833
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java
@@@ -19,7 -19,8 +19,9 @@@
package org.apache.iotdb.db.engine.cache;
import java.io.IOException;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
++import org.apache.iotdb.db.conf.IoTDBConfig;
++import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@@ -30,13 -31,10 +32,14 @@@
public class TsFileMetaDataCache {
private static final Logger logger = LoggerFactory.getLogger(TsFileMetaDataCache.class);
++ private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
+
-
- private static final int MEMORY_THRESHOLD_IN_B = (int) (50 * 0.3 * 1024 * 1024 * 1024);
++ private static final long MEMORY_THRESHOLD_IN_B = (long) (0.25 * config
++ .getAllocateMemoryForRead());
/**
- * key: The file seriesPath of tsfile.
+ * key: The file seriesPath of tsfile. value: TsFileMetaData
*/
- private ConcurrentHashMap<String, TsFileMetaData> cache;
+ private LruLinkedHashMap<String, TsFileMetaData> cache;
private AtomicLong cacheHintNum = new AtomicLong();
private AtomicLong cacheRequestNum = new AtomicLong();
diff --cc iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java
index 9b012c2,ea63b37..5f7da01
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java
@@@ -19,18 -19,10 +19,19 @@@
package org.apache.iotdb.db.engine.cache;
import java.io.IOException;
-import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
++import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import org.apache.iotdb.db.query.control.FileReaderManager;
+import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetaData;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadata;
+import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadataIndex;
import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
+import org.apache.iotdb.tsfile.read.common.Path;
/**
* This class is used to read metadata(<code>TsFileMetaData</code> and
@@@ -66,36 -59,14 +67,35 @@@ public class TsFileMetadataUtils
if (!fileMetaData.getDeviceMap().containsKey(deviceId)) {
return null;
} else {
- try (TsFileSequenceReader reader = new TsFileSequenceReader(filePath)) {
- long offset = fileMetaData.getDeviceMap().get(deviceId).getOffset();
- int size = fileMetaData.getDeviceMap().get(deviceId).getLen();
- ByteBuffer data = ByteBuffer.allocate(size);
- reader.readRaw(offset, size, data);
- data.flip();
- return TsDeviceMetadata.deserializeFrom(data);
+ // get the index information of TsDeviceMetadata
+ TsDeviceMetadataIndex index = fileMetaData.getDeviceMetadataIndex(deviceId);
+ TsFileSequenceReader tsFileReader = FileReaderManager.getInstance().get(filePath, true);
+ // read TsDeviceMetadata from file
- TsDeviceMetadata tsDeviceMetadata = tsFileReader.readTsDeviceMetaData(index);
- return tsDeviceMetadata;
++ return tsFileReader.readTsDeviceMetaData(index);
+ }
+ }
+
+ /**
+ * get ChunkMetaData List of sensors in sensorSet included in all ChunkGroups of this device. If
+ * sensorSet is null, then return metadata of all sensor included in this device.
+ */
- public static ConcurrentHashMap<Path, List<ChunkMetaData>> getChunkMetaDataList(
++ public static Map<Path, List<ChunkMetaData>> getChunkMetaDataList(
+ Set<String> sensorSet, TsDeviceMetadata tsDeviceMetadata) {
- ConcurrentHashMap<Path, List<ChunkMetaData>> pathToChunkMetaDataList = new ConcurrentHashMap<>();
++ Map<Path, List<ChunkMetaData>> pathToChunkMetaDataList = new ConcurrentHashMap<>();
+ for (ChunkGroupMetaData chunkGroupMetaData : tsDeviceMetadata.getChunkGroupMetaDataList()) {
+ List<ChunkMetaData> chunkMetaDataListInOneChunkGroup = chunkGroupMetaData
+ .getChunkMetaDataList();
+ String deviceId = chunkGroupMetaData.getDeviceID();
+ for (ChunkMetaData chunkMetaData : chunkMetaDataListInOneChunkGroup) {
- if (sensorSet == null || sensorSet.contains(chunkMetaData.getMeasurementUid())) {
++ if (sensorSet.isEmpty() || sensorSet.contains(chunkMetaData.getMeasurementUid())) {
+ Path path = new Path(deviceId, chunkMetaData.getMeasurementUid());
+ pathToChunkMetaDataList.putIfAbsent(path, new ArrayList<>());
+ chunkMetaData.setVersion(chunkGroupMetaData.getVersion());
+ pathToChunkMetaDataList.get(path).add(chunkMetaData);
+ }
}
}
+ return pathToChunkMetaDataList;
}
+
}