You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2019/07/25 01:04:12 UTC
[incubator-iotdb] branch master updated: [IOTDB-144]meta data cache
for query (#262)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 126eac7 [IOTDB-144]meta data cache for query (#262)
126eac7 is described below
commit 126eac76d3ff7736d6c3e0c3064c98c8878a113e
Author: suyue <23...@qq.com>
AuthorDate: Thu Jul 25 09:04:06 2019 +0800
[IOTDB-144]meta data cache for query (#262)
* metadata cache for query
---
License | 1 +
.../org/apache/iotdb/db/engine/StorageEngine.java | 10 +
.../iotdb/db/engine/cache/DeviceMetaDataCache.java | 185 ++--
.../iotdb/db/engine/cache/LRULinkedHashMap.java | 90 ++
.../iotdb/db/engine/cache/RamUsageEstimator.java | 1019 ++++++++++++++++++++
.../iotdb/db/engine/cache/TsFileMetaDataCache.java | 99 +-
.../iotdb/db/engine/cache/TsFileMetadataUtils.java | 61 +-
.../engine/storagegroup/StorageGroupProcessor.java | 44 +-
.../iotdb/db/query/control/JobFileManager.java | 4 +-
.../resourceRelated/SeqResourceIterateReader.java | 11 +-
.../SeqResourceReaderByTimestamp.java | 11 +-
.../resourceRelated/UnseqResourceMergeReader.java | 21 +-
.../UnseqResourceReaderByTimestamp.java | 17 +-
13 files changed, 1415 insertions(+), 158 deletions(-)
diff --git a/License b/License
index 58f5c09..b3163ac 100644
--- a/License
+++ b/License
@@ -221,6 +221,7 @@ iotdb/src/main/java/org/apache/iotdb/db/sql/parse/ParseDriver.java
iotdb/src/main/java/org/apache/iotdb/db/sql/parse/ParseError.java
iotdb/src/main/java/org/apache/iotdb/db/sql/parse/ParseException.java
iotdb/src/main/java/org/apache/iotdb/db/sql/parse/ParseUtils.java
+iotdb/src/main/java/org/apache/iotdb/db/engine/cache/RamUsageEstimator.java
================================================================
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
index 49b3692..8b5f811 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.commons.io.FileUtils;
import org.apache.iotdb.db.conf.IoTDBConfig;
@@ -242,6 +243,15 @@ public class StorageEngine implements IService {
}
/**
+ * returns the top k% measurements that are recently used in queries.
+ */
+ public Set calTopKMeasurement(String deviceId, String sensorId, double k)
+ throws StorageEngineException {
+ StorageGroupProcessor storageGroupProcessor = getProcessor(deviceId);
+ return storageGroupProcessor.calTopKMeasurement(sensorId, k);
+ }
+
+ /**
* Append one specified tsfile to the storage group. <b>This method is only provided for
* transmission module</b>
*
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
index 994622a..369e5e0 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
@@ -19,33 +19,60 @@
package org.apache.iotdb.db.engine.cache;
import java.io.IOException;
-import java.util.LinkedHashMap;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
import java.util.Map;
-import java.util.Objects;
+import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.StorageEngine;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadata;
import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData;
+import org.apache.iotdb.tsfile.read.common.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * This class is used to cache <code>DeviceMetaDataCache</code> of tsfile in IoTDB.
+ * This class is used to cache <code>List<ChunkMetaData></code> of tsfile in IoTDB. The caching
+ * strategy is LRU.
*/
public class DeviceMetaDataCache {
private static final Logger logger = LoggerFactory.getLogger(DeviceMetaDataCache.class);
+ private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
- private static final int CACHE_SIZE = 100;
+ private static StorageEngine storageEngine = StorageEngine.getInstance();
+
+ private static final long MEMORY_THRESHOLD_IN_B = (long) (0.3 * config
+ .getAllocateMemoryForRead());
/**
- * key: the file path + deviceId.
+ * key: file path dot deviceId dot sensorId.
+ * <p>
+ * value: chunkMetaData list of one timeseries in the file.
*/
- private LinkedHashMap<String, TsDeviceMetadata> lruCache;
+ private LRULinkedHashMap<String, List<ChunkMetaData>> lruCache;
- private AtomicLong cacheHintNum = new AtomicLong();
+ private AtomicLong cacheHitNum = new AtomicLong();
private AtomicLong cacheRequestNum = new AtomicLong();
- private DeviceMetaDataCache(int cacheSize) {
- lruCache = new LruLinkedHashMap(cacheSize, true);
+ /**
+ * approximate estimation of chunkMetaData size
+ */
+ private long chunkMetaDataSize = 0;
+
+ private DeviceMetaDataCache(long memoryThreshold) {
+ lruCache = new LRULinkedHashMap<String, List<ChunkMetaData>>(memoryThreshold, true) {
+ @Override
+ protected long calEntrySize(String key, List<ChunkMetaData> value) {
+ if (chunkMetaDataSize == 0 && !value.isEmpty()) {
+ chunkMetaDataSize = RamUsageEstimator.sizeOf(value.get(0));
+ }
+ return value.size() * chunkMetaDataSize + key.length() * 2;
+ }
+ };
}
public static DeviceMetaDataCache getInstance() {
@@ -53,43 +80,86 @@ public class DeviceMetaDataCache {
}
/**
- * get {@link TsDeviceMetadata}. THREAD SAFE.
+ * get {@link ChunkMetaData}. THREAD SAFE.
*/
- public TsDeviceMetadata get(String filePath, String deviceId, TsFileMetaData fileMetaData)
+ public List<ChunkMetaData> get(String filePath, Path seriesPath)
throws IOException {
- // The key(the tsfile path and deviceId) for the lruCache
+ StringBuilder builder = new StringBuilder(filePath).append(".").append(seriesPath.getDevice());
+ String pathDeviceStr = builder.toString();
+ String key = builder.append(".").append(seriesPath.getMeasurement()).toString();
+ Object devicePathObject = pathDeviceStr.intern();
- String jointPath = filePath + deviceId;
- Object jointPathObject = jointPath.intern();
synchronized (lruCache) {
cacheRequestNum.incrementAndGet();
- if (lruCache.containsKey(jointPath)) {
- cacheHintNum.incrementAndGet();
+ if (lruCache.containsKey(key)) {
+ cacheHitNum.incrementAndGet();
if (logger.isDebugEnabled()) {
logger.debug(
- "Cache hint: the number of requests for cache is {}, "
+ "Cache hit: the number of requests for cache is {}, "
+ "the number of hints for cache is {}",
- cacheRequestNum.get(), cacheHintNum.get());
+ cacheRequestNum.get(), cacheHitNum.get());
}
- return lruCache.get(jointPath);
+ return new ArrayList<>(lruCache.get(key));
}
}
- synchronized (jointPathObject) {
+ synchronized (devicePathObject) {
synchronized (lruCache) {
- if (lruCache.containsKey(jointPath)) {
- return lruCache.get(jointPath);
+ if (lruCache.containsKey(key)) {
+ cacheHitNum.incrementAndGet();
+ return new ArrayList<>(lruCache.get(key));
}
}
if (logger.isDebugEnabled()) {
- logger.debug("Cache didn't hint: the number of requests for cache is {}",
+ logger.debug("Cache didn't hit: the number of requests for cache is {}",
cacheRequestNum.get());
}
- TsDeviceMetadata blockMetaData = TsFileMetadataUtils
- .getTsRowGroupBlockMetaData(filePath, deviceId,
- fileMetaData);
+ TsFileMetaData fileMetaData = TsFileMetaDataCache.getInstance().get(filePath);
+ TsDeviceMetadata deviceMetaData = TsFileMetadataUtils
+ .getTsDeviceMetaData(filePath, seriesPath, fileMetaData);
+ // If measurement isn't included in the tsfile, empty list is returned.
+ if(deviceMetaData == null){
+ return new ArrayList<>();
+ }
+ Map<Path, List<ChunkMetaData>> chunkMetaData = TsFileMetadataUtils
+ .getChunkMetaDataList(calHotSensorSet(seriesPath), deviceMetaData);
synchronized (lruCache) {
- lruCache.put(jointPath, blockMetaData);
- return lruCache.get(jointPath);
+ chunkMetaData.forEach((path, chunkMetaDataList) -> {
+ String k = pathDeviceStr + "." + path.getMeasurement();
+ if (!lruCache.containsKey(k)) {
+ lruCache.put(k, chunkMetaDataList);
+ }
+ });
+ if (chunkMetaData.containsKey(seriesPath)) {
+ return new ArrayList<>(chunkMetaData.get(seriesPath));
+ }
+ return new ArrayList<>();
+ }
+ }
+ }
+
+ /**
+ * calculate the most frequently query sensors set.
+ *
+ * @param seriesPath the series to be queried in a query statements.
+ */
+ private Set<String> calHotSensorSet(Path seriesPath) throws IOException {
+ double usedMemProportion = lruCache.getUsedMemoryProportion();
+
+ if (usedMemProportion < 0.6) {
+ return new HashSet<>();
+ } else {
+ double hotSensorProportion;
+ if (usedMemProportion < 0.8) {
+ hotSensorProportion = 0.1;
+ } else {
+ hotSensorProportion = 0.05;
+ }
+ try {
+ return storageEngine
+ .calTopKMeasurement(seriesPath.getDevice(), seriesPath.getMeasurement(),
+ hotSensorProportion);
+ } catch (Exception e) {
+ throw new IOException(e);
}
}
}
@@ -104,64 +174,11 @@ public class DeviceMetaDataCache {
}
/**
- * the default LRU cache size is 100. The singleton pattern.
+ * singleton pattern.
*/
private static class RowGroupBlockMetaDataCacheSingleton {
private static final DeviceMetaDataCache INSTANCE = new
- DeviceMetaDataCache(CACHE_SIZE);
- }
-
- /**
- * This class is a map used to cache the <code>RowGroupBlockMetaData</code>. The caching strategy
- * is LRU.
- *
- */
- private class LruLinkedHashMap extends LinkedHashMap<String, TsDeviceMetadata> {
-
- private static final long serialVersionUID = 1290160928914532649L;
- private static final float LOAD_FACTOR_MAP = 0.75f;
- private int maxCapacity;
-
- public LruLinkedHashMap(int maxCapacity, boolean isLru) {
- super(maxCapacity, LOAD_FACTOR_MAP, isLru);
- this.maxCapacity = maxCapacity;
- }
-
- @Override
- protected boolean removeEldestEntry(Map.Entry<String, TsDeviceMetadata> eldest) {
- return size() > maxCapacity;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- return super.equals(o);
- }
-
- @Override
- public int hashCode() {
- return super.hashCode();
- }
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
- DeviceMetaDataCache that = (DeviceMetaDataCache) o;
- return Objects.equals(lruCache, that.lruCache) &&
- Objects.equals(cacheHintNum, that.cacheHintNum) &&
- Objects.equals(cacheRequestNum, that.cacheRequestNum);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(lruCache, cacheHintNum, cacheRequestNum);
+ DeviceMetaDataCache(MEMORY_THRESHOLD_IN_B);
}
-}
+}
\ No newline at end of file
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/LRULinkedHashMap.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/LRULinkedHashMap.java
new file mode 100644
index 0000000..58ab293
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/LRULinkedHashMap.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.cache;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * This class is an LRU cache. <b>Note: It's not thread safe.</b>
+ */
+public abstract class LRULinkedHashMap<K, V> extends LinkedHashMap<K, V> {
+
+ private static final long serialVersionUID = 1290160928914532649L;
+ private static final float LOAD_FACTOR_MAP = 0.75f;
+ private static final int INITIAL_CAPACITY = 128;
+ /**
+ * maximum memory threshold.
+ */
+ private long maxMemInB;
+ /**
+ * current used memory.
+ */
+ private long usedMemInB;
+
+ public LRULinkedHashMap(long maxMemInB, boolean isLru) {
+ super(INITIAL_CAPACITY, LOAD_FACTOR_MAP, isLru);
+ this.maxMemInB = maxMemInB;
+ }
+
+ @Override
+ protected boolean removeEldestEntry(Map.Entry<K, V> eldest) {
+ if (usedMemInB > maxMemInB) {
+ usedMemInB -= calEntrySize(eldest.getKey(), eldest.getValue());
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public V put(K key, V value) {
+ usedMemInB += calEntrySize(key, value);
+ return super.put(key, value);
+ }
+
+ /**
+ * approximately estimate the additional size of key and value.
+ */
+ protected abstract long calEntrySize(K key, V value);
+
+ /**
+ * calculate the proportion of used memory.
+ */
+ public double getUsedMemoryProportion() {
+ return usedMemInB * 1.0 / maxMemInB;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ return super.equals(o);
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/RamUsageEstimator.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/RamUsageEstimator.java
new file mode 100644
index 0000000..9355b28
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/RamUsageEstimator.java
@@ -0,0 +1,1019 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.engine.cache;
+
+import java.lang.management.ManagementFactory;
+import java.lang.reflect.Array;
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.text.DecimalFormat;
+import java.text.DecimalFormatSymbols;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.IdentityHashMap;
+import java.util.Iterator;
+import java.util.Locale;
+import java.util.Map;
+import java.util.NoSuchElementException;
+
+/**
+ * This class is copied from apache lucene, version 4.6.1. Estimates the size(memory representation)
+ * of Java objects. https://github.com/apache/lucene-solr/blob/releases/lucene-solr/4.6.1/lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java
+ *
+ * @lucene.internal
+ * @see #sizeOf(Object)
+ * @see #shallowSizeOf(Object)
+ * @see #shallowSizeOfInstance(Class)
+ */
+public final class RamUsageEstimator {
+
+ /**
+ * JVM diagnostic features.
+ */
+ public static enum JvmFeature {
+ OBJECT_REFERENCE_SIZE("Object reference size estimated using array index scale"),
+ ARRAY_HEADER_SIZE("Array header size estimated using array based offset"),
+ FIELD_OFFSETS("Shallow instance size based on field offsets"),
+ OBJECT_ALIGNMENT("Object alignment retrieved from HotSpotDiagnostic MX bean");
+
+ public final String description;
+
+ private JvmFeature(String description) {
+ this.description = description;
+ }
+
+ @Override
+ public String toString() {
+ return super.name() + " (" + description + ")";
+ }
+ }
+
+ /**
+ * JVM info string for debugging and reports.
+ */
+ public final static String JVM_INFO_STRING;
+
+ /**
+ * One kilobyte bytes.
+ */
+ public static final long ONE_KB = 1024;
+
+ /**
+ * One megabyte bytes.
+ */
+ public static final long ONE_MB = ONE_KB * ONE_KB;
+
+ /**
+ * One gigabyte bytes.
+ */
+ public static final long ONE_GB = ONE_KB * ONE_MB;
+
+ /**
+ * No instantiation.
+ */
+ private RamUsageEstimator() {
+ }
+
+ public final static int NUM_BYTES_BOOLEAN = 1;
+ public final static int NUM_BYTES_BYTE = 1;
+ public final static int NUM_BYTES_CHAR = 2;
+ public final static int NUM_BYTES_SHORT = 2;
+ public final static int NUM_BYTES_INT = 4;
+ public final static int NUM_BYTES_FLOAT = 4;
+ public final static int NUM_BYTES_LONG = 8;
+ public final static int NUM_BYTES_DOUBLE = 8;
+
+ /**
+ * Number of bytes this jvm uses to represent an object reference.
+ */
+ public final static int NUM_BYTES_OBJECT_REF;
+
+ /**
+ * Number of bytes to represent an object header (no fields, no alignments).
+ */
+ public final static int NUM_BYTES_OBJECT_HEADER;
+
+ /**
+ * Number of bytes to represent an array header (no content, but with alignments).
+ */
+ public final static int NUM_BYTES_ARRAY_HEADER;
+
+ /**
+ * A constant specifying the object alignment boundary inside the JVM. Objects will always take a
+ * full multiple of this constant, possibly wasting some space.
+ */
+ public final static int NUM_BYTES_OBJECT_ALIGNMENT;
+
+ /**
+ * Sizes of primitive classes.
+ */
+ private static final Map<Class<?>, Integer> primitiveSizes;
+
+ static {
+ primitiveSizes = new IdentityHashMap<Class<?>, Integer>();
+ primitiveSizes.put(boolean.class, Integer.valueOf(NUM_BYTES_BOOLEAN));
+ primitiveSizes.put(byte.class, Integer.valueOf(NUM_BYTES_BYTE));
+ primitiveSizes.put(char.class, Integer.valueOf(NUM_BYTES_CHAR));
+ primitiveSizes.put(short.class, Integer.valueOf(NUM_BYTES_SHORT));
+ primitiveSizes.put(int.class, Integer.valueOf(NUM_BYTES_INT));
+ primitiveSizes.put(float.class, Integer.valueOf(NUM_BYTES_FLOAT));
+ primitiveSizes.put(double.class, Integer.valueOf(NUM_BYTES_DOUBLE));
+ primitiveSizes.put(long.class, Integer.valueOf(NUM_BYTES_LONG));
+ }
+
+ /**
+ * A handle to <code>sun.misc.Unsafe</code>.
+ */
+ private final static Object theUnsafe;
+
+ /**
+ * A handle to <code>sun.misc.Unsafe#fieldOffset(Field)</code>.
+ */
+ private final static Method objectFieldOffsetMethod;
+
+ /**
+ * All the supported "internal" JVM features detected at clinit.
+ */
+ private final static EnumSet<JvmFeature> supportedFeatures;
+
+ /**
+ * Initialize constants and try to collect information about the JVM internals.
+ */
+ static {
+ // Initialize empirically measured defaults. We'll modify them to the current
+ // JVM settings later on if possible.
+ int referenceSize = Constants.JRE_IS_64BIT ? 8 : 4;
+ int objectHeader = Constants.JRE_IS_64BIT ? 16 : 8;
+ // The following is objectHeader + NUM_BYTES_INT, but aligned (object alignment)
+ // so on 64 bit JVMs it'll be align(16 + 4, @8) = 24.
+ int arrayHeader = Constants.JRE_IS_64BIT ? 24 : 12;
+
+ supportedFeatures = EnumSet.noneOf(JvmFeature.class);
+
+ Class<?> unsafeClass = null;
+ Object tempTheUnsafe = null;
+ try {
+ unsafeClass = Class.forName("sun.misc.Unsafe");
+ final Field unsafeField = unsafeClass.getDeclaredField("theUnsafe");
+ unsafeField.setAccessible(true);
+ tempTheUnsafe = unsafeField.get(null);
+ } catch (Exception e) {
+ // Ignore.
+ }
+ theUnsafe = tempTheUnsafe;
+
+ // get object reference size by getting scale factor of Object[] arrays:
+ try {
+ final Method arrayIndexScaleM = unsafeClass.getMethod("arrayIndexScale", Class.class);
+ referenceSize = ((Number) arrayIndexScaleM.invoke(theUnsafe, Object[].class)).intValue();
+ supportedFeatures.add(JvmFeature.OBJECT_REFERENCE_SIZE);
+ } catch (Exception e) {
+ // ignore.
+ }
+
+ // "best guess" based on reference size. We will attempt to modify
+ // these to exact values if there is supported infrastructure.
+ objectHeader = Constants.JRE_IS_64BIT ? (8 + referenceSize) : 8;
+ arrayHeader = Constants.JRE_IS_64BIT ? (8 + 2 * referenceSize) : 12;
+
+ // get the object header size:
+ // - first try out if the field offsets are not scaled (see warning in Unsafe docs)
+ // - get the object header size by getting the field offset of the first field of a dummy object
+ // If the scaling is byte-wise and unsafe is available, enable dynamic size measurement for
+ // estimateRamUsage().
+ Method tempObjectFieldOffsetMethod = null;
+ try {
+ final Method objectFieldOffsetM = unsafeClass.getMethod("objectFieldOffset", Field.class);
+ final Field dummy1Field = DummyTwoLongObject.class.getDeclaredField("dummy1");
+ final int ofs1 = ((Number) objectFieldOffsetM.invoke(theUnsafe, dummy1Field)).intValue();
+ final Field dummy2Field = DummyTwoLongObject.class.getDeclaredField("dummy2");
+ final int ofs2 = ((Number) objectFieldOffsetM.invoke(theUnsafe, dummy2Field)).intValue();
+ if (Math.abs(ofs2 - ofs1) == NUM_BYTES_LONG) {
+ final Field baseField = DummyOneFieldObject.class.getDeclaredField("base");
+ objectHeader = ((Number) objectFieldOffsetM.invoke(theUnsafe, baseField)).intValue();
+ supportedFeatures.add(JvmFeature.FIELD_OFFSETS);
+ tempObjectFieldOffsetMethod = objectFieldOffsetM;
+ }
+ } catch (Exception e) {
+ // Ignore.
+ }
+ objectFieldOffsetMethod = tempObjectFieldOffsetMethod;
+
+ // Get the array header size by retrieving the array base offset
+ // (offset of the first element of an array).
+ try {
+ final Method arrayBaseOffsetM = unsafeClass.getMethod("arrayBaseOffset", Class.class);
+ // we calculate that only for byte[] arrays, it's actually the same for all types:
+ arrayHeader = ((Number) arrayBaseOffsetM.invoke(theUnsafe, byte[].class)).intValue();
+ supportedFeatures.add(JvmFeature.ARRAY_HEADER_SIZE);
+ } catch (Exception e) {
+ // Ignore.
+ }
+
+ NUM_BYTES_OBJECT_REF = referenceSize;
+ NUM_BYTES_OBJECT_HEADER = objectHeader;
+ NUM_BYTES_ARRAY_HEADER = arrayHeader;
+
+ // Try to get the object alignment (the default seems to be 8 on Hotspot,
+ // regardless of the architecture).
+ int objectAlignment = 8;
+ try {
+ final Class<?> beanClazz = Class.forName("com.sun.management.HotSpotDiagnosticMXBean");
+ // Try to get the diagnostic mxbean without calling {@link ManagementFactory#getPlatformMBeanServer()}
+ // which starts AWT thread (and shows junk in the dock) on a Mac:
+ Object hotSpotBean;
+ // Java 7+, HotSpot
+ try {
+ hotSpotBean = ManagementFactory.class
+ .getMethod("getPlatformMXBean", Class.class)
+ .invoke(null, beanClazz);
+ } catch (Exception e1) {
+ // Java 6, HotSpot
+ try {
+ Class<?> sunMF = Class.forName("sun.management.ManagementFactory");
+ hotSpotBean = sunMF.getMethod("getDiagnosticMXBean").invoke(null);
+ } catch (Exception e2) {
+ // Last resort option is an attempt to get it from ManagementFactory's server anyway (may start AWT).
+ hotSpotBean = ManagementFactory.newPlatformMXBeanProxy(
+ ManagementFactory.getPlatformMBeanServer(),
+ "com.sun.management:type=HotSpotDiagnostic", beanClazz);
+ }
+ }
+ if (hotSpotBean != null) {
+ final Method getVMOptionMethod = beanClazz.getMethod("getVMOption", String.class);
+ final Object vmOption = getVMOptionMethod.invoke(hotSpotBean, "ObjectAlignmentInBytes");
+ objectAlignment = Integer.parseInt(
+ vmOption.getClass().getMethod("getValue").invoke(vmOption).toString()
+ );
+ supportedFeatures.add(JvmFeature.OBJECT_ALIGNMENT);
+ }
+ } catch (Exception e) {
+ // Ignore.
+ }
+
+ NUM_BYTES_OBJECT_ALIGNMENT = objectAlignment;
+
+ JVM_INFO_STRING = "[JVM: " +
+ Constants.JVM_NAME + ", " + Constants.JVM_VERSION + ", " + Constants.JVM_VENDOR + ", " +
+ Constants.JAVA_VENDOR + ", " + Constants.JAVA_VERSION + "]";
+ }
+
+ /**
+ * Cached information about a given class.
+ */
+ private static final class ClassCache {
+
+ public final long alignedShallowInstanceSize;
+ public final Field[] referenceFields;
+
+ public ClassCache(long alignedShallowInstanceSize, Field[] referenceFields) {
+ this.alignedShallowInstanceSize = alignedShallowInstanceSize;
+ this.referenceFields = referenceFields;
+ }
+ }
+
+ // Object with just one field to determine the object header size by getting the offset of the dummy field:
+ @SuppressWarnings("unused")
+ private static final class DummyOneFieldObject {
+
+ public byte base;
+ }
+
+ // Another test object for checking, if the difference in offsets of dummy1 and dummy2 is 8 bytes.
+ // Only then we can be sure that those are real, unscaled offsets:
+ @SuppressWarnings("unused")
+ private static final class DummyTwoLongObject {
+
+ public long dummy1, dummy2;
+ }
+
+ /**
+ * Returns true, if the current JVM is fully supported by {@code RamUsageEstimator}. If this
+ * method returns {@code false} you are maybe using a 3rd party Java VM that is not supporting
+ * Oracle/Sun private APIs. The memory estimates can be imprecise then (no way of detecting
+ * compressed references, alignments, etc.). Lucene still tries to use sensible defaults.
+ */
+ public static boolean isSupportedJVM() {
+ return supportedFeatures.size() == JvmFeature.values().length;
+ }
+
+ /**
+ * Aligns an object size to be the next multiple of {@link #NUM_BYTES_OBJECT_ALIGNMENT}.
+ */
+ public static long alignObjectSize(long size) {
+ size += (long) NUM_BYTES_OBJECT_ALIGNMENT - 1L;
+ return size - (size % NUM_BYTES_OBJECT_ALIGNMENT);
+ }
+
+ /**
+ * Returns the size in bytes of the byte[] object.
+ */
+ public static long sizeOf(byte[] arr) {
+ return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + arr.length);
+ }
+
+ /**
+ * Returns the size in bytes of the boolean[] object.
+ */
+ public static long sizeOf(boolean[] arr) {
+ return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + arr.length);
+ }
+
+ /**
+ * Returns the size in bytes of the char[] object.
+ */
+ public static long sizeOf(char[] arr) {
+ return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_CHAR * arr.length);
+ }
+
+ /**
+ * Returns the size in bytes of the short[] object.
+ */
+ public static long sizeOf(short[] arr) {
+ return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_SHORT * arr.length);
+ }
+
+ /**
+ * Returns the size in bytes of the int[] object.
+ */
+ public static long sizeOf(int[] arr) {
+ return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_INT * arr.length);
+ }
+
+ /**
+ * Returns the size in bytes of the float[] object.
+ */
+ public static long sizeOf(float[] arr) {
+ return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_FLOAT * arr.length);
+ }
+
+ /**
+ * Returns the size in bytes of the long[] object.
+ */
+ public static long sizeOf(long[] arr) {
+ return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_LONG * arr.length);
+ }
+
+ /**
+ * Returns the size in bytes of the double[] object.
+ */
+ public static long sizeOf(double[] arr) {
+ return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_DOUBLE * arr.length);
+ }
+
+ /**
+ * Estimates the RAM usage by the given object. It will walk the object tree and sum up all
+ * referenced objects.
+ *
+ * <p><b>Resource Usage:</b> This method internally uses a set of
+ * every object seen during traversals so it does allocate memory (it isn't side-effect free).
+ * After the method exits, this memory should be GCed.</p>
+ */
+ public static long sizeOf(Object obj) {
+ return measureObjectSize(obj);
+ }
+
+ /**
+ * Estimates a "shallow" memory usage of the given object. For arrays, this will be the memory
+ * taken by array storage (no subreferences will be followed). For objects, this will be the
+ * memory taken by the fields.
+ *
+ * JVM object alignments are also applied.
+ */
+ public static long shallowSizeOf(Object obj) {
+ if (obj == null) {
+ return 0;
+ }
+ final Class<?> clz = obj.getClass();
+ if (clz.isArray()) {
+ return shallowSizeOfArray(obj);
+ } else {
+ return shallowSizeOfInstance(clz);
+ }
+ }
+
+ /**
+ * Returns the shallow instance size in bytes an instance of the given class would occupy. This
+ * works with all conventional classes and primitive types, but not with arrays (the size then
+ * depends on the number of elements and varies from object to object).
+ *
+ * @throws IllegalArgumentException if {@code clazz} is an array class.
+ * @see #shallowSizeOf(Object)
+ */
+ public static long shallowSizeOfInstance(Class<?> clazz) {
+ if (clazz.isArray()) {
+ throw new IllegalArgumentException("This method does not work with array classes.");
+ }
+ if (clazz.isPrimitive()) {
+ return primitiveSizes.get(clazz);
+ }
+
+ long size = NUM_BYTES_OBJECT_HEADER;
+
+ // Walk type hierarchy
+ for (; clazz != null; clazz = clazz.getSuperclass()) {
+ final Field[] fields = clazz.getDeclaredFields();
+ for (Field f : fields) {
+ if (!Modifier.isStatic(f.getModifiers())) {
+ size = adjustForField(size, f);
+ }
+ }
+ }
+ return alignObjectSize(size);
+ }
+
+ /**
+ * Return shallow size of any <code>array</code>.
+ */
+ private static long shallowSizeOfArray(Object array) {
+ long size = NUM_BYTES_ARRAY_HEADER;
+ final int len = Array.getLength(array);
+ if (len > 0) {
+ Class<?> arrayElementClazz = array.getClass().getComponentType();
+ if (arrayElementClazz.isPrimitive()) {
+ size += (long) len * primitiveSizes.get(arrayElementClazz);
+ } else {
+ size += (long) NUM_BYTES_OBJECT_REF * len;
+ }
+ }
+ return alignObjectSize(size);
+ }
+
+ /*
+ * Non-recursive version of object descend. This consumes more memory than recursive in-depth
+ * traversal but prevents stack overflows on long chains of objects
+ * or complex graphs (a max. recursion depth on my machine was ~5000 objects linked in a chain
+ * so not too much).
+ */
+ private static long measureObjectSize(Object root) {
+ // Objects seen so far.
+ final IdentityHashSet<Object> seen = new IdentityHashSet<Object>();
+ // Class cache with reference Field and precalculated shallow size.
+ final IdentityHashMap<Class<?>, ClassCache> classCache = new IdentityHashMap<Class<?>, ClassCache>();
+ // Stack of objects pending traversal. Recursion caused stack overflows.
+ final ArrayList<Object> stack = new ArrayList<Object>();
+ stack.add(root);
+
+ long totalSize = 0;
+ while (!stack.isEmpty()) {
+ final Object ob = stack.remove(stack.size() - 1);
+
+ if (ob == null || seen.contains(ob)) {
+ continue;
+ }
+ seen.add(ob);
+
+ final Class<?> obClazz = ob.getClass();
+ if (obClazz.isArray()) {
+ /*
+ * Consider an array, possibly of primitive types. Push any of its references to
+ * the processing stack and accumulate this array's shallow size.
+ */
+ long size = NUM_BYTES_ARRAY_HEADER;
+ final int len = Array.getLength(ob);
+ if (len > 0) {
+ Class<?> componentClazz = obClazz.getComponentType();
+ if (componentClazz.isPrimitive()) {
+ size += (long) len * primitiveSizes.get(componentClazz);
+ } else {
+ size += (long) NUM_BYTES_OBJECT_REF * len;
+
+ // Push refs for traversal later.
+ for (int i = len; --i >= 0; ) {
+ final Object o = Array.get(ob, i);
+ if (o != null && !seen.contains(o)) {
+ stack.add(o);
+ }
+ }
+ }
+ }
+ totalSize += alignObjectSize(size);
+ } else {
+ /*
+ * Consider an object. Push any references it has to the processing stack
+ * and accumulate this object's shallow size.
+ */
+ try {
+ ClassCache cachedInfo = classCache.get(obClazz);
+ if (cachedInfo == null) {
+ classCache.put(obClazz, cachedInfo = createCacheEntry(obClazz));
+ }
+
+ for (Field f : cachedInfo.referenceFields) {
+ // Fast path to eliminate redundancies.
+ final Object o = f.get(ob);
+ if (o != null && !seen.contains(o)) {
+ stack.add(o);
+ }
+ }
+
+ totalSize += cachedInfo.alignedShallowInstanceSize;
+ } catch (IllegalAccessException e) {
+ // this should never happen as we enabled setAccessible().
+ throw new RuntimeException("Reflective field access failed?", e);
+ }
+ }
+ }
+
+ // Help the GC (?).
+ seen.clear();
+ stack.clear();
+ classCache.clear();
+
+ return totalSize;
+ }
+
+ /**
+ * Create a cached information about shallow size and reference fields for a given class.
+ */
+ private static ClassCache createCacheEntry(final Class<?> clazz) {
+ ClassCache cachedInfo;
+ long shallowInstanceSize = NUM_BYTES_OBJECT_HEADER;
+ final ArrayList<Field> referenceFields = new ArrayList<Field>(32);
+ for (Class<?> c = clazz; c != null; c = c.getSuperclass()) {
+ final Field[] fields = c.getDeclaredFields();
+ for (final Field f : fields) {
+ if (!Modifier.isStatic(f.getModifiers())) {
+ shallowInstanceSize = adjustForField(shallowInstanceSize, f);
+
+ if (!f.getType().isPrimitive()) {
+ f.setAccessible(true);
+ referenceFields.add(f);
+ }
+ }
+ }
+ }
+
+ cachedInfo = new ClassCache(
+ alignObjectSize(shallowInstanceSize),
+ referenceFields.toArray(new Field[referenceFields.size()]));
+ return cachedInfo;
+ }
+
+ /**
+ * This method returns the maximum representation size of an object. <code>sizeSoFar</code> is the
+ * object's size measured so far. <code>f</code> is the field being probed.
+ *
+ * <p>The returned offset will be the maximum of whatever was measured so far and
+ * <code>f</code> field's offset and representation size (unaligned).
+ */
+ private static long adjustForField(long sizeSoFar, final Field f) {
+ final Class<?> type = f.getType();
+ final int fsize = type.isPrimitive() ? primitiveSizes.get(type) : NUM_BYTES_OBJECT_REF;
+ if (objectFieldOffsetMethod != null) {
+ try {
+ final long offsetPlusSize =
+ ((Number) objectFieldOffsetMethod.invoke(theUnsafe, f)).longValue() + fsize;
+ return Math.max(sizeSoFar, offsetPlusSize);
+ } catch (IllegalAccessException ex) {
+ throw new RuntimeException("Access problem with sun.misc.Unsafe", ex);
+ } catch (InvocationTargetException ite) {
+ final Throwable cause = ite.getCause();
+ if (cause instanceof RuntimeException) {
+ throw (RuntimeException) cause;
+ }
+ if (cause instanceof Error) {
+ throw (Error) cause;
+ }
+ // this should never happen (Unsafe does not declare
+ // checked Exceptions for this method), but who knows?
+ throw new RuntimeException("Call to Unsafe's objectFieldOffset() throwed " +
+ "checked Exception when accessing field " +
+ f.getDeclaringClass().getName() + "#" + f.getName(), cause);
+ }
+ } else {
+ // TODO: No alignments based on field type/ subclass fields alignments?
+ return sizeSoFar + fsize;
+ }
+ }
+
+ /**
+ * Return the set of unsupported JVM features that improve the estimation.
+ */
+ public static EnumSet<JvmFeature> getUnsupportedFeatures() {
+ EnumSet<JvmFeature> unsupported = EnumSet.allOf(JvmFeature.class);
+ unsupported.removeAll(supportedFeatures);
+ return unsupported;
+ }
+
+ /**
+ * Return the set of supported JVM features that improve the estimation.
+ */
+ public static EnumSet<JvmFeature> getSupportedFeatures() {
+ return EnumSet.copyOf(supportedFeatures);
+ }
+
+ /**
+ * Returns <code>size</code> in human-readable units (GB, MB, KB or bytes).
+ */
+ public static String humanReadableUnits(long bytes) {
+ return humanReadableUnits(bytes,
+ new DecimalFormat("0.#", DecimalFormatSymbols.getInstance(Locale.ROOT)));
+ }
+
+ /**
+ * Returns <code>size</code> in human-readable units (GB, MB, KB or bytes).
+ */
+ public static String humanReadableUnits(long bytes, DecimalFormat df) {
+ if (bytes / ONE_GB > 0) {
+ return df.format((float) bytes / ONE_GB) + " GB";
+ } else if (bytes / ONE_MB > 0) {
+ return df.format((float) bytes / ONE_MB) + " MB";
+ } else if (bytes / ONE_KB > 0) {
+ return df.format((float) bytes / ONE_KB) + " KB";
+ } else {
+ return bytes + " bytes";
+ }
+ }
+
+ /**
+ * Return a human-readable size of a given object.
+ *
+ * @see #sizeOf(Object)
+ * @see #humanReadableUnits(long)
+ */
+ public static String humanSizeOf(Object object) {
+ return humanReadableUnits(sizeOf(object));
+ }
+
+ /**
+ * An identity hash set implemented using open addressing. No null keys are allowed.
+ *
+ * TODO: If this is useful outside this class, make it public - needs some work
+ */
+ static final class IdentityHashSet<KType> implements Iterable<KType> {
+
+ /**
+ * Default load factor.
+ */
+ public final static float DEFAULT_LOAD_FACTOR = 0.75f;
+
+ /**
+ * Minimum capacity for the set.
+ */
+ public final static int MIN_CAPACITY = 4;
+
+ /**
+ * All of set entries. Always of power of two length.
+ */
+ public Object[] keys;
+
+ /**
+ * Cached number of assigned slots.
+ */
+ public int assigned;
+
+ /**
+ * The load factor for this set (fraction of allocated or deleted slots before the buffers must
+ * be rehashed or reallocated).
+ */
+ public final float loadFactor;
+
+ /**
+ * Cached capacity threshold at which we must resize the buffers.
+ */
+ private int resizeThreshold;
+
+ /**
+ * Creates a hash set with the default capacity of 16. load factor of {@value
+ * #DEFAULT_LOAD_FACTOR}. `
+ */
+ public IdentityHashSet() {
+ this(16, DEFAULT_LOAD_FACTOR);
+ }
+
+ /**
+ * Creates a hash set with the given capacity, load factor of {@value #DEFAULT_LOAD_FACTOR}.
+ */
+ public IdentityHashSet(int initialCapacity) {
+ this(initialCapacity, DEFAULT_LOAD_FACTOR);
+ }
+
+ /**
+ * Creates a hash set with the given capacity and load factor.
+ */
+ public IdentityHashSet(int initialCapacity, float loadFactor) {
+ initialCapacity = Math.max(MIN_CAPACITY, initialCapacity);
+
+ assert initialCapacity > 0 : "Initial capacity must be between (0, "
+ + Integer.MAX_VALUE + "].";
+ assert loadFactor > 0 && loadFactor < 1 : "Load factor must be between (0, 1).";
+ this.loadFactor = loadFactor;
+ allocateBuffers(roundCapacity(initialCapacity));
+ }
+
+ /**
+ * Adds a reference to the set. Null keys are not allowed.
+ */
+ public boolean add(KType e) {
+ assert e != null : "Null keys not allowed.";
+
+ if (assigned >= resizeThreshold) {
+ expandAndRehash();
+ }
+
+ final int mask = keys.length - 1;
+ int slot = rehash(e) & mask;
+ Object existing;
+ while ((existing = keys[slot]) != null) {
+ if (e == existing) {
+ return false; // already found.
+ }
+ slot = (slot + 1) & mask;
+ }
+ assigned++;
+ keys[slot] = e;
+ return true;
+ }
+
+ /**
+ * Checks if the set contains a given ref.
+ */
+ public boolean contains(KType e) {
+ final int mask = keys.length - 1;
+ int slot = rehash(e) & mask;
+ Object existing;
+ while ((existing = keys[slot]) != null) {
+ if (e == existing) {
+ return true;
+ }
+ slot = (slot + 1) & mask;
+ }
+ return false;
+ }
+
+ /**
+ * Rehash via MurmurHash.
+ *
+ * <p>The implementation is based on the
+ * finalization step from Austin Appleby's
+ * <code>MurmurHash3</code>.
+ *
+ * @see "http://sites.google.com/site/murmurhash/"
+ */
+ private static int rehash(Object o) {
+ int k = System.identityHashCode(o);
+ k ^= k >>> 16;
+ k *= 0x85ebca6b;
+ k ^= k >>> 13;
+ k *= 0xc2b2ae35;
+ k ^= k >>> 16;
+ return k;
+ }
+
+ /**
+ * Expand the internal storage buffers (capacity) or rehash current keys and values if there are
+ * a lot of deleted slots.
+ */
+ private void expandAndRehash() {
+ final Object[] oldKeys = this.keys;
+
+ assert assigned >= resizeThreshold;
+ allocateBuffers(nextCapacity(keys.length));
+
+ /*
+ * Rehash all assigned slots from the old hash table.
+ */
+ final int mask = keys.length - 1;
+ for (int i = 0; i < oldKeys.length; i++) {
+ final Object key = oldKeys[i];
+ if (key != null) {
+ int slot = rehash(key) & mask;
+ while (keys[slot] != null) {
+ slot = (slot + 1) & mask;
+ }
+ keys[slot] = key;
+ }
+ }
+ Arrays.fill(oldKeys, null);
+ }
+
+ /**
+ * Allocate internal buffers for a given capacity.
+ *
+ * @param capacity New capacity (must be a power of two).
+ */
+ private void allocateBuffers(int capacity) {
+ this.keys = new Object[capacity];
+ this.resizeThreshold = (int) (capacity * DEFAULT_LOAD_FACTOR);
+ }
+
+ /**
+ * Return the next possible capacity, counting from the current buffers' size.
+ */
+ protected int nextCapacity(int current) {
+ assert current > 0 && Long.bitCount(current) == 1 : "Capacity must be a power of two.";
+ assert ((current << 1) > 0) : "Maximum capacity exceeded ("
+ + (0x80000000 >>> 1) + ").";
+
+ if (current < MIN_CAPACITY / 2) {
+ current = MIN_CAPACITY / 2;
+ }
+ return current << 1;
+ }
+
+ /**
+ * Round the capacity to the next allowed value.
+ */
+ protected int roundCapacity(int requestedCapacity) {
+ // Maximum positive integer that is a power of two.
+ if (requestedCapacity > (0x80000000 >>> 1)) {
+ return (0x80000000 >>> 1);
+ }
+
+ int capacity = MIN_CAPACITY;
+ while (capacity < requestedCapacity) {
+ capacity <<= 1;
+ }
+
+ return capacity;
+ }
+
+ public void clear() {
+ assigned = 0;
+ Arrays.fill(keys, null);
+ }
+
+ public int size() {
+ return assigned;
+ }
+
+ public boolean isEmpty() {
+ return size() == 0;
+ }
+
+ @Override
+ public Iterator<KType> iterator() {
+ return new Iterator<KType>() {
+ int pos = -1;
+ Object nextElement = fetchNext();
+
+ @Override
+ public boolean hasNext() {
+ return nextElement != null;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public KType next() {
+ Object r = this.nextElement;
+ if (r == null) {
+ throw new NoSuchElementException();
+ }
+ this.nextElement = fetchNext();
+ return (KType) r;
+ }
+
+ private Object fetchNext() {
+ pos++;
+ while (pos < keys.length && keys[pos] == null) {
+ pos++;
+ }
+
+ return (pos >= keys.length ? null : keys[pos]);
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+ }
+
+
+ /**
+ * Some useful constants.
+ **/
+
+ static class Constants {
+
+ private Constants() {
+ } // can't construct
+
+ /**
+ * JVM vendor info.
+ */
+ public static final String JVM_VENDOR = System.getProperty("java.vm.vendor");
+ public static final String JVM_VERSION = System.getProperty("java.vm.version");
+ public static final String JVM_NAME = System.getProperty("java.vm.name");
+
+ /**
+ * The value of <tt>System.getProperty("java.version")</tt>.
+ **/
+ public static final String JAVA_VERSION = System.getProperty("java.version");
+
+ /**
+ * The value of <tt>System.getProperty("os.name")</tt>.
+ **/
+ public static final String OS_NAME = System.getProperty("os.name");
+ /**
+ * True iff running on Linux.
+ */
+ public static final boolean LINUX = OS_NAME.startsWith("Linux");
+ /**
+ * True iff running on Windows.
+ */
+ public static final boolean WINDOWS = OS_NAME.startsWith("Windows");
+ /**
+ * True iff running on SunOS.
+ */
+ public static final boolean SUN_OS = OS_NAME.startsWith("SunOS");
+ /**
+ * True iff running on Mac OS X
+ */
+ public static final boolean MAC_OS_X = OS_NAME.startsWith("Mac OS X");
+ /**
+ * True iff running on FreeBSD
+ */
+ public static final boolean FREE_BSD = OS_NAME.startsWith("FreeBSD");
+
+ public static final String OS_ARCH = System.getProperty("os.arch");
+ public static final String OS_VERSION = System.getProperty("os.version");
+ public static final String JAVA_VENDOR = System.getProperty("java.vendor");
+
+ /**
+ * @deprecated With Lucene 4.0, we are always on Java 6
+ */
+ @Deprecated
+ public static final boolean JRE_IS_MINIMUM_JAVA6 =
+ new Boolean(true).booleanValue(); // prevent inlining in foreign class files
+
+ public static final boolean JRE_IS_MINIMUM_JAVA7;
+ public static final boolean JRE_IS_MINIMUM_JAVA8;
+
+ /**
+ * True iff running on a 64bit JVM
+ */
+ public static final boolean JRE_IS_64BIT;
+
+ static {
+ boolean is64Bit = false;
+ try {
+ final Class<?> unsafeClass = Class.forName("sun.misc.Unsafe");
+ final Field unsafeField = unsafeClass.getDeclaredField("theUnsafe");
+ unsafeField.setAccessible(true);
+ final Object unsafe = unsafeField.get(null);
+ final int addressSize = ((Number) unsafeClass.getMethod("addressSize")
+ .invoke(unsafe)).intValue();
+ //System.out.println("Address size: " + addressSize);
+ is64Bit = addressSize >= 8;
+ } catch (Exception e) {
+ final String x = System.getProperty("sun.arch.data.model");
+ if (x != null) {
+ is64Bit = x.indexOf("64") != -1;
+ } else {
+ if (OS_ARCH != null && OS_ARCH.indexOf("64") != -1) {
+ is64Bit = true;
+ } else {
+ is64Bit = false;
+ }
+ }
+ }
+ JRE_IS_64BIT = is64Bit;
+
+ // this method only exists in Java 7:
+ boolean v7 = true;
+ try {
+ Throwable.class.getMethod("getSuppressed");
+ } catch (NoSuchMethodException nsme) {
+ v7 = false;
+ }
+ JRE_IS_MINIMUM_JAVA7 = v7;
+
+ if (JRE_IS_MINIMUM_JAVA7) {
+ // this method only exists in Java 8:
+ boolean v8 = true;
+ try {
+ Collections.class.getMethod("emptySortedSet");
+ } catch (NoSuchMethodException nsme) {
+ v8 = false;
+ }
+ JRE_IS_MINIMUM_JAVA8 = v8;
+ } else {
+ JRE_IS_MINIMUM_JAVA8 = false;
+ }
+ }
+
+ }
+}
\ No newline at end of file
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java
index cff1a06..c29b345 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java
@@ -19,8 +19,9 @@
package org.apache.iotdb.db.engine.cache;
import java.io.IOException;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -31,15 +32,48 @@ import org.slf4j.LoggerFactory;
public class TsFileMetaDataCache {
private static final Logger logger = LoggerFactory.getLogger(TsFileMetaDataCache.class);
+ private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
+
+ private static final long MEMORY_THRESHOLD_IN_B = (long) (0.25 * config
+ .getAllocateMemoryForRead());
/**
- * key: The file seriesPath of tsfile.
+ * key: Tsfile path. value: TsFileMetaData
*/
- private ConcurrentHashMap<String, TsFileMetaData> cache;
- private AtomicLong cacheHintNum = new AtomicLong();
+ private LRULinkedHashMap<String, TsFileMetaData> cache;
+ private AtomicLong cacheHitNum = new AtomicLong();
private AtomicLong cacheRequestNum = new AtomicLong();
+ /**
+ * estimated size of a deviceIndexMap entry in TsFileMetaData.
+ */
+ private long deviceIndexMapEntrySize = 0;
+ /**
+ * estimated size of measurementSchema entry in TsFileMetaData.
+ */
+ private long measurementSchemaEntrySize = 0;
+ /**
+ * estimated size of version and CreateBy in TsFileMetaData.
+ */
+ private long versionAndCreatebySize = 10;
+
private TsFileMetaDataCache() {
- cache = new ConcurrentHashMap<>();
+ cache = new LRULinkedHashMap<String, TsFileMetaData>(MEMORY_THRESHOLD_IN_B, true) {
+ @Override
+ protected long calEntrySize(String key, TsFileMetaData value) {
+ if (deviceIndexMapEntrySize == 0 && value.getDeviceMap().size() > 0) {
+ deviceIndexMapEntrySize = RamUsageEstimator
+ .sizeOf(value.getDeviceMap().entrySet().iterator().next());
+ }
+ if (measurementSchemaEntrySize == 0 && value.getMeasurementSchema().size() > 0) {
+ measurementSchemaEntrySize = RamUsageEstimator
+ .sizeOf(value.getMeasurementSchema().entrySet().iterator().next());
+ }
+ long valueSize = value.getDeviceMap().size() * deviceIndexMapEntrySize
+ + measurementSchemaEntrySize * value.getMeasurementSchema().size()
+ + versionAndCreatebySize;
+ return key.length() * 2 + valueSize;
+ }
+ };
}
public static TsFileMetaDataCache getInstance() {
@@ -47,51 +81,64 @@ public class TsFileMetaDataCache {
}
/**
- * get the TsFileMetaData for the given path.
+ * get the TsFileMetaData for given path.
*
* @param path -given path
*/
public TsFileMetaData get(String path) throws IOException {
Object internPath = path.intern();
- synchronized (internPath) {
- cacheRequestNum.incrementAndGet();
- if (!cache.containsKey(path)) {
- // read value from tsfile
- TsFileMetaData fileMetaData = TsFileMetadataUtils.getTsFileMetaData(path);
- cache.put(path, fileMetaData);
- if (logger.isDebugEnabled()) {
- logger.debug("Cache didn't hint: the number of requests for cache is {}",
- cacheRequestNum.get());
- }
- return cache.get(path);
- } else {
- cacheHintNum.incrementAndGet();
+ cacheRequestNum.incrementAndGet();
+ synchronized (cache) {
+ if (cache.containsKey(path)) {
+ cacheHitNum.incrementAndGet();
if (logger.isDebugEnabled()) {
logger.debug(
- "Cache hint: the number of requests for cache is {}, the number of hints for cache "
- + "is {}",
- cacheRequestNum.get(), cacheHintNum.get());
+ "Cache hit: the number of requests for cache is {}, "
+ + "the number of hints for cache is {}",
+ cacheRequestNum.get(), cacheHitNum.get());
}
return cache.get(path);
}
}
+ synchronized (internPath) {
+ synchronized (cache) {
+ if (cache.containsKey(path)) {
+ cacheHitNum.incrementAndGet();
+ return cache.get(path);
+ }
+ }
+ if (logger.isDebugEnabled()) {
+ logger.debug("Cache didn't hit: the number of requests for cache is {}",
+ cacheRequestNum.get());
+ }
+ TsFileMetaData fileMetaData = TsFileMetadataUtils.getTsFileMetaData(path);
+ synchronized (cache) {
+ cache.put(path, fileMetaData);
+ return fileMetaData;
+ }
+ }
}
public void remove(String path) {
- cache.remove(path);
+ synchronized (cache) {
+ cache.remove(path);
+ }
}
public void clear() {
- cache.clear();
+ synchronized (cache) {
+ cache.clear();
+ }
}
- /*
+ /**
* Singleton pattern
*/
private static class TsFileMetaDataCacheHolder {
- private TsFileMetaDataCacheHolder() {}
+ private TsFileMetaDataCacheHolder() {
+ }
private static final TsFileMetaDataCache INSTANCE = new TsFileMetaDataCache();
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java
index ea63b37..8a4424b 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java
@@ -19,10 +19,19 @@
package org.apache.iotdb.db.engine.cache;
import java.io.IOException;
-import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import org.apache.iotdb.db.query.control.FileReaderManager;
+import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetaData;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadata;
+import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadataIndex;
import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
+import org.apache.iotdb.tsfile.read.common.Path;
/**
* This class is used to read metadata(<code>TsFileMetaData</code> and
@@ -30,7 +39,7 @@ import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
*/
public class TsFileMetadataUtils {
- private TsFileMetadataUtils(){
+ private TsFileMetadataUtils() {
}
@@ -41,32 +50,52 @@ public class TsFileMetadataUtils {
* @return -meta data
*/
public static TsFileMetaData getTsFileMetaData(String filePath) throws IOException {
- try (TsFileSequenceReader reader = new TsFileSequenceReader(filePath)) {
- return reader.readFileMetadata();
- }
+ TsFileSequenceReader reader = FileReaderManager.getInstance().get(filePath, true);
+ return reader.readFileMetadata();
}
/**
* get row group block meta data.
*
* @param filePath -file path
- * @param deviceId -device id
- * @param fileMetaData -ts file meta data
+ * @param seriesPath -series path
+ * @param fileMetaData -tsfile meta data
* @return -device meta data
*/
- public static TsDeviceMetadata getTsRowGroupBlockMetaData(String filePath, String deviceId,
+ public static TsDeviceMetadata getTsDeviceMetaData(String filePath, Path seriesPath,
TsFileMetaData fileMetaData) throws IOException {
- if (!fileMetaData.getDeviceMap().containsKey(deviceId)) {
+ if (!fileMetaData.getMeasurementSchema().containsKey(seriesPath.getMeasurement())) {
return null;
} else {
- try (TsFileSequenceReader reader = new TsFileSequenceReader(filePath)) {
- long offset = fileMetaData.getDeviceMap().get(deviceId).getOffset();
- int size = fileMetaData.getDeviceMap().get(deviceId).getLen();
- ByteBuffer data = ByteBuffer.allocate(size);
- reader.readRaw(offset, size, data);
- data.flip();
- return TsDeviceMetadata.deserializeFrom(data);
+ // get the index information of TsDeviceMetadata
+ TsDeviceMetadataIndex index = fileMetaData.getDeviceMetadataIndex(seriesPath.getDevice());
+ TsFileSequenceReader tsFileReader = FileReaderManager.getInstance().get(filePath, true);
+ // read TsDeviceMetadata from file
+ return tsFileReader.readTsDeviceMetaData(index);
+ }
+ }
+
+ /**
+ * get ChunkMetaData List of sensors in sensorSet included in all ChunkGroups of this device. If
+ * sensorSet is empty, then return metadata of all sensor included in this device.
+ */
+ public static Map<Path, List<ChunkMetaData>> getChunkMetaDataList(
+ Set<String> sensorSet, TsDeviceMetadata tsDeviceMetadata) {
+ Map<Path, List<ChunkMetaData>> pathToChunkMetaDataList = new ConcurrentHashMap<>();
+ for (ChunkGroupMetaData chunkGroupMetaData : tsDeviceMetadata.getChunkGroupMetaDataList()) {
+ List<ChunkMetaData> chunkMetaDataListInOneChunkGroup = chunkGroupMetaData
+ .getChunkMetaDataList();
+ String deviceId = chunkGroupMetaData.getDeviceID();
+ for (ChunkMetaData chunkMetaData : chunkMetaDataListInOneChunkGroup) {
+ if (sensorSet.isEmpty() || sensorSet.contains(chunkMetaData.getMeasurementUid())) {
+ Path path = new Path(deviceId, chunkMetaData.getMeasurementUid());
+ pathToChunkMetaDataList.putIfAbsent(path, new ArrayList<>());
+ chunkMetaData.setVersion(chunkGroupMetaData.getVersion());
+ pathToChunkMetaDataList.get(path).add(chunkMetaData);
+ }
}
}
+ return pathToChunkMetaDataList;
}
+
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index f04469e..0e93c94 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -26,16 +26,19 @@ import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Set;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.commons.io.FileUtils;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.directories.DirectoryManager;
-import org.apache.iotdb.db.engine.StorageEngine;
import org.apache.iotdb.db.engine.modification.Deletion;
import org.apache.iotdb.db.engine.modification.ModificationFile;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
@@ -147,6 +150,12 @@ public class StorageGroupProcessor {
*/
private ModificationFile mergingModification;
+ /**
+ * This linked list records the access order of sensors used by query.
+ */
+ private LinkedList<String> lruForSensorUsedInQuery = new LinkedList<>();
+ private static final int MAX_CACHE_SENSORS = 5000;
+
public StorageGroupProcessor(String systemInfoDir, String storageGroupName)
throws ProcessorException {
@@ -160,7 +169,7 @@ public class StorageGroupProcessor {
if (storageGroupSysDir.mkdirs()) {
logger.info("Storage Group system Directory {} doesn't exist, create it",
storageGroupSysDir.getPath());
- } else if(!storageGroupSysDir.exists()) {
+ } else if (!storageGroupSysDir.exists()) {
logger.error("craete Storage Group system Directory {} failed",
storageGroupSysDir.getPath());
}
@@ -304,7 +313,9 @@ public class StorageGroupProcessor {
tsFileProcessor = workUnSequenceTsFileProcessor;
}
} catch (DiskSpaceInsufficientException e) {
- logger.error("disk space is insufficient when creating TsFile processor, change system mode to read-only", e);
+ logger.error(
+ "disk space is insufficient when creating TsFile processor, change system mode to read-only",
+ e);
IoTDBDescriptor.getInstance().getConfig().setReadOnly(true);
return false;
}
@@ -447,6 +458,12 @@ public class StorageGroupProcessor {
// TODO need a read lock, please consider the concurrency with flush manager threads.
public QueryDataSource query(String deviceId, String measurementId, QueryContext context) {
insertLock.readLock().lock();
+ synchronized (lruForSensorUsedInQuery) {
+ if (lruForSensorUsedInQuery.size() >= MAX_CACHE_SENSORS) {
+ lruForSensorUsedInQuery.removeFirst();
+ }
+ lruForSensorUsedInQuery.add(measurementId);
+ }
try {
List<TsFileResource> seqResources = getFileReSourceListForQuery(sequenceFileList,
deviceId, measurementId, context);
@@ -458,6 +475,27 @@ public class StorageGroupProcessor {
}
}
+ /**
+ * returns the top k% measurements which are recently used in queries.
+ */
+ public Set calTopKMeasurement(String sensorId, double k) {
+ int num = (int) (lruForSensorUsedInQuery.size() * k);
+ Set<String> sensorSet = new HashSet<>(num + 1);
+ synchronized (lruForSensorUsedInQuery) {
+ Iterator<String> iterator = lruForSensorUsedInQuery.descendingIterator();
+ while (iterator.hasNext() && sensorSet.size() < num) {
+ String sensor = iterator.next();
+ if (sensorSet.contains(sensor)) {
+ iterator.remove();
+ } else {
+ sensorSet.add(sensor);
+ }
+ }
+ }
+ sensorSet.add(sensorId);
+ return sensorSet;
+ }
+
private void writeLock() {
insertLock.writeLock().lock();
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/control/JobFileManager.java b/server/src/main/java/org/apache/iotdb/db/query/control/JobFileManager.java
index 315cc48..f102e44 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/control/JobFileManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/control/JobFileManager.java
@@ -76,11 +76,11 @@ public class JobFileManager {
*/
void removeUsedFilesForGivenJob(long jobId) {
for (String filePath : sealedFilePathsMap.get(jobId)) {
- FileReaderManager.getInstance().decreaseFileReaderReference(filePath, false);
+ FileReaderManager.getInstance().decreaseFileReaderReference(filePath, true);
}
sealedFilePathsMap.remove(jobId);
for (String filePath : unsealedFilePathsMap.get(jobId)) {
- FileReaderManager.getInstance().decreaseFileReaderReference(filePath, true);
+ FileReaderManager.getInstance().decreaseFileReaderReference(filePath, false);
}
unsealedFilePathsMap.remove(jobId);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/SeqResourceIterateReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/SeqResourceIterateReader.java
index 8c08649..9ba9128 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/SeqResourceIterateReader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/SeqResourceIterateReader.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.db.query.reader.resourceRelated;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
+import org.apache.iotdb.db.engine.cache.DeviceMetaDataCache;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.query.context.QueryContext;
@@ -35,7 +36,6 @@ import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.controller.ChunkLoader;
import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl;
-import org.apache.iotdb.tsfile.read.controller.MetadataQuerierByFileImpl;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.read.reader.series.FileSeriesReader;
import org.apache.iotdb.tsfile.read.reader.series.FileSeriesReaderWithFilter;
@@ -158,10 +158,9 @@ public class SeqResourceIterateReader extends IterateReader {
private IAggregateReader initSealedTsFileReader(TsFileResource sealedTsFile, Filter filter,
QueryContext context) throws IOException {
// prepare metaDataList
- TsFileSequenceReader tsFileReader = FileReaderManager.getInstance()
- .get(sealedTsFile.getFile().getPath(), true);
- MetadataQuerierByFileImpl metadataQuerier = new MetadataQuerierByFileImpl(tsFileReader);
- List<ChunkMetaData> metaDataList = metadataQuerier.getChunkMetaDataList(seriesPath);
+ List<ChunkMetaData> metaDataList = DeviceMetaDataCache.getInstance()
+ .get(sealedTsFile.getFile().getPath(), seriesPath);
+
List<Modification> pathModifications = context.getPathModifications(sealedTsFile.getModFile(),
seriesPath.getFullPath());
if (!pathModifications.isEmpty()) {
@@ -172,6 +171,8 @@ public class SeqResourceIterateReader extends IterateReader {
Collections.reverse(metaDataList);
}
// prepare chunkLoader
+ TsFileSequenceReader tsFileReader = FileReaderManager.getInstance()
+ .get(sealedTsFile.getFile().getPath(), true);
ChunkLoader chunkLoader = new ChunkLoaderImpl(tsFileReader);
// init fileSeriesReader
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/SeqResourceReaderByTimestamp.java b/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/SeqResourceReaderByTimestamp.java
index f8a5984..07b2313 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/SeqResourceReaderByTimestamp.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/SeqResourceReaderByTimestamp.java
@@ -20,6 +20,7 @@ package org.apache.iotdb.db.query.reader.resourceRelated;
import java.io.IOException;
import java.util.List;
+import org.apache.iotdb.db.engine.cache.DeviceMetaDataCache;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.query.context.QueryContext;
@@ -33,7 +34,6 @@ import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.controller.ChunkLoader;
import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl;
-import org.apache.iotdb.tsfile.read.controller.MetadataQuerierByFileImpl;
import org.apache.iotdb.tsfile.read.reader.series.FileSeriesReaderByTimestamp;
/**
@@ -171,16 +171,17 @@ public class SeqResourceReaderByTimestamp implements IReaderByTimestamp {
private IReaderByTimestamp initSealedTsFileReaderByTimestamp(TsFileResource sealedTsFile,
QueryContext context) throws IOException {
// prepare metaDataList
- TsFileSequenceReader tsFileReader = FileReaderManager.getInstance()
- .get(sealedTsFile.getFile().getPath(), true);
- MetadataQuerierByFileImpl metadataQuerier = new MetadataQuerierByFileImpl(tsFileReader);
- List<ChunkMetaData> metaDataList = metadataQuerier.getChunkMetaDataList(seriesPath);
+ List<ChunkMetaData> metaDataList = DeviceMetaDataCache.getInstance()
+ .get(sealedTsFile.getFile().getPath(), seriesPath);
+
List<Modification> pathModifications = context.getPathModifications(sealedTsFile.getModFile(),
seriesPath.getFullPath());
if (!pathModifications.isEmpty()) {
QueryUtils.modifyChunkMetaData(metaDataList, pathModifications);
}
// prepare chunkLoader
+ TsFileSequenceReader tsFileReader = FileReaderManager.getInstance()
+ .get(sealedTsFile.getFile().getPath(), true);
ChunkLoader chunkLoader = new ChunkLoaderImpl(tsFileReader);
return new FileSeriesReaderByTimestampAdapter(
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/UnseqResourceMergeReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/UnseqResourceMergeReader.java
index 9a857ab..0d3eb66 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/UnseqResourceMergeReader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/UnseqResourceMergeReader.java
@@ -20,6 +20,7 @@ package org.apache.iotdb.db.query.reader.resourceRelated;
import java.io.IOException;
import java.util.List;
+import org.apache.iotdb.db.engine.cache.DeviceMetaDataCache;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.query.context.QueryContext;
@@ -34,7 +35,6 @@ import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Chunk;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl;
-import org.apache.iotdb.tsfile.read.controller.MetadataQuerierByFileImpl;
import org.apache.iotdb.tsfile.read.filter.DigestForFilter;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReader;
@@ -61,7 +61,6 @@ public class UnseqResourceMergeReader extends PriorityMergeReader {
int priorityValue = 1;
for (TsFileResource tsFileResource : unseqResources) {
- TsFileSequenceReader tsFileReader;
// prepare metaDataList
List<ChunkMetaData> metaDataList;
@@ -69,10 +68,8 @@ public class UnseqResourceMergeReader extends PriorityMergeReader {
if (isTsFileNotSatisfied(tsFileResource, filter)) {
continue;
}
- tsFileReader = FileReaderManager.getInstance()
- .get(tsFileResource.getFile().getPath(), true);
- MetadataQuerierByFileImpl metadataQuerier = new MetadataQuerierByFileImpl(tsFileReader);
- metaDataList = metadataQuerier.getChunkMetaDataList(seriesPath);
+ metaDataList = DeviceMetaDataCache.getInstance()
+ .get(tsFileResource.getFile().getPath(), seriesPath);
List<Modification> pathModifications = context
.getPathModifications(tsFileResource.getModFile(), seriesPath.getFullPath());
if (!pathModifications.isEmpty()) {
@@ -84,13 +81,17 @@ public class UnseqResourceMergeReader extends PriorityMergeReader {
continue;
}
}
- tsFileReader = FileReaderManager.getInstance()
- .get(tsFileResource.getFile().getPath(), false);
metaDataList = tsFileResource.getChunkMetaDatas();
}
- // create and add ChunkReader with priority
- ChunkLoaderImpl chunkLoader = new ChunkLoaderImpl(tsFileReader);
+ ChunkLoaderImpl chunkLoader = null;
+ if (!metaDataList.isEmpty()) {
+ // create and add ChunkReader with priority
+ TsFileSequenceReader tsFileReader = FileReaderManager.getInstance()
+ .get(tsFileResource.getFile().getPath(), tsFileResource.isClosed());
+ chunkLoader = new ChunkLoaderImpl(tsFileReader);
+ }
+
for (ChunkMetaData chunkMetaData : metaDataList) {
if (filter != null) {
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/UnseqResourceReaderByTimestamp.java b/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/UnseqResourceReaderByTimestamp.java
index d2a5047..6a6123d 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/UnseqResourceReaderByTimestamp.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/UnseqResourceReaderByTimestamp.java
@@ -20,6 +20,7 @@ package org.apache.iotdb.db.query.reader.resourceRelated;
import java.io.IOException;
import java.util.List;
+import org.apache.iotdb.db.engine.cache.DeviceMetaDataCache;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.query.context.QueryContext;
@@ -33,7 +34,6 @@ import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Chunk;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl;
-import org.apache.iotdb.tsfile.read.controller.MetadataQuerierByFileImpl;
import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReaderByTimestamp;
/**
@@ -52,14 +52,12 @@ public class UnseqResourceReaderByTimestamp extends PriorityMergeReaderByTimesta
int priorityValue = 1;
for (TsFileResource tsFileResource : unseqResources) {
- TsFileSequenceReader tsFileReader = FileReaderManager.getInstance()
- .get(tsFileResource.getFile().getPath(), tsFileResource.isClosed());
// prepare metaDataList
List<ChunkMetaData> metaDataList;
if (tsFileResource.isClosed()) {
- MetadataQuerierByFileImpl metadataQuerier = new MetadataQuerierByFileImpl(tsFileReader);
- metaDataList = metadataQuerier.getChunkMetaDataList(seriesPath);
+ metaDataList = DeviceMetaDataCache.getInstance()
+ .get(tsFileResource.getFile().getPath(), seriesPath);
List<Modification> pathModifications = context
.getPathModifications(tsFileResource.getModFile(), seriesPath.getFullPath());
if (!pathModifications.isEmpty()) {
@@ -69,8 +67,13 @@ public class UnseqResourceReaderByTimestamp extends PriorityMergeReaderByTimesta
metaDataList = tsFileResource.getChunkMetaDatas();
}
- // create and add ChunkReaderByTimestamp with priority
- ChunkLoaderImpl chunkLoader = new ChunkLoaderImpl(tsFileReader);
+ ChunkLoaderImpl chunkLoader = null;
+ if (!metaDataList.isEmpty()) {
+ // create and add ChunkReader with priority
+ TsFileSequenceReader tsFileReader = FileReaderManager.getInstance()
+ .get(tsFileResource.getFile().getPath(), tsFileResource.isClosed());
+ chunkLoader = new ChunkLoaderImpl(tsFileReader);
+ }
for (ChunkMetaData chunkMetaData : metaDataList) {
Chunk chunk = chunkLoader.getChunk(chunkMetaData);