You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2020/05/13 14:51:47 UTC
[incubator-iotdb] branch TyLRU created (now 03a6b58)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a change to branch TyLRU
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.
at 03a6b58 change LRU
This branch includes the following new commits:
new 03a6b58 change LRU
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-iotdb] 01/01: change LRU
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch TyLRU
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 03a6b58566eac33b1a5a12fa1e1fe8ffdbc03775
Author: JackieTien97 <Ja...@foxmail.com>
AuthorDate: Wed May 13 22:51:18 2020 +0800
change LRU
---
.../iotdb/db/engine/cache/AccountableString.java | 27 +++++++++
.../apache/iotdb/db/engine/cache/ChunkCache.java | 12 ++--
.../iotdb/db/engine/cache/ChunkMetadataCache.java | 35 ++++++-----
.../iotdb/db/engine/cache/LRULinkedHashMap.java | 69 +++++++++++++++++-----
.../db/engine/cache/TimeSeriesMetadataCache.java | 27 +++++++--
.../iotdb/tsfile/common/cache/Accountable.java | 8 +++
.../iotdb/tsfile/file/metadata/ChunkMetadata.java | 23 ++++++--
.../tsfile/file/metadata/TimeseriesMetadata.java | 15 ++++-
.../org/apache/iotdb/tsfile/read/common/Chunk.java | 15 ++++-
9 files changed, 182 insertions(+), 49 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/AccountableString.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/AccountableString.java
new file mode 100644
index 0000000..7551552
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/AccountableString.java
@@ -0,0 +1,27 @@
+package org.apache.iotdb.db.engine.cache;
+
+import org.apache.iotdb.tsfile.common.cache.Accountable;
+
+public class AccountableString implements Accountable {
+
+ private String string;
+ private long RAMSize;
+
+ public AccountableString(String string) {
+ this.string = string;
+ }
+
+ public String getString() {
+ return string;
+ }
+
+ @Override
+ public void setRAMSize(long size) {
+ this.RAMSize = size;
+ }
+
+ @Override
+ public long getRAMSize() {
+ return RAMSize;
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
index b4eefb3..325131d 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
@@ -56,22 +56,24 @@ public class ChunkCache {
if (cacheEnable) {
logger.info("ChunkCache size = " + MEMORY_THRESHOLD_IN_CHUNK_CACHE);
}
- lruCache = new LRULinkedHashMap<ChunkMetadata, Chunk>(MEMORY_THRESHOLD_IN_CHUNK_CACHE, true) {
+ lruCache = new LRULinkedHashMap<ChunkMetadata, Chunk>(MEMORY_THRESHOLD_IN_CHUNK_CACHE) {
@Override
protected long calEntrySize(ChunkMetadata key, Chunk value) {
+ long currentSize;
if (count < 10) {
- long currentSize = RamUsageEstimator.shallowSizeOf(key) + RamUsageEstimator.sizeOf(value);
+ currentSize = RamUsageEstimator.shallowSizeOf(key) + RamUsageEstimator.sizeOf(value);
averageSize = ((averageSize * count) + currentSize) / (++count);
- return currentSize;
} else if (count < 100000) {
count++;
- return averageSize;
+ currentSize = averageSize;
} else {
averageSize = RamUsageEstimator.shallowSizeOf(key) + RamUsageEstimator.sizeOf(value);
count = 1;
- return averageSize;
+ currentSize = averageSize;
}
+ key.setRAMSize(currentSize);
+ return currentSize;
}
};
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
index 82adca5..3740fe3 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
@@ -53,7 +53,7 @@ public class ChunkMetadataCache {
* <p>
* value: chunkMetaData list of one timeseries in the file.
*/
- private final LRULinkedHashMap<String, List<ChunkMetadata>> lruCache;
+ private final LRULinkedHashMap<AccountableString, List<ChunkMetadata>> lruCache;
private final ReadWriteLock lock = new ReentrantReadWriteLock();
@@ -65,27 +65,30 @@ public class ChunkMetadataCache {
if (cacheEnable) {
logger.info("ChunkMetadataCache size = " + memoryThreshold);
}
- lruCache = new LRULinkedHashMap<String, List<ChunkMetadata>>(memoryThreshold, true) {
+ lruCache = new LRULinkedHashMap<AccountableString, List<ChunkMetadata>>(memoryThreshold) {
@Override
- protected long calEntrySize(String key, List<ChunkMetadata> value) {
+ protected long calEntrySize(AccountableString key, List<ChunkMetadata> value) {
if (value.isEmpty()) {
- return key.getBytes().length + averageSize * value.size();
+ return key.getString().getBytes().length;
}
+ long entrySize;
if (count < 10) {
- long currentSize = RamUsageEstimator.shallowSizeOf(value.get(0)) + RamUsageEstimator
- .shallowSizeOf(value.get(0).getStatistics());
+ long currentSize = RamUsageEstimator.shallowSizeOf(value.get(0))
+ + RamUsageEstimator.shallowSizeOf(value.get(0).getStatistics());
averageSize = ((averageSize * count) + currentSize) / (++count);
IoTDBConfigDynamicAdapter.setChunkMetadataSizeInByte(averageSize);
- return key.getBytes().length + currentSize * value.size();
+ entrySize = RamUsageEstimator.sizeOf(key) + currentSize * value.size();
} else if (count < 100000) {
count++;
- return key.getBytes().length + averageSize * value.size();
+ entrySize = RamUsageEstimator.sizeOf(key) + averageSize * value.size();
} else {
averageSize = RamUsageEstimator.shallowSizeOf(value.get(0)) + RamUsageEstimator
.shallowSizeOf(value.get(0).getStatistics());
count = 1;
- return key.getBytes().length + averageSize * value.size();
+ entrySize = RamUsageEstimator.sizeOf(key) + averageSize * value.size();
}
+ key.setRAMSize(entrySize);
+ return entrySize;
}
};
}
@@ -115,8 +118,8 @@ public class ChunkMetadataCache {
return tsFileReader.getChunkMetadataList(seriesPath);
}
- String key = (filePath + IoTDBConstant.PATH_SEPARATOR
- + seriesPath.getDevice() + seriesPath.getMeasurement()).intern();
+ AccountableString key = new AccountableString((filePath + IoTDBConstant.PATH_SEPARATOR
+ + seriesPath.getDevice() + seriesPath.getMeasurement()).intern());
cacheRequestNum.incrementAndGet();
@@ -192,15 +195,19 @@ public class ChunkMetadataCache {
* clear LRUCache.
*/
public void clear() {
- synchronized (lruCache) {
+ lock.writeLock().lock();
+ if (lruCache != null) {
lruCache.clear();
}
+ lock.writeLock().unlock();
}
public void remove(TsFileResource resource) {
- synchronized (lruCache) {
- lruCache.entrySet().removeIf(e -> e.getKey().startsWith(resource.getPath()));
+ lock.writeLock().lock();
+ if (resource != null) {
+ lruCache.entrySet().removeIf(e -> e.getKey().getString().startsWith(resource.getPath()));
}
+ lock.writeLock().unlock();
}
/**
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/LRULinkedHashMap.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/LRULinkedHashMap.java
index 8172588..5c38fb7 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/LRULinkedHashMap.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/LRULinkedHashMap.java
@@ -19,48 +19,81 @@
package org.apache.iotdb.db.engine.cache;
+import java.util.Iterator;
import java.util.LinkedHashMap;
-import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import org.apache.iotdb.tsfile.common.cache.Accountable;
/**
* This class is an LRU cache. <b>Note: It's not thread safe.</b>
*/
-public abstract class LRULinkedHashMap<K, V> extends LinkedHashMap<K, V> {
+public abstract class LRULinkedHashMap<K extends Accountable, V> {
- private static final long serialVersionUID = 1290160928914532649L;
private static final float LOAD_FACTOR_MAP = 0.75f;
private static final int INITIAL_CAPACITY = 128;
+ private static final float RETAIN_PERCENT = 0.9f;
+
+ private final LinkedHashMap<K, V> linkedHashMap;
+
/**
* maximum memory threshold.
*/
- private long maxMemory;
+ private final long maxMemory;
/**
* current used memory.
*/
private long usedMemory;
+ /**
+ *
+ */
+ private final long retainMemory;
+
protected int count = 0;
protected long averageSize = 0;
- public LRULinkedHashMap(long maxMemory, boolean isLru) {
- super(INITIAL_CAPACITY, LOAD_FACTOR_MAP, isLru);
+ public LRULinkedHashMap(long maxMemory) {
+ this.linkedHashMap = new LinkedHashMap<>(INITIAL_CAPACITY, LOAD_FACTOR_MAP);
this.maxMemory = maxMemory;
+ this.retainMemory = (long) (maxMemory * RETAIN_PERCENT);
}
- @Override
- protected boolean removeEldestEntry(Map.Entry<K, V> eldest) {
+ public V put(K key, V value) {
+ long size = calEntrySize(key, value);
+ key.setRAMSize(size);
+ usedMemory += size;
+ V v = linkedHashMap.put(key, value);
if (usedMemory > maxMemory) {
- usedMemory -= calEntrySize(eldest.getKey(), eldest.getValue());
- return true;
- } else {
- return false;
+ Iterator<Entry<K, V>> iterator = linkedHashMap.entrySet().iterator();
+ while (usedMemory > retainMemory && iterator.hasNext()) {
+ Entry<K, V> entry = iterator.next();
+ usedMemory -= entry.getKey().getRAMSize();
+ iterator.remove();
+ }
}
+ return v;
}
- @Override
- public V put(K key, V value) {
- usedMemory += calEntrySize(key, value);
- return super.put(key, value);
+ public V get(K key) {
+ return linkedHashMap.get(key);
+ }
+
+ public boolean containsKey(K key) {
+ return linkedHashMap.containsKey(key);
+ }
+
+ public void clear() {
+ linkedHashMap.clear();
+ usedMemory = 0;
+ }
+
+ public V remove(K key) {
+ V v = linkedHashMap.remove(key);
+ if (v != null && key != null) {
+ usedMemory -= key.getRAMSize();
+ }
+ return v;
}
/**
@@ -87,6 +120,10 @@ public abstract class LRULinkedHashMap<K, V> extends LinkedHashMap<K, V> {
return averageSize;
}
+ public Set<Entry<K, V>> entrySet() {
+ return linkedHashMap.entrySet();
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
index 8a52a51..d1189d3 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
@@ -30,6 +30,7 @@ import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.query.control.FileReaderManager;
+import org.apache.iotdb.tsfile.common.cache.Accountable;
import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Path;
@@ -63,22 +64,24 @@ public class TimeSeriesMetadataCache {
.info("TimeseriesMetadataCache size = " + MEMORY_THRESHOLD_IN_TIME_SERIES_METADATA_CACHE);
}
lruCache = new LRULinkedHashMap<TimeSeriesMetadataCacheKey, TimeseriesMetadata>(
- MEMORY_THRESHOLD_IN_TIME_SERIES_METADATA_CACHE, true) {
+ MEMORY_THRESHOLD_IN_TIME_SERIES_METADATA_CACHE) {
@Override
protected long calEntrySize(TimeSeriesMetadataCacheKey key, TimeseriesMetadata value) {
+ long currentSize;
if (count < 10) {
- long currentSize = RamUsageEstimator.shallowSizeOf(key) + RamUsageEstimator.sizeOf(value);
+ currentSize = RamUsageEstimator.shallowSizeOf(key) + RamUsageEstimator.sizeOf(value);
averageSize = ((averageSize * count) + currentSize) / (++count);
- return currentSize;
} else if (count < 100000) {
count++;
- return averageSize;
+ currentSize = averageSize;
} else {
averageSize = RamUsageEstimator.shallowSizeOf(key) + RamUsageEstimator.sizeOf(value);
count = 1;
- return averageSize;
+ currentSize = averageSize;
}
+ key.setRAMSize(currentSize);
+ return currentSize;
}
};
}
@@ -198,12 +201,15 @@ public class TimeSeriesMetadataCache {
lock.writeLock().unlock();
}
- public static class TimeSeriesMetadataCacheKey {
+ public static class TimeSeriesMetadataCacheKey implements Accountable {
private String filePath;
private String device;
private String measurement;
+ private long RAMSize;
+
+
public TimeSeriesMetadataCacheKey(String filePath, String device, String measurement) {
this.filePath = filePath;
this.device = device;
@@ -228,6 +234,15 @@ public class TimeSeriesMetadataCache {
public int hashCode() {
return Objects.hash(filePath, device, measurement);
}
+
+ public void setRAMSize(long size) {
+ this.RAMSize = size;
+ }
+
+ @Override
+ public long getRAMSize() {
+ return 0;
+ }
}
/**
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/cache/Accountable.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/cache/Accountable.java
new file mode 100644
index 0000000..067853d
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/cache/Accountable.java
@@ -0,0 +1,8 @@
+package org.apache.iotdb.tsfile.common.cache;
+
+public interface Accountable {
+
+ void setRAMSize(long size);
+
+ long getRAMSize();
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
index 12d96c7..a45ee7c 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
@@ -18,20 +18,20 @@
*/
package org.apache.iotdb.tsfile.file.metadata;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
-import org.apache.iotdb.tsfile.read.controller.IChunkLoader;
-import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
-
import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.util.Objects;
+import org.apache.iotdb.tsfile.common.cache.Accountable;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
+import org.apache.iotdb.tsfile.read.controller.IChunkLoader;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
/**
* Metadata of one chunk.
*/
-public class ChunkMetadata {
+public class ChunkMetadata implements Accountable {
private String measurementUid;
@@ -62,6 +62,8 @@ public class ChunkMetadata {
private Statistics statistics;
+ private long RAMSize;
+
private ChunkMetadata() {
}
@@ -209,4 +211,13 @@ public class ChunkMetadata {
public void setModified(boolean modified) {
this.modified = modified;
}
+
+ public void setRAMSize(long size) {
+ this.RAMSize = size;
+ }
+
+ @Override
+ public long getRAMSize() {
+ return 0;
+ }
}
\ No newline at end of file
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
index 0522ed7..3696851 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
@@ -23,12 +23,13 @@ import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.util.List;
+import org.apache.iotdb.tsfile.common.cache.Accountable;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.controller.IChunkMetadataLoader;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
-public class TimeseriesMetadata {
+public class TimeseriesMetadata implements Accountable {
private long startOffsetOfChunkMetaDataList;
private int chunkMetaDataListDataSize;
@@ -43,6 +44,9 @@ public class TimeseriesMetadata {
private IChunkMetadataLoader chunkMetadataLoader;
+ private long RAMSize;
+
+
public static TimeseriesMetadata deserializeFrom(ByteBuffer buffer) {
TimeseriesMetadata timeseriesMetaData = new TimeseriesMetadata();
timeseriesMetaData.setMeasurementId(ReadWriteIOUtils.readString(buffer));
@@ -125,4 +129,13 @@ public class TimeseriesMetadata {
public void setModified(boolean modified) {
this.modified = modified;
}
+
+ public void setRAMSize(long size) {
+ this.RAMSize = size;
+ }
+
+ @Override
+ public long getRAMSize() {
+ return 0;
+ }
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/Chunk.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/Chunk.java
index 5120d08..f53a051 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/Chunk.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/Chunk.java
@@ -20,13 +20,14 @@ package org.apache.iotdb.tsfile.read.common;
import java.nio.ByteBuffer;
+import org.apache.iotdb.tsfile.common.cache.Accountable;
import org.apache.iotdb.tsfile.encoding.common.EndianType;
import org.apache.iotdb.tsfile.file.header.ChunkHeader;
/**
* used in query.
*/
-public class Chunk {
+public class Chunk implements Accountable {
private ChunkHeader chunkHeader;
private ByteBuffer chunkData;
@@ -36,6 +37,8 @@ public class Chunk {
private long deletedAt;
private EndianType endianType;
+ private long RAMSize;
+
public Chunk(ChunkHeader header, ByteBuffer buffer, long deletedAt, EndianType endianType) {
this.chunkHeader = header;
this.chunkData = buffer;
@@ -62,4 +65,14 @@ public class Chunk {
public void setDeletedAt(long deletedAt) {
this.deletedAt = deletedAt;
}
+
+ @Override
+ public void setRAMSize(long size) {
+ this.RAMSize = size;
+ }
+
+ @Override
+ public long getRAMSize() {
+ return RAMSize;
+ }
}