You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2021/12/07 06:46:17 UTC
[iotdb] branch master updated: memory leak fix: replace RandomDeleteCache with LoadingCache as its size can't limit in … (#4526)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 88f9a6b memory leak fix: replace RandomDeleteCache with LoadingCache as its size can't limit in … (#4526)
88f9a6b is described below
commit 88f9a6b53643fd139aaf91c0393e2d4cc334b5b8
Author: Jianyun Cheng <ch...@360.cn>
AuthorDate: Tue Dec 7 14:45:35 2021 +0800
memory leak fix: replace RandomDeleteCache with LoadingCache as its size can't limit in … (#4526)
---
.../org/apache/iotdb/db/metadata/MManager.java | 113 ++++++++-------------
.../apache/iotdb/db/utils/RandomDeleteCache.java | 76 --------------
2 files changed, 43 insertions(+), 146 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
index 433bd31..dde8efd 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
@@ -24,19 +24,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.StorageEngine;
import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
import org.apache.iotdb.db.engine.trigger.executor.TriggerEngine;
-import org.apache.iotdb.db.exception.metadata.AliasAlreadyExistException;
-import org.apache.iotdb.db.exception.metadata.DataTypeMismatchException;
-import org.apache.iotdb.db.exception.metadata.DeleteFailedException;
-import org.apache.iotdb.db.exception.metadata.DifferentTemplateException;
-import org.apache.iotdb.db.exception.metadata.MNodeTypeMismatchException;
-import org.apache.iotdb.db.exception.metadata.MetadataException;
-import org.apache.iotdb.db.exception.metadata.NoTemplateOnMNodeException;
-import org.apache.iotdb.db.exception.metadata.PathAlreadyExistException;
-import org.apache.iotdb.db.exception.metadata.PathNotExistException;
-import org.apache.iotdb.db.exception.metadata.StorageGroupAlreadySetException;
-import org.apache.iotdb.db.exception.metadata.StorageGroupNotSetException;
-import org.apache.iotdb.db.exception.metadata.TemplateIsInUseException;
-import org.apache.iotdb.db.exception.metadata.UndefinedTemplateException;
+import org.apache.iotdb.db.exception.metadata.*;
import org.apache.iotdb.db.metadata.lastCache.LastCacheManager;
import org.apache.iotdb.db.metadata.logfile.MLogReader;
import org.apache.iotdb.db.metadata.logfile.MLogWriter;
@@ -57,36 +45,16 @@ import org.apache.iotdb.db.qp.physical.PhysicalPlan;
import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan;
import org.apache.iotdb.db.qp.physical.crud.InsertTabletPlan;
-import org.apache.iotdb.db.qp.physical.sys.ActivateTemplatePlan;
-import org.apache.iotdb.db.qp.physical.sys.AppendTemplatePlan;
-import org.apache.iotdb.db.qp.physical.sys.AutoCreateDeviceMNodePlan;
-import org.apache.iotdb.db.qp.physical.sys.ChangeAliasPlan;
-import org.apache.iotdb.db.qp.physical.sys.ChangeTagOffsetPlan;
-import org.apache.iotdb.db.qp.physical.sys.CreateAlignedTimeSeriesPlan;
-import org.apache.iotdb.db.qp.physical.sys.CreateContinuousQueryPlan;
-import org.apache.iotdb.db.qp.physical.sys.CreateTemplatePlan;
-import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan;
-import org.apache.iotdb.db.qp.physical.sys.DeleteStorageGroupPlan;
-import org.apache.iotdb.db.qp.physical.sys.DeleteTimeSeriesPlan;
-import org.apache.iotdb.db.qp.physical.sys.DropContinuousQueryPlan;
-import org.apache.iotdb.db.qp.physical.sys.PruneTemplatePlan;
-import org.apache.iotdb.db.qp.physical.sys.SetStorageGroupPlan;
-import org.apache.iotdb.db.qp.physical.sys.SetTTLPlan;
-import org.apache.iotdb.db.qp.physical.sys.SetTemplatePlan;
-import org.apache.iotdb.db.qp.physical.sys.ShowDevicesPlan;
-import org.apache.iotdb.db.qp.physical.sys.ShowTimeSeriesPlan;
-import org.apache.iotdb.db.qp.physical.sys.UnsetTemplatePlan;
+import org.apache.iotdb.db.qp.physical.sys.*;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.dataset.ShowDevicesResult;
import org.apache.iotdb.db.query.dataset.ShowTimeSeriesResult;
import org.apache.iotdb.db.rescon.MemTableManager;
import org.apache.iotdb.db.service.IoTDB;
-import org.apache.iotdb.db.utils.RandomDeleteCache;
import org.apache.iotdb.db.utils.SchemaUtils;
import org.apache.iotdb.db.utils.TestOnly;
import org.apache.iotdb.db.utils.TypeInferenceUtils;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
-import org.apache.iotdb.tsfile.exception.cache.CacheException;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
@@ -95,22 +63,17 @@ import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema;
import org.apache.iotdb.tsfile.write.schema.TimeseriesSchema;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.checkerframework.checker.nullness.qual.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import java.util.Map.Entry;
-import java.util.Set;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
@@ -189,7 +152,7 @@ public class MManager {
private MTree mtree;
// device -> DeviceMNode
- private RandomDeleteCache<PartialPath, IMNode> mNodeCache;
+ private LoadingCache<PartialPath, IMNode> mNodeCache;
private TagManager tagManager = TagManager.getInstance();
private TemplateManager templateManager = TemplateManager.getInstance();
@@ -229,17 +192,17 @@ public class MManager {
int cacheSize = config.getmManagerCacheSize();
mNodeCache =
- new RandomDeleteCache<PartialPath, IMNode>(cacheSize) {
-
- @Override
- public IMNode loadObjectByKey(PartialPath key) throws CacheException {
- try {
- return mtree.getNodeByPathWithStorageGroupCheck(key);
- } catch (MetadataException e) {
- throw new CacheException(e);
- }
- }
- };
+ Caffeine.newBuilder()
+ .maximumSize(cacheSize)
+ .build(
+ new com.github.benmanes.caffeine.cache.CacheLoader<PartialPath, IMNode>() {
+ @Override
+ public @Nullable IMNode load(@NonNull PartialPath partialPath)
+ throws MetadataException {
+
+ return mtree.getNodeByPathWithStorageGroupCheck(partialPath);
+ }
+ });
if (config.isEnableMTreeSnapshot()) {
timedCreateMTreeSnapshotThread =
@@ -355,7 +318,7 @@ public class MManager {
this.mtree.clear();
}
if (this.mNodeCache != null) {
- this.mNodeCache.clear();
+ this.mNodeCache.invalidateAll();
}
this.totalSeriesNumber.set(0);
this.templateManager.clear();
@@ -481,7 +444,7 @@ public class MManager {
plan.getAlias());
// the cached mNode may be replaced by new entityMNode in mtree
- mNodeCache.removeObject(path.getDevicePath());
+ mNodeCache.invalidate(path.getDevicePath());
// update tag index
if (plan.getTags() != null) {
@@ -590,7 +553,7 @@ public class MManager {
plan.getCompressors());
// the cached mNode may be replaced by new entityMNode in mtree
- mNodeCache.removeObject(prefixPath);
+ mNodeCache.invalidate(prefixPath);
// update statistics and schemaDataTypeNumMap
totalSeriesNumber.addAndGet(measurements.size());
@@ -698,7 +661,7 @@ public class MManager {
}
while (node.isEmptyInternal()) {
- mNodeCache.removeObject(node.getPartialPath());
+ mNodeCache.invalidate(node.getPartialPath());
node = node.getParent();
}
totalSeriesNumber.addAndGet(-1);
@@ -749,7 +712,7 @@ public class MManager {
logger.info("Current series number {} come back to normal level", totalSeriesNumber);
allowToCreateNewSeries = true;
}
- mNodeCache.clear();
+ mNodeCache.invalidateAll();
// try to delete storage group
List<IMeasurementMNode> leafMNodes = mtree.deleteStorageGroup(storageGroup);
@@ -801,11 +764,15 @@ public class MManager {
try {
node = mNodeCache.get(path);
return node;
- } catch (CacheException e) {
- if (!autoCreateSchema) {
- throw new PathNotExistException(path.getFullPath());
+ } catch (Exception e) {
+ if (e.getCause() instanceof MetadataException) {
+ if (!autoCreateSchema) {
+ throw new PathNotExistException(path.getFullPath());
+ }
+ shouldSetStorageGroup = e.getCause() instanceof StorageGroupNotSetException;
+ } else {
+ throw e;
}
- shouldSetStorageGroup = e.getCause() instanceof StorageGroupNotSetException;
}
try {
@@ -1232,8 +1199,11 @@ public class MManager {
res.add(measurementPath);
}
}
- } catch (CacheException e) {
- throw new PathNotExistException(devicePath.getFullPath());
+ } catch (Exception e) {
+ if (e.getCause() instanceof MetadataException) {
+ throw new PathNotExistException(devicePath.getFullPath());
+ }
+ throw e;
}
return new ArrayList<>(res);
@@ -1272,8 +1242,11 @@ public class MManager {
try {
node = mNodeCache.get(path);
return node;
- } catch (CacheException e) {
- throw new PathNotExistException(path.getFullPath());
+ } catch (Exception e) {
+ if (e.getCause() instanceof MetadataException) {
+ throw new PathNotExistException(path.getFullPath());
+ }
+ throw e;
}
}
@@ -2078,7 +2051,7 @@ public class MManager {
mountedMNode.setUseTemplate(true);
if (node != mountedMNode) {
- mNodeCache.removeObject(mountedMNode.getPartialPath());
+ mNodeCache.invalidate(mountedMNode.getPartialPath());
}
if (!isRecovering) {
try {
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/RandomDeleteCache.java b/server/src/main/java/org/apache/iotdb/db/utils/RandomDeleteCache.java
deleted file mode 100644
index cea3ecb..0000000
--- a/server/src/main/java/org/apache/iotdb/db/utils/RandomDeleteCache.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.utils;
-
-import org.apache.iotdb.tsfile.common.cache.Cache;
-import org.apache.iotdb.tsfile.exception.cache.CacheException;
-
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-public abstract class RandomDeleteCache<K, V> implements Cache<K, V> {
-
- private int cacheSize;
- private Map<K, V> cache;
-
- public RandomDeleteCache(int cacheSize) {
- this.cacheSize = cacheSize;
- this.cache = new ConcurrentHashMap<>();
- }
-
- @Override
- public V get(K key) throws CacheException {
- V v = cache.get(key);
- if (v == null) {
- randomRemoveObjectIfCacheIsFull();
- cache.put(key, loadObjectByKey(key));
- v = cache.get(key);
- }
- return v;
- }
-
- private void randomRemoveObjectIfCacheIsFull() {
- if (cache.size() == this.cacheSize) {
- removeFirstObject();
- }
- }
-
- private void removeFirstObject() {
- if (cache.size() == 0) {
- return;
- }
- K key = cache.keySet().iterator().next();
- cache.remove(key);
- }
-
- public abstract V loadObjectByKey(K key) throws CacheException;
-
- public void removeObject(K key) {
- cache.remove(key);
- }
-
- @Override
- public void clear() {
- cache.clear();
- }
-
- public int size() {
- return cache.size();
- }
-}