You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2021/12/08 01:00:05 UTC
[iotdb] branch rel/0.12 updated: replace RandomDeleteCache with Caffine CacheLoader as it can't limit the (#4532)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch rel/0.12
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/0.12 by this push:
new 5538be6 replace RandomDeleteCache with Caffine CacheLoader as it can't limit the (#4532)
5538be6 is described below
commit 5538be63420d0ac1d0010dd3cb0ec856be1d2833
Author: Jianyun Cheng <ch...@360.cn>
AuthorDate: Wed Dec 8 08:59:33 2021 +0800
replace RandomDeleteCache with Caffine CacheLoader as it can't limit the (#4532)
---
.../org/apache/iotdb/db/metadata/MManager.java | 108 ++++++++-------------
.../apache/iotdb/db/utils/RandomDeleteCache.java | 76 ---------------
2 files changed, 42 insertions(+), 142 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
index 9c01309..b03dbd6 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
@@ -25,17 +25,7 @@ import org.apache.iotdb.db.engine.StorageEngine;
import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
import org.apache.iotdb.db.engine.storagegroup.StorageGroupProcessor;
import org.apache.iotdb.db.exception.StorageEngineException;
-import org.apache.iotdb.db.exception.metadata.AliasAlreadyExistException;
-import org.apache.iotdb.db.exception.metadata.DataTypeMismatchException;
-import org.apache.iotdb.db.exception.metadata.DeleteFailedException;
-import org.apache.iotdb.db.exception.metadata.DuplicatedTemplateException;
-import org.apache.iotdb.db.exception.metadata.IllegalPathException;
-import org.apache.iotdb.db.exception.metadata.MetadataException;
-import org.apache.iotdb.db.exception.metadata.PathAlreadyExistException;
-import org.apache.iotdb.db.exception.metadata.PathNotExistException;
-import org.apache.iotdb.db.exception.metadata.StorageGroupAlreadySetException;
-import org.apache.iotdb.db.exception.metadata.StorageGroupNotSetException;
-import org.apache.iotdb.db.exception.metadata.UndefinedTemplateException;
+import org.apache.iotdb.db.exception.metadata.*;
import org.apache.iotdb.db.metadata.logfile.MLogReader;
import org.apache.iotdb.db.metadata.logfile.MLogWriter;
import org.apache.iotdb.db.metadata.logfile.TagLogFile;
@@ -46,33 +36,17 @@ import org.apache.iotdb.db.metadata.template.Template;
import org.apache.iotdb.db.monitor.MonitorConstants;
import org.apache.iotdb.db.qp.constant.SQLConstant;
import org.apache.iotdb.db.qp.physical.PhysicalPlan;
-import org.apache.iotdb.db.qp.physical.crud.CreateTemplatePlan;
-import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
-import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan;
-import org.apache.iotdb.db.qp.physical.crud.InsertTabletPlan;
-import org.apache.iotdb.db.qp.physical.crud.SetDeviceTemplatePlan;
-import org.apache.iotdb.db.qp.physical.sys.AutoCreateDeviceMNodePlan;
-import org.apache.iotdb.db.qp.physical.sys.ChangeAliasPlan;
-import org.apache.iotdb.db.qp.physical.sys.ChangeTagOffsetPlan;
-import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan;
-import org.apache.iotdb.db.qp.physical.sys.DeleteStorageGroupPlan;
-import org.apache.iotdb.db.qp.physical.sys.DeleteTimeSeriesPlan;
-import org.apache.iotdb.db.qp.physical.sys.SetStorageGroupPlan;
-import org.apache.iotdb.db.qp.physical.sys.SetTTLPlan;
-import org.apache.iotdb.db.qp.physical.sys.SetUsingDeviceTemplatePlan;
-import org.apache.iotdb.db.qp.physical.sys.ShowDevicesPlan;
-import org.apache.iotdb.db.qp.physical.sys.ShowTimeSeriesPlan;
+import org.apache.iotdb.db.qp.physical.crud.*;
+import org.apache.iotdb.db.qp.physical.sys.*;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.dataset.ShowDevicesResult;
import org.apache.iotdb.db.query.dataset.ShowTimeSeriesResult;
import org.apache.iotdb.db.rescon.MemTableManager;
import org.apache.iotdb.db.service.IoTDB;
-import org.apache.iotdb.db.utils.RandomDeleteCache;
import org.apache.iotdb.db.utils.SchemaUtils;
import org.apache.iotdb.db.utils.TestOnly;
import org.apache.iotdb.db.utils.TypeInferenceUtils;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
-import org.apache.iotdb.tsfile.exception.cache.CacheException;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
@@ -81,26 +55,19 @@ import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
import org.apache.iotdb.tsfile.write.schema.TimeseriesSchema;
+import com.github.benmanes.caffeine.cache.CacheLoader;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.checkerframework.checker.nullness.qual.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
-import java.util.ArrayDeque;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Deque;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
+import java.util.*;
import java.util.Map.Entry;
-import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@@ -142,7 +109,7 @@ public class MManager {
private TagLogFile tagLogFile;
private boolean isRecovering;
// device -> DeviceMNode
- private RandomDeleteCache<PartialPath, Pair<MNode, Template>> mNodeCache;
+ private LoadingCache<PartialPath, Pair<MNode, Template>> mNodeCache;
// tag key -> tag value -> LeafMNode
private Map<String, Map<String, Set<MeasurementMNode>>> tagIndex = new ConcurrentHashMap<>();
@@ -194,17 +161,16 @@ public class MManager {
int cacheSize = config.getmManagerCacheSize();
mNodeCache =
- new RandomDeleteCache<PartialPath, Pair<MNode, Template>>(cacheSize) {
-
- @Override
- public Pair<MNode, Template> loadObjectByKey(PartialPath key) throws CacheException {
- try {
- return mtree.getNodeByPathWithStorageGroupCheck(key);
- } catch (MetadataException e) {
- throw new CacheException(e);
- }
- }
- };
+ Caffeine.newBuilder()
+ .maximumSize(cacheSize)
+ .build(
+ new CacheLoader<PartialPath, Pair<MNode, Template>>() {
+ @Override
+ public @Nullable Pair<MNode, Template> load(@NonNull PartialPath key)
+ throws Exception {
+ return mtree.getNodeByPathWithStorageGroupCheck(key);
+ }
+ });
if (config.isEnableMTreeSnapshot()) {
timedCreateMTreeSnapshotThread =
@@ -311,7 +277,7 @@ public class MManager {
public void clear() {
try {
this.mtree = new MTree();
- this.mNodeCache.clear();
+ this.mNodeCache.invalidateAll();
this.tagIndex.clear();
this.totalSeriesNumber.set(0);
this.templateMap.clear();
@@ -505,7 +471,7 @@ public class MManager {
*/
public String deleteTimeseries(PartialPath prefixPath) throws MetadataException {
if (isStorageGroup(prefixPath)) {
- mNodeCache.clear();
+ mNodeCache.invalidateAll();
}
try {
List<PartialPath> allTimeseries = mtree.getAllTimeseriesPath(prefixPath);
@@ -598,7 +564,7 @@ public class MManager {
PartialPath storageGroupPath = pair.left;
// TODO: delete the path node and all its ancestors
- mNodeCache.clear();
+ mNodeCache.invalidateAll();
totalSeriesNumber.addAndGet(-1);
if (!allowToCreateNewSeries
&& totalSeriesNumber.get() * ESTIMATED_SERIES_SIZE < MTREE_SIZE_THRESHOLD) {
@@ -642,7 +608,7 @@ public class MManager {
logger.info("Current series number {} come back to normal level", totalSeriesNumber);
allowToCreateNewSeries = true;
}
- mNodeCache.clear();
+ mNodeCache.invalidateAll();
// try to delete storage group
List<MeasurementMNode> leafMNodes = mtree.deleteStorageGroup(storageGroup);
@@ -1075,11 +1041,15 @@ public class MManager {
try {
node = mNodeCache.get(path);
return node;
- } catch (CacheException e) {
- if (!autoCreateSchema) {
- throw new PathNotExistException(path.getFullPath());
+ } catch (Exception e) {
+ if (e.getCause() instanceof MetadataException) {
+ if (!autoCreateSchema) {
+ throw new PathNotExistException(path.getFullPath());
+ }
+ shouldSetStorageGroup = e.getCause() instanceof StorageGroupNotSetException;
+ } else {
+ throw e;
}
- shouldSetStorageGroup = e.getCause() instanceof StorageGroupNotSetException;
}
try {
@@ -1139,8 +1109,11 @@ public class MManager {
if (mNodeTemplatePair.left.isUseTemplate() && mNodeTemplatePair.right != null) {
res.addAll(mNodeTemplatePair.right.getSchemaMap().values());
}
- } catch (CacheException e) {
- throw new PathNotExistException(path.getFullPath());
+ } catch (Exception e) {
+ if (e.getCause() instanceof MetadataException) {
+ throw new PathNotExistException(path.getFullPath());
+ }
+ throw e;
}
return new ArrayList<>(res);
@@ -1151,8 +1124,11 @@ public class MManager {
try {
node = mNodeCache.get(path).left;
return node;
- } catch (CacheException e) {
- throw new PathNotExistException(path.getFullPath());
+ } catch (Exception e) {
+ if (e.getCause() instanceof MetadataException) {
+ throw new PathNotExistException(path.getFullPath());
+ }
+ throw e;
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/RandomDeleteCache.java b/server/src/main/java/org/apache/iotdb/db/utils/RandomDeleteCache.java
deleted file mode 100644
index cea3ecb..0000000
--- a/server/src/main/java/org/apache/iotdb/db/utils/RandomDeleteCache.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.utils;
-
-import org.apache.iotdb.tsfile.common.cache.Cache;
-import org.apache.iotdb.tsfile.exception.cache.CacheException;
-
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-public abstract class RandomDeleteCache<K, V> implements Cache<K, V> {
-
- private int cacheSize;
- private Map<K, V> cache;
-
- public RandomDeleteCache(int cacheSize) {
- this.cacheSize = cacheSize;
- this.cache = new ConcurrentHashMap<>();
- }
-
- @Override
- public V get(K key) throws CacheException {
- V v = cache.get(key);
- if (v == null) {
- randomRemoveObjectIfCacheIsFull();
- cache.put(key, loadObjectByKey(key));
- v = cache.get(key);
- }
- return v;
- }
-
- private void randomRemoveObjectIfCacheIsFull() {
- if (cache.size() == this.cacheSize) {
- removeFirstObject();
- }
- }
-
- private void removeFirstObject() {
- if (cache.size() == 0) {
- return;
- }
- K key = cache.keySet().iterator().next();
- cache.remove(key);
- }
-
- public abstract V loadObjectByKey(K key) throws CacheException;
-
- public void removeObject(K key) {
- cache.remove(key);
- }
-
- @Override
- public void clear() {
- cache.clear();
- }
-
- public int size() {
- return cache.size();
- }
-}