You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2021/11/11 06:21:45 UTC
[iotdb] branch master updated: [IOTDB-1924]Remove the operation of
clearing the cache after the compaction is over (#4315)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 87dc695 [IOTDB-1924]Remove the operation of clearing the cache after the compaction is over (#4315)
87dc695 is described below
commit 87dc695e2bd3a8a6ea1997e3e910b56e8ae0d8cd
Author: liuxuxin <37...@users.noreply.github.com>
AuthorDate: Thu Nov 11 14:21:14 2021 +0800
[IOTDB-1924]Remove the operation of clearing the cache after the compaction is over (#4315)
---
.../cross/inplace/InplaceCompactionTask.java | 7 -
.../cross/inplace/task/MergeFileTask.java | 8 --
.../inner/utils/InnerSpaceCompactionUtils.java | 4 -
.../compaction/inner/InnerCompactionCacheTest.java | 148 ---------------------
4 files changed, 167 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/inplace/InplaceCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/inplace/InplaceCompactionTask.java
index 716b89d..f0b368b 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/inplace/InplaceCompactionTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/inplace/InplaceCompactionTask.java
@@ -19,8 +19,6 @@
package org.apache.iotdb.db.engine.compaction.cross.inplace;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.engine.cache.ChunkCache;
-import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache;
import org.apache.iotdb.db.engine.compaction.cross.AbstractCrossSpaceCompactionTask;
import org.apache.iotdb.db.engine.compaction.cross.inplace.manage.CrossSpaceMergeResource;
import org.apache.iotdb.db.engine.compaction.cross.inplace.task.CrossSpaceMergeTask;
@@ -147,11 +145,6 @@ public class InplaceCompactionTask extends AbstractCrossSpaceCompactionTask {
for (TsFileResource unSeqFileMerged : selectedUnSeqTsFileResourceList) {
unSeqTsFileResourceList.remove(unSeqFileMerged);
}
- // clean cache
- if (IoTDBDescriptor.getInstance().getConfig().isMetaDataCacheEnable()) {
- ChunkCache.getInstance().clear();
- TimeSeriesMetadataCache.getInstance().clear();
- }
} finally {
unSeqTsFileResourceList.writeUnlock();
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/inplace/task/MergeFileTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/inplace/task/MergeFileTask.java
index 8db7b0f..02453ad 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/inplace/task/MergeFileTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/inplace/task/MergeFileTask.java
@@ -19,9 +19,6 @@
package org.apache.iotdb.db.engine.compaction.cross.inplace.task;
-import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.engine.cache.ChunkCache;
-import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache;
import org.apache.iotdb.db.engine.compaction.cross.inplace.manage.CrossSpaceMergeContext;
import org.apache.iotdb.db.engine.compaction.cross.inplace.manage.CrossSpaceMergeResource;
import org.apache.iotdb.db.engine.compaction.cross.inplace.recover.MergeLogger;
@@ -374,11 +371,6 @@ public class MergeFileTask {
} catch (Exception e) {
logger.error(e.getMessage(), e);
} finally {
- // clean cache
- if (IoTDBDescriptor.getInstance().getConfig().isMetaDataCacheEnable()) {
- ChunkCache.getInstance().clear();
- TimeSeriesMetadataCache.getInstance().clear();
- }
seqFile.writeUnlock();
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/utils/InnerSpaceCompactionUtils.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/utils/InnerSpaceCompactionUtils.java
index e2dd85e..0c92ff2 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/utils/InnerSpaceCompactionUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/utils/InnerSpaceCompactionUtils.java
@@ -20,8 +20,6 @@
package org.apache.iotdb.db.engine.compaction.inner.utils;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.engine.cache.ChunkCache;
-import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache;
import org.apache.iotdb.db.engine.compaction.cross.inplace.manage.CrossSpaceMergeResource;
import org.apache.iotdb.db.engine.compaction.cross.inplace.manage.MergeManager;
import org.apache.iotdb.db.engine.compaction.cross.inplace.selector.ICrossSpaceMergeFileSelector;
@@ -562,8 +560,6 @@ public class InnerSpaceCompactionUtils {
public static void deleteTsFile(TsFileResource seqFile) {
seqFile.writeLock();
try {
- ChunkCache.getInstance().clear();
- TimeSeriesMetadataCache.getInstance().clear();
FileReaderManager.getInstance().closeFileAndRemoveReader(seqFile.getTsFilePath());
seqFile.setDeleted(true);
seqFile.delete();
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerCompactionCacheTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerCompactionCacheTest.java
deleted file mode 100644
index 3e051cb..0000000
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerCompactionCacheTest.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.engine.compaction.inner;
-
-import org.apache.iotdb.db.conf.IoTDBConstant;
-import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.constant.TestConstant;
-import org.apache.iotdb.db.engine.cache.ChunkCache;
-import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache;
-import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache.TimeSeriesMetadataCacheKey;
-import org.apache.iotdb.db.engine.compaction.CompactionScheduler;
-import org.apache.iotdb.db.engine.compaction.CompactionTaskManager;
-import org.apache.iotdb.db.engine.compaction.inner.sizetiered.SizeTieredCompactionTask;
-import org.apache.iotdb.db.engine.storagegroup.TsFileManager;
-import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
-import org.apache.iotdb.db.exception.StorageEngineException;
-import org.apache.iotdb.db.exception.metadata.MetadataException;
-import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
-import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
-import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
-import org.apache.iotdb.tsfile.read.common.Path;
-
-import org.apache.commons.io.FileUtils;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
-import java.util.Set;
-import java.util.TreeSet;
-
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-public class InnerCompactionCacheTest extends InnerCompactionTest {
-
- File tempSGDir;
-
- @Override
- @Before
- public void setUp() throws IOException, WriteProcessException, MetadataException {
- tempSGDir = new File(TestConstant.getTestTsFileDir("root.compactionTest", 0, 0));
- if (!tempSGDir.exists()) {
- Assert.assertTrue(tempSGDir.mkdirs());
- }
- super.setUp();
- tempSGDir = new File(TestConstant.BASE_OUTPUT_PATH.concat("tempSG"));
- tempSGDir.mkdirs();
- tsFileManager = new TsFileManager(COMPACTION_TEST_SG, "0", tempSGDir.getAbsolutePath());
- }
-
- @Override
- @After
- public void tearDown() throws IOException, StorageEngineException {
- super.tearDown();
- FileUtils.deleteDirectory(tempSGDir);
- }
-
- @Test
- public void testCompactionChunkCache() throws Exception {
- IoTDBDescriptor.getInstance().getConfig().setTargetCompactionFileSize(150000L);
- TsFileResource tsFileResource = seqResources.get(1);
- TsFileSequenceReader reader = new TsFileSequenceReader(tsFileResource.getTsFilePath());
- List<Path> paths = reader.getAllPaths();
- Set<String> allSensors = new TreeSet<>();
- for (Path path : paths) {
- allSensors.add(path.getMeasurement());
- }
- ChunkMetadata firstChunkMetadata = reader.getChunkMetadataList(paths.get(0)).get(0);
- firstChunkMetadata.setFilePath(tsFileResource.getTsFile().getAbsolutePath());
- TimeSeriesMetadataCacheKey firstTimeSeriesMetadataCacheKey =
- new TimeSeriesMetadataCacheKey(
- seqResources.get(1).getTsFilePath(),
- paths.get(0).getDevice(),
- paths.get(0).getMeasurement());
-
- // add cache
- ChunkCache.getInstance().get(firstChunkMetadata);
- TimeSeriesMetadataCache.getInstance().get(firstTimeSeriesMetadataCacheKey, allSensors);
-
- tsFileManager.addAll(seqResources, true);
- tsFileManager.addAll(unseqResources, false);
- CompactionScheduler.addPartitionCompaction(COMPACTION_TEST_SG + "-0", 0);
- File targetFile =
- new File(
- TestConstant.getTestTsFileDir("root.compactionTest", 0, 0)
- .concat(
- 0
- + IoTDBConstant.FILE_NAME_SEPARATOR
- + 0
- + IoTDBConstant.FILE_NAME_SEPARATOR
- + 1
- + IoTDBConstant.FILE_NAME_SEPARATOR
- + 0
- + ".tsfile"));
- if (targetFile.exists()) {
- assertTrue(targetFile.delete());
- }
- SizeTieredCompactionTask sizeTieredCompactionTask =
- new SizeTieredCompactionTask(
- COMPACTION_TEST_SG,
- "0",
- 0,
- tsFileManager,
- tsFileManager.getSequenceListByTimePartition(0),
- seqResources,
- true,
- CompactionTaskManager.currentTaskNum);
- sizeTieredCompactionTask.call();
-
- try {
- firstChunkMetadata.setFilePath(null);
- ChunkCache.getInstance().get(firstChunkMetadata);
- fail();
- } catch (NullPointerException e) {
- assertTrue(true);
- }
-
- try {
- TimeSeriesMetadataCache.getInstance().get(firstTimeSeriesMetadataCacheKey, new TreeSet<>());
- fail();
- } catch (Exception e) {
- assertTrue(true);
- }
-
- reader.close();
- }
-}