[iotdb] branch master updated: [ISSUE-2709, IOTDB-1178] Fix cache not cleared after unseq compaction bug, Fix windows 70, 10 ci bug in unseq compaction ci (#2707)

[qi...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git


The following commit(s) were added to refs/heads/master by this push:
     new d3e20ea  [ISSUE-2709,IOTDB-1178] Fix cache not cleared after unseq compaction bug, Fix windows 70,10 ci bug in unseq compaction ci (#2707)
d3e20ea is described below

commit d3e20eaea5ac89f1ad17cf7fcd09c72d43cbd0c7
Author: zhanglingzhe0820 <44...@qq.com>
AuthorDate: Mon Mar 1 16:22:11 2021 +0800

    [ISSUE-2709,IOTDB-1178] Fix cache not cleared after unseq compaction bug, Fix windows 70,10 ci bug in unseq compaction ci (#2707)
---
 .../level/LevelCompactionTsFileManagement.java     |   4 +
 .../iotdb/db/engine/merge/task/MergeFileTask.java  |   4 +-
 .../compaction/LevelCompactionCacheTest.java       | 125 +++++++++++++++++++++
 3 files changed, 130 insertions(+), 3 deletions(-)

diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
index 1a19e3d..0f4d3e5 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
@@ -20,7 +20,9 @@
 package org.apache.iotdb.db.engine.compaction.level;
 
 import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.cache.ChunkCache;
 import org.apache.iotdb.db.engine.cache.ChunkMetadataCache;
+import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache;
 import org.apache.iotdb.db.engine.compaction.TsFileManagement;
 import org.apache.iotdb.db.engine.compaction.utils.CompactionLogAnalyzer;
 import org.apache.iotdb.db.engine.compaction.utils.CompactionLogger;
@@ -154,7 +156,9 @@ public class LevelCompactionTsFileManagement extends TsFileManagement {
   private void deleteLevelFile(TsFileResource seqFile) {
     seqFile.writeLock();
     try {
+      ChunkCache.getInstance().clear();
       ChunkMetadataCache.getInstance().remove(seqFile);
+      TimeSeriesMetadataCache.getInstance().clear();
       FileReaderManager.getInstance().closeFileAndRemoveReader(seqFile.getTsFilePath());
       seqFile.setDeleted(true);
       seqFile.delete();
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java
index 2fe879f..9d3ca6b 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java
@@ -326,8 +326,7 @@ class MergeFileTask {
     seqFile.writeLock();
     try {
       resource.removeFileReader(seqFile);
-      ChunkMetadataCache.getInstance().remove(seqFile);
-
+      FileReaderManager.getInstance().closeFileAndRemoveReader(seqFile.getTsFilePath());
       File newMergeFile = seqFile.getTsFile();
       newMergeFile.delete();
       fsFactory.moveFile(fileWriter.getFile(), newMergeFile);
@@ -340,7 +339,6 @@ class MergeFileTask {
         ChunkCache.getInstance().clear();
         ChunkMetadataCache.getInstance().clear();
         TimeSeriesMetadataCache.getInstance().clear();
-        FileReaderManager.getInstance().closeFileAndRemoveReader(seqFile.getTsFilePath());
       }
       seqFile.writeUnlock();
     }
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionCacheTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionCacheTest.java
new file mode 100644
index 0000000..4169816
--- /dev/null
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionCacheTest.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.compaction;
+
+import org.apache.iotdb.db.constant.TestConstant;
+import org.apache.iotdb.db.engine.cache.ChunkCache;
+import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache;
+import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache.TimeSeriesMetadataCacheKey;
+import org.apache.iotdb.db.engine.compaction.TsFileManagement.CompactionMergeTask;
+import org.apache.iotdb.db.engine.compaction.level.LevelCompactionTsFileManagement;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.exception.StorageEngineException;
+import org.apache.iotdb.db.exception.metadata.IllegalPathException;
+import org.apache.iotdb.db.exception.metadata.MetadataException;
+import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
+import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
+import org.apache.iotdb.tsfile.read.common.Path;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class LevelCompactionCacheTest extends LevelCompactionTest {
+
+  File tempSGDir;
+  boolean compactionMergeWorking = false;
+
+  @Override
+  @Before
+  public void setUp() throws IOException, WriteProcessException, MetadataException {
+    super.setUp();
+    tempSGDir = new File(TestConstant.BASE_OUTPUT_PATH.concat("tempSG"));
+    tempSGDir.mkdirs();
+  }
+
+  @Override
+  @After
+  public void tearDown() throws IOException, StorageEngineException {
+    super.tearDown();
+    FileUtils.deleteDirectory(tempSGDir);
+  }
+
+  @Test
+  public void testCompactionChunkCache() throws IllegalPathException, IOException {
+    LevelCompactionTsFileManagement levelCompactionTsFileManagement =
+        new LevelCompactionTsFileManagement(COMPACTION_TEST_SG, tempSGDir.getPath());
+    TsFileResource tsFileResource = seqResources.get(1);
+    TsFileSequenceReader reader = new TsFileSequenceReader(tsFileResource.getTsFilePath());
+    List<Path> paths = reader.getAllPaths();
+    Set<String> allSensors = new TreeSet<>();
+    for (Path path : paths) {
+      allSensors.add(path.getMeasurement());
+    }
+    ChunkMetadata firstChunkMetadata = reader.getChunkMetadataList(paths.get(0)).get(0);
+    TimeSeriesMetadataCacheKey firstTimeSeriesMetadataCacheKey =
+        new TimeSeriesMetadataCacheKey(
+            seqResources.get(1).getTsFilePath(),
+            paths.get(0).getDevice(),
+            paths.get(0).getMeasurement());
+
+    // add cache
+    ChunkCache.getInstance().get(firstChunkMetadata, reader);
+    TimeSeriesMetadataCache.getInstance().get(firstTimeSeriesMetadataCacheKey, allSensors);
+
+    levelCompactionTsFileManagement.addAll(seqResources, true);
+    levelCompactionTsFileManagement.addAll(unseqResources, false);
+    levelCompactionTsFileManagement.forkCurrentFileList(0);
+    CompactionMergeTask compactionMergeTask =
+        levelCompactionTsFileManagement
+        .new CompactionMergeTask(this::closeCompactionMergeCallBack, 0);
+    compactionMergeWorking = true;
+    compactionMergeTask.run();
+    while (compactionMergeWorking) {
+      // wait
+    }
+
+    try {
+      ChunkCache.getInstance().get(firstChunkMetadata, null);
+      fail();
+    } catch (NullPointerException e) {
+      assertTrue(true);
+    }
+
+    try {
+      TimeSeriesMetadataCache.getInstance().get(firstTimeSeriesMetadataCacheKey, new TreeSet<>());
+      fail();
+    } catch (Exception e) {
+      assertTrue(true);
+    }
+    reader.close();
+  }
+
+  /** close compaction merge callback, to release some locks */
+  private void closeCompactionMergeCallBack() {
+    this.compactionMergeWorking = false;
+  }
+}