You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ta...@apache.org on 2023/02/03 07:58:13 UTC
[iotdb] branch rel/1.0 updated: fix the ConcurrentModificationException (#8972)
This is an automated email from the ASF dual-hosted git repository.
tanxinyu pushed a commit to branch rel/1.0
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/1.0 by this push:
new fad7f73f50 fix the ConcurrentModificationException (#8972)
fad7f73f50 is described below
commit fad7f73f50ef2d1c32f5792a08669237358123cb
Author: Houliang Qi <ne...@163.com>
AuthorDate: Fri Feb 3 15:58:06 2023 +0800
fix the ConcurrentModificationException (#8972)
---
.../exception/CompactionExceptionHandler.java | 4 +--
.../execute/task/CrossSpaceCompactionTask.java | 6 ++--
.../execute/task/InnerSpaceCompactionTask.java | 4 +--
.../compaction/execute/utils/CompactionUtils.java | 2 +-
.../compaction/schedule/CompactionScheduler.java | 8 +++---
.../iotdb/db/engine/storagegroup/DataRegion.java | 2 +-
.../engine/storagegroup/HashLastFlushTimeMap.java | 2 +-
.../storagegroup/IDTableLastFlushTimeMap.java | 2 +-
.../db/engine/storagegroup/TsFileManager.java | 12 ++++----
.../cross/CrossSpaceCompactionExceptionTest.java | 32 +++++++++++-----------
...eCompactionWithFastPerformerValidationTest.java | 12 ++++----
.../inner/InnerSpaceCompactionExceptionTest.java | 9 +++---
.../SizeTieredCompactionSelectorTest.java | 4 +--
13 files changed, 51 insertions(+), 48 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/exception/CompactionExceptionHandler.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/exception/CompactionExceptionHandler.java
index 70dcebf3f7..a510a77221 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/exception/CompactionExceptionHandler.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/exception/CompactionExceptionHandler.java
@@ -140,9 +140,9 @@ public class CompactionExceptionHandler {
String fullStorageGroupName)
throws IOException {
TsFileResourceList unseqTsFileResourceList =
- tsFileManager.getUnsequenceListByTimePartition(timePartition);
+ tsFileManager.getOrCreateUnsequenceListByTimePartition(timePartition);
TsFileResourceList seqTsFileResourceList =
- tsFileManager.getSequenceListByTimePartition(timePartition);
+ tsFileManager.getOrCreateSequenceListByTimePartition(timePartition);
// delete compaction mods files
CompactionUtils.deleteCompactionModsFile(sourceSeqResourceList, sourceUnseqResourceList);
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java
index 2fef6dd48d..a87cdd3d29 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java
@@ -83,8 +83,10 @@ public class CrossSpaceCompactionTask extends AbstractCompactionTask {
serialId);
this.selectedSequenceFiles = selectedSequenceFiles;
this.selectedUnsequenceFiles = selectedUnsequenceFiles;
- this.seqTsFileResourceList = tsFileManager.getSequenceListByTimePartition(timePartition);
- this.unseqTsFileResourceList = tsFileManager.getUnsequenceListByTimePartition(timePartition);
+ this.seqTsFileResourceList =
+ tsFileManager.getOrCreateSequenceListByTimePartition(timePartition);
+ this.unseqTsFileResourceList =
+ tsFileManager.getOrCreateUnsequenceListByTimePartition(timePartition);
this.performer = performer;
this.hashCode = this.toString().hashCode();
this.memoryCost = memoryCost;
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/InnerSpaceCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/InnerSpaceCompactionTask.java
index f65c3799b7..a5c4b9fbd5 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/InnerSpaceCompactionTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/InnerSpaceCompactionTask.java
@@ -90,9 +90,9 @@ public class InnerSpaceCompactionTask extends AbstractCompactionTask {
isHoldingReadLock[i] = false;
}
if (sequence) {
- tsFileResourceList = tsFileManager.getSequenceListByTimePartition(timePartition);
+ tsFileResourceList = tsFileManager.getOrCreateSequenceListByTimePartition(timePartition);
} else {
- tsFileResourceList = tsFileManager.getUnsequenceListByTimePartition(timePartition);
+ tsFileResourceList = tsFileManager.getOrCreateUnsequenceListByTimePartition(timePartition);
}
this.hashCode = this.toString().hashCode();
collectSelectedFilesInfo();
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/CompactionUtils.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/CompactionUtils.java
index 00bf68b85b..e60d4d7409 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/CompactionUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/CompactionUtils.java
@@ -257,7 +257,7 @@ public class CompactionUtils {
public static boolean validateTsFileResources(
TsFileManager manager, String storageGroupName, long timePartition) {
List<TsFileResource> resources =
- manager.getSequenceListByTimePartition(timePartition).getArrayList();
+ manager.getOrCreateSequenceListByTimePartition(timePartition).getArrayList();
resources.sort(
(f1, f2) ->
Long.compareUnsigned(
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/schedule/CompactionScheduler.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/schedule/CompactionScheduler.java
index 7551d9b03d..0173423537 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/schedule/CompactionScheduler.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/schedule/CompactionScheduler.java
@@ -106,8 +106,8 @@ public class CompactionScheduler {
List<List<TsFileResource>> taskList =
innerSpaceCompactionSelector.selectInnerSpaceTask(
sequence
- ? tsFileManager.getSequenceListByTimePartition(timePartition)
- : tsFileManager.getUnsequenceListByTimePartition(timePartition));
+ ? tsFileManager.getOrCreateSequenceListByTimePartition(timePartition)
+ : tsFileManager.getOrCreateUnsequenceListByTimePartition(timePartition));
for (List<TsFileResource> task : taskList) {
ICompactionPerformer performer =
sequence
@@ -147,8 +147,8 @@ public class CompactionScheduler {
.createInstance(logicalStorageGroupName, dataRegionId, timePartition, tsFileManager);
List<CrossCompactionTaskResource> taskList =
crossSpaceCompactionSelector.selectCrossSpaceTask(
- tsFileManager.getSequenceListByTimePartition(timePartition),
- tsFileManager.getUnsequenceListByTimePartition(timePartition));
+ tsFileManager.getOrCreateSequenceListByTimePartition(timePartition),
+ tsFileManager.getOrCreateUnsequenceListByTimePartition(timePartition));
List<Long> memoryCost = crossSpaceCompactionSelector.getCompactionMemoryCost();
for (int i = 0, size = taskList.size(); i < size; ++i) {
CompactionTaskManager.getInstance()
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
index db4e9e8fcd..21b320e264 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
@@ -2334,7 +2334,7 @@ public class DataRegion implements IDataRegionForQuery {
writeLock("loadNewTsFile");
try {
List<TsFileResource> sequenceList =
- tsFileManager.getSequenceListByTimePartition(newFilePartitionId);
+ tsFileManager.getOrCreateSequenceListByTimePartition(newFilePartitionId);
int insertPos = findInsertionPosition(newTsFileResource, sequenceList);
LoadTsFileType tsFileType = getLoadingTsFileType(insertPos, sequenceList);
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/HashLastFlushTimeMap.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/HashLastFlushTimeMap.java
index 020077bb6b..50d524a3e5 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/HashLastFlushTimeMap.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/HashLastFlushTimeMap.java
@@ -225,7 +225,7 @@ public class HashLastFlushTimeMap implements ILastFlushTimeMap {
private long recoverFlushTime(long partitionId, String devicePath) {
List<TsFileResource> tsFileResourceList =
- tsFileManager.getSequenceListByTimePartition(partitionId);
+ tsFileManager.getOrCreateSequenceListByTimePartition(partitionId);
for (int i = tsFileResourceList.size() - 1; i >= 0; i--) {
if (tsFileResourceList.get(i).timeIndex.mayContainsDevice(devicePath)) {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/IDTableLastFlushTimeMap.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/IDTableLastFlushTimeMap.java
index 6ad437fdc4..03dd87476a 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/IDTableLastFlushTimeMap.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/IDTableLastFlushTimeMap.java
@@ -164,7 +164,7 @@ public class IDTableLastFlushTimeMap implements ILastFlushTimeMap {
private long recoverFlushTime(long partitionId, String devicePath) {
List<TsFileResource> tsFileResourceList =
- tsFileManager.getSequenceListByTimePartition(partitionId);
+ tsFileManager.getOrCreateSequenceListByTimePartition(partitionId);
for (int i = tsFileResourceList.size() - 1; i >= 0; i--) {
if (tsFileResourceList.get(i).timeIndex.mayContainsDevice(devicePath)) {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileManager.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileManager.java
index 96d0060c95..86fa5e742e 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileManager.java
@@ -85,21 +85,21 @@ public class TsFileManager {
}
}
- public TsFileResourceList getSequenceListByTimePartition(long timePartition) {
- readLock();
+ public TsFileResourceList getOrCreateSequenceListByTimePartition(long timePartition) {
+ writeLock("getOrCreateSequenceListByTimePartition");
try {
return sequenceFiles.computeIfAbsent(timePartition, l -> new TsFileResourceList());
} finally {
- readUnlock();
+ writeUnlock();
}
}
- public TsFileResourceList getUnsequenceListByTimePartition(long timePartition) {
- readLock();
+ public TsFileResourceList getOrCreateUnsequenceListByTimePartition(long timePartition) {
+ writeLock("getOrCreateUnsequenceListByTimePartition");
try {
return unsequenceFiles.computeIfAbsent(timePartition, l -> new TsFileResourceList());
} finally {
- readUnlock();
+ writeUnlock();
}
}
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionExceptionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionExceptionTest.java
index 2a4ac6a860..6bbaa0a49b 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionExceptionTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionExceptionTest.java
@@ -145,8 +145,8 @@ public class CrossSpaceCompactionExceptionTest extends AbstractCompactionTest {
+ TsFileResource.RESOURCE_SUFFIX)
.exists());
}
- Assert.assertEquals(4, tsFileManager.getSequenceListByTimePartition(0).size());
- Assert.assertEquals(5, tsFileManager.getUnsequenceListByTimePartition(0).size());
+ Assert.assertEquals(4, tsFileManager.getOrCreateSequenceListByTimePartition(0).size());
+ Assert.assertEquals(5, tsFileManager.getOrCreateUnsequenceListByTimePartition(0).size());
Assert.assertTrue(tsFileManager.isAllowCompaction());
}
@@ -220,8 +220,8 @@ public class CrossSpaceCompactionExceptionTest extends AbstractCompactionTest {
+ TsFileResource.RESOURCE_SUFFIX)
.exists());
}
- Assert.assertEquals(4, tsFileManager.getSequenceListByTimePartition(0).size());
- Assert.assertEquals(5, tsFileManager.getUnsequenceListByTimePartition(0).size());
+ Assert.assertEquals(4, tsFileManager.getOrCreateSequenceListByTimePartition(0).size());
+ Assert.assertEquals(5, tsFileManager.getOrCreateUnsequenceListByTimePartition(0).size());
Assert.assertTrue(tsFileManager.isAllowCompaction());
}
@@ -253,13 +253,13 @@ public class CrossSpaceCompactionExceptionTest extends AbstractCompactionTest {
performer.perform();
CompactionUtils.moveTargetFile(targetResources, false, COMPACTION_TEST_SG);
for (TsFileResource resource : seqResources) {
- tsFileManager.getSequenceListByTimePartition(0).remove(resource);
+ tsFileManager.getOrCreateSequenceListByTimePartition(0).remove(resource);
}
for (TsFileResource resource : unseqResources) {
- tsFileManager.getUnsequenceListByTimePartition(0).remove(resource);
+ tsFileManager.getOrCreateUnsequenceListByTimePartition(0).remove(resource);
}
for (TsFileResource resource : targetResources) {
- tsFileManager.getSequenceListByTimePartition(0).keepOrderInsert(resource);
+ tsFileManager.getOrCreateSequenceListByTimePartition(0).keepOrderInsert(resource);
}
seqResources.get(0).getTsFile().delete();
compactionLogger.close();
@@ -301,8 +301,8 @@ public class CrossSpaceCompactionExceptionTest extends AbstractCompactionTest {
Assert.assertTrue(
new File(resource.getTsFilePath() + TsFileResource.RESOURCE_SUFFIX).exists());
}
- Assert.assertEquals(4, tsFileManager.getSequenceListByTimePartition(0).size());
- Assert.assertEquals(0, tsFileManager.getUnsequenceListByTimePartition(0).size());
+ Assert.assertEquals(4, tsFileManager.getOrCreateSequenceListByTimePartition(0).size());
+ Assert.assertEquals(0, tsFileManager.getOrCreateUnsequenceListByTimePartition(0).size());
Assert.assertTrue(tsFileManager.isAllowCompaction());
}
@@ -356,13 +356,13 @@ public class CrossSpaceCompactionExceptionTest extends AbstractCompactionTest {
}
CompactionUtils.combineModsInCrossCompaction(seqResources, unseqResources, targetResources);
for (TsFileResource resource : seqResources) {
- tsFileManager.getSequenceListByTimePartition(0).remove(resource);
+ tsFileManager.getOrCreateSequenceListByTimePartition(0).remove(resource);
}
for (TsFileResource resource : unseqResources) {
- tsFileManager.getUnsequenceListByTimePartition(0).remove(resource);
+ tsFileManager.getOrCreateUnsequenceListByTimePartition(0).remove(resource);
}
for (TsFileResource resource : targetResources) {
- tsFileManager.getSequenceListByTimePartition(0).keepOrderInsert(resource);
+ tsFileManager.getOrCreateSequenceListByTimePartition(0).keepOrderInsert(resource);
}
seqResources.get(0).remove();
@@ -418,8 +418,8 @@ public class CrossSpaceCompactionExceptionTest extends AbstractCompactionTest {
// compaction log file should not exist
Assert.assertFalse(compactionLogFile.exists());
- Assert.assertEquals(4, tsFileManager.getSequenceListByTimePartition(0).size());
- Assert.assertEquals(0, tsFileManager.getUnsequenceListByTimePartition(0).size());
+ Assert.assertEquals(4, tsFileManager.getOrCreateSequenceListByTimePartition(0).size());
+ Assert.assertEquals(0, tsFileManager.getOrCreateUnsequenceListByTimePartition(0).size());
Assert.assertTrue(tsFileManager.isAllowCompaction());
}
@@ -541,8 +541,8 @@ public class CrossSpaceCompactionExceptionTest extends AbstractCompactionTest {
// compaction log file should not exist
Assert.assertFalse(compactionLogFile.exists());
- Assert.assertEquals(4, tsFileManager.getSequenceListByTimePartition(0).size());
- Assert.assertEquals(5, tsFileManager.getUnsequenceListByTimePartition(0).size());
+ Assert.assertEquals(4, tsFileManager.getOrCreateSequenceListByTimePartition(0).size());
+ Assert.assertEquals(5, tsFileManager.getOrCreateUnsequenceListByTimePartition(0).size());
Assert.assertTrue(tsFileManager.isAllowCompaction());
}
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java
index 2ec09f1daa..9aced0d1c5 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java
@@ -2224,8 +2224,8 @@ public class CrossSpaceCompactionWithFastPerformerValidationTest extends Abstrac
CrossCompactionTaskResource sourceFiles =
crossSpaceCompactionSelector
.selectCrossSpaceTask(
- tsFileManager.getSequenceListByTimePartition(0),
- tsFileManager.getUnsequenceListByTimePartition(0))
+ tsFileManager.getOrCreateSequenceListByTimePartition(0),
+ tsFileManager.getOrCreateUnsequenceListByTimePartition(0))
.get(0);
Assert.assertEquals(2, sourceFiles.getSeqFiles().size());
Assert.assertEquals(1, sourceFiles.getUnseqFiles().size());
@@ -2251,8 +2251,8 @@ public class CrossSpaceCompactionWithFastPerformerValidationTest extends Abstrac
0,
crossSpaceCompactionSelector
.selectCrossSpaceTask(
- tsFileManager.getSequenceListByTimePartition(0),
- tsFileManager.getUnsequenceListByTimePartition(0))
+ tsFileManager.getOrCreateSequenceListByTimePartition(0),
+ tsFileManager.getOrCreateUnsequenceListByTimePartition(0))
.size());
// Target file of the first task should not be selected to participate in other inner compaction
@@ -2271,8 +2271,8 @@ public class CrossSpaceCompactionWithFastPerformerValidationTest extends Abstrac
// compaction task
List<CrossCompactionTaskResource> pairs =
crossSpaceCompactionSelector.selectCrossSpaceTask(
- tsFileManager.getSequenceListByTimePartition(0),
- tsFileManager.getUnsequenceListByTimePartition(0));
+ tsFileManager.getOrCreateSequenceListByTimePartition(0),
+ tsFileManager.getOrCreateUnsequenceListByTimePartition(0));
Assert.assertEquals(1, pairs.size());
Assert.assertEquals(2, pairs.get(0).getSeqFiles().size());
Assert.assertEquals(1, pairs.get(0).getUnseqFiles().size());
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionExceptionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionExceptionTest.java
index c53bab30e7..25688361cb 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionExceptionTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionExceptionTest.java
@@ -181,9 +181,9 @@ public class InnerSpaceCompactionExceptionTest extends AbstractInnerSpaceCompact
CompactionUtils.moveTargetFile(
Collections.singletonList(targetResource), true, COMPACTION_TEST_SG);
for (TsFileResource resource : seqResources) {
- tsFileManager.getSequenceListByTimePartition(0).remove(resource);
+ tsFileManager.getOrCreateSequenceListByTimePartition(0).remove(resource);
}
- tsFileManager.getSequenceListByTimePartition(0).keepOrderInsert(targetResource);
+ tsFileManager.getOrCreateSequenceListByTimePartition(0).keepOrderInsert(targetResource);
FileUtils.delete(seqResources.get(0).getTsFile().getPath());
seqResources.get(0).remove();
compactionLogger.close();
@@ -209,8 +209,9 @@ public class InnerSpaceCompactionExceptionTest extends AbstractInnerSpaceCompact
}
Assert.assertTrue(tsFileManager.isAllowCompaction());
- Assert.assertEquals(1, tsFileManager.getSequenceListByTimePartition(0).size());
- Assert.assertEquals(targetResource, tsFileManager.getSequenceListByTimePartition(0).get(0));
+ Assert.assertEquals(1, tsFileManager.getOrCreateSequenceListByTimePartition(0).size());
+ Assert.assertEquals(
+ targetResource, tsFileManager.getOrCreateSequenceListByTimePartition(0).get(0));
}
/**
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelectorTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelectorTest.java
index 470dd48a94..730a4e5679 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelectorTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelectorTest.java
@@ -55,14 +55,14 @@ public class SizeTieredCompactionSelectorTest {
Assert.assertEquals(
1,
new SizeTieredCompactionSelector("root.test", "0", i, true, manager)
- .selectInnerSpaceTask(manager.getSequenceListByTimePartition(i))
+ .selectInnerSpaceTask(manager.getOrCreateSequenceListByTimePartition(i))
.size());
}
Assert.assertEquals(
0,
new SizeTieredCompactionSelector("root.test", "0", 9, true, manager)
- .selectInnerSpaceTask(manager.getSequenceListByTimePartition(9))
+ .selectInnerSpaceTask(manager.getOrCreateSequenceListByTimePartition(9))
.size());
}
}