You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ta...@apache.org on 2021/11/09 12:18:45 UTC
[iotdb] 01/04: add log when getting the log
This is an automated email from the ASF dual-hosted git repository.
tanxinyu pushed a commit to branch rel_0.12_debug_compaction_stop
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 482d8fd5509aa21c3249f1f5a99327892fc6d25d
Author: Liu Xuxin <li...@outlook.com>
AuthorDate: Tue Nov 9 15:22:14 2021 +0800
add log when getting the log
---
.../db/engine/compaction/TsFileManagement.java | 4 +-
.../level/LevelCompactionTsFileManagement.java | 2 +-
.../iotdb/db/engine/merge/task/MergeFileTask.java | 4 +-
.../engine/storagegroup/StorageGroupProcessor.java | 10 ++--
.../db/engine/storagegroup/TsFileResource.java | 54 +++++++++++++++++-----
.../iotdb/db/engine/upgrade/UpgradeTask.java | 8 ++--
.../iotdb/db/query/control/FileReaderManager.java | 4 +-
.../org/apache/iotdb/db/utils/UpgradeUtils.java | 8 ++--
8 files changed, 62 insertions(+), 32 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/TsFileManagement.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/TsFileManagement.java
index a9d3a41..a9e09c4 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/TsFileManagement.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/TsFileManagement.java
@@ -322,7 +322,7 @@ public abstract class TsFileManagement {
boolean fileLockGot;
boolean compactionLockGot;
while (true) {
- fileLockGot = seqFile.tryWriteLock();
+ fileLockGot = seqFile.tryWriteLock("doubleWriteLock");
compactionLockGot = tryWriteLock();
if (fileLockGot && compactionLockGot) {
@@ -359,7 +359,7 @@ public abstract class TsFileManagement {
}
for (TsFileResource unseqFile : unseqFiles) {
- unseqFile.writeLock();
+ unseqFile.writeLock("removeUnseqFiles");
try {
unseqFile.remove();
} finally {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
index 0698d7e..894421c 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
@@ -151,7 +151,7 @@ public class LevelCompactionTsFileManagement extends TsFileManagement {
}
private void deleteLevelFile(TsFileResource seqFile) {
- seqFile.writeLock();
+ seqFile.writeLock("deleteLevelFile");
try {
ChunkCache.getInstance().clear();
TimeSeriesMetadataCache.getInstance().clear();
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java
index 36d5b5c..588c8b1 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java
@@ -162,7 +162,7 @@ public class MergeFileTask {
return;
}
- seqFile.writeLock();
+ seqFile.writeLock("moveMergedToOld");
try {
if (Thread.currentThread().isInterrupted()) {
return;
@@ -348,7 +348,7 @@ public class MergeFileTask {
updatePlanIndexes(seqFile);
- seqFile.writeLock();
+ seqFile.writeLock("moveUnmergedToNew");
try {
if (Thread.currentThread().isInterrupted()) {
return;
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index 8361382..767c1fc 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -1395,7 +1395,7 @@ public class StorageGroupProcessor {
resource.setDeleted(true);
// ensure that the file is not used by any queries
- if (resource.tryWriteLock()) {
+ if (resource.tryWriteLock("checkFileTTL")) {
try {
// physical removal
resource.remove();
@@ -2107,7 +2107,7 @@ public class StorageGroupProcessor {
return;
}
for (TsFileResource resource : resources) {
- resource.writeLock();
+ resource.writeLock("loadUpgradedResources");
try {
UpgradeUtils.moveUpgradedFiles(resource);
tsFileManagement.addAll(resource.getUpgradedResources(), isseq);
@@ -2429,7 +2429,7 @@ public class StorageGroupProcessor {
TsFileResource existingTsFile = iterator.next();
if (newTsFile.isPlanRangeCovers(existingTsFile)
&& !newTsFile.getTsFile().equals(existingTsFile.getTsFile())
- && existingTsFile.tryWriteLock()) {
+ && existingTsFile.tryWriteLock("removeFullyOverlapFiles")) {
logger.info(
"{} is covered by {}: [{}, {}], [{}, {}], remove it",
existingTsFile,
@@ -2739,7 +2739,7 @@ public class StorageGroupProcessor {
if (tsFileResourceToBeDeleted == null) {
return false;
}
- tsFileResourceToBeDeleted.writeLock();
+ tsFileResourceToBeDeleted.writeLock("deleteTsfile");
try {
tsFileResourceToBeDeleted.remove();
logger.info("Delete tsfile {} successfully.", tsFileResourceToBeDeleted.getTsFile());
@@ -2793,7 +2793,7 @@ public class StorageGroupProcessor {
if (tsFileResourceToBeMoved == null) {
return false;
}
- tsFileResourceToBeMoved.writeLock();
+ tsFileResourceToBeMoved.writeLock("moveTsfile");
try {
tsFileResourceToBeMoved.moveTo(targetDir);
logger.info(
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
index 08d30e8..2a8dd63 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
@@ -31,7 +31,8 @@ import org.apache.iotdb.db.exception.PartitionViolationException;
import org.apache.iotdb.db.service.UpgradeSevice;
import org.apache.iotdb.db.utils.TestOnly;
import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
-import org.apache.iotdb.tsfile.file.metadata.*;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
+import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
@@ -50,9 +51,21 @@ import java.io.OutputStream;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
import java.nio.file.Paths;
-import java.util.*;
-
-import static org.apache.iotdb.db.conf.IoTDBConstant.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Random;
+import java.util.Set;
+
+import static org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SEPARATOR;
+import static org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SUFFIX_INDEX;
+import static org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SUFFIX_MERGECNT_INDEX;
+import static org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SUFFIX_SEPARATOR;
+import static org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SUFFIX_TIME_INDEX;
+import static org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SUFFIX_UNSEQMERGECNT_INDEX;
+import static org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SUFFIX_VERSION_INDEX;
import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.TSFILE_SUFFIX;
@SuppressWarnings("java:S1135") // ignore todos
@@ -67,6 +80,9 @@ public class TsFileResource {
// tsfile
private File file;
+ private String writeLockHolder;
+ private List<String> readLockHolders = new ArrayList<>();
+
public static final String RESOURCE_SUFFIX = ".resource";
static final String TEMP_SUFFIX = ".temp";
@@ -403,12 +419,15 @@ public class TsFileResource {
return processor;
}
- public void writeLock() {
+ public void writeLock(String holder) {
if (originTsFileResource == null) {
tsFileLock.writeLock();
} else {
- originTsFileResource.writeLock();
+ originTsFileResource.writeLock(holder);
}
+ this.writeLockHolder = holder;
+ logger.warn("write lock stack of {}", holder, new RuntimeException("writeLock"));
+ logger.info("{} get the write lock of {}", holder, this.file);
}
public void writeUnlock() {
@@ -417,30 +436,41 @@ public class TsFileResource {
} else {
originTsFileResource.writeUnlock();
}
+ writeLockHolder = "";
}
/**
* If originTsFileResource is not null, we should acquire the read lock of originTsFileResource
* before construct the current TsFileResource
*/
- public void readLock() {
+ public void readLock(String holder) {
if (originTsFileResource == null) {
tsFileLock.readLock();
} else {
- originTsFileResource.readLock();
+ originTsFileResource.readLock(holder);
}
+ readLockHolders.add(holder);
+ logger.warn("read lock stack of {}", holder, new RuntimeException("readLock"));
+ logger.info("{} get the read lock of {}", holder, this.file);
}
- public void readUnlock() {
+ public void readUnlock(String holder) {
if (originTsFileResource == null) {
tsFileLock.readUnlock();
} else {
- originTsFileResource.readUnlock();
+ originTsFileResource.readUnlock(holder);
}
+ readLockHolders.remove(holder);
}
- public boolean tryWriteLock() {
- return tsFileLock.tryWriteLock();
+ public boolean tryWriteLock(String holder) {
+ if (tsFileLock.tryWriteLock()) {
+ this.writeLockHolder = holder;
+ logger.info("{} get the write lock of {}", holder, this.file);
+ logger.warn("write lock stack of {}", holder, new RuntimeException("writeLock"));
+ return true;
+ }
+ return false;
}
void doUpgrade() {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/upgrade/UpgradeTask.java b/server/src/main/java/org/apache/iotdb/db/engine/upgrade/UpgradeTask.java
index bcf29d8..120f7dd 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/upgrade/UpgradeTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/upgrade/UpgradeTask.java
@@ -82,7 +82,7 @@ public class UpgradeTask extends WrappedRunnable {
}
private List<TsFileResource> generateUpgradedFiles() throws IOException, WriteProcessException {
- upgradeResource.readLock();
+ upgradeResource.readLock("generateUpgradedFiles");
String oldTsfilePath = upgradeResource.getTsFile().getAbsolutePath();
List<TsFileResource> upgradedResources = new ArrayList<>();
UpgradeLog.writeUpgradeLogFile(
@@ -92,13 +92,13 @@ public class UpgradeTask extends WrappedRunnable {
UpgradeLog.writeUpgradeLogFile(
oldTsfilePath + COMMA_SEPERATOR + UpgradeCheckStatus.AFTER_UPGRADE_FILE);
} finally {
- upgradeResource.readUnlock();
+ upgradeResource.readUnlock("generateUpgradedFiles");
}
return upgradedResources;
}
private List<TsFileResource> findUpgradedFiles() throws IOException {
- upgradeResource.readLock();
+ upgradeResource.readLock("findUpgradedFiles");
List<TsFileResource> upgradedResources = new ArrayList<>();
String oldTsfilePath = upgradeResource.getTsFile().getAbsolutePath();
UpgradeLog.writeUpgradeLogFile(
@@ -122,7 +122,7 @@ public class UpgradeTask extends WrappedRunnable {
UpgradeLog.writeUpgradeLogFile(
oldTsfilePath + COMMA_SEPERATOR + UpgradeCheckStatus.AFTER_UPGRADE_FILE);
} finally {
- upgradeResource.readUnlock();
+ upgradeResource.readUnlock("findUpgradedFiles");
}
return upgradedResources;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java b/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java
index cf329af..3fc199f 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java
@@ -192,7 +192,7 @@ public class FileReaderManager implements IService {
* of a reader equals zero, the reader can be closed and removed.
*/
void increaseFileReaderReference(TsFileResource tsFile, boolean isClosed) {
- tsFile.readLock();
+ tsFile.readLock("fileReaderReference");
synchronized (this) {
if (!isClosed) {
unclosedReferenceMap
@@ -218,7 +218,7 @@ public class FileReaderManager implements IService {
closedReferenceMap.get(tsFile.getTsFilePath()).decrementAndGet();
}
}
- tsFile.readUnlock();
+ tsFile.readUnlock("fileReaderReference");
}
/**
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/UpgradeUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/UpgradeUtils.java
index f8e2b82..302a1b9 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/UpgradeUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/UpgradeUtils.java
@@ -62,16 +62,16 @@ public class UpgradeUtils {
/** judge whether a tsfile needs to be upgraded */
public static boolean isNeedUpgrade(TsFileResource tsFileResource) {
- tsFileResource.readLock();
+ tsFileResource.readLock("isNeedUpgrade");
// case the TsFile's length is equal to 0, the TsFile does not need to be upgraded
try {
if (tsFileResource.getTsFile().length() == 0) {
return false;
}
} finally {
- tsFileResource.readUnlock();
+ tsFileResource.readUnlock("isNeedUpgrade");
}
- tsFileResource.readLock();
+ tsFileResource.readLock("isNeedUpgrade");
try (TsFileSequenceReaderForV2 tsFileSequenceReader =
new TsFileSequenceReaderForV2(tsFileResource.getTsFile().getAbsolutePath())) {
String versionNumber = tsFileSequenceReader.readVersionNumberV2();
@@ -85,7 +85,7 @@ public class UpgradeUtils {
tsFileResource.getTsFile().getAbsolutePath(),
e);
} finally {
- tsFileResource.readUnlock();
+ tsFileResource.readUnlock("isNeedUpgrade");
}
return false;
}