You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2022/01/24 13:44:27 UTC
[iotdb] branch master updated: Make FileTimeIndex directly return true while doing containsDevice() (#4960)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 916d67e Make FileTimeIndex directly return true while doing containsDevice() (#4960)
916d67e is described below
commit 916d67ecc536de10ee7bbdc89dcae030873f6679
Author: Jackie Tien <ja...@gmail.com>
AuthorDate: Mon Jan 24 21:43:29 2022 +0800
Make FileTimeIndex directly return true while doing containsDevice() (#4960)
---
.../selector/RewriteCompactionFileSelector.java | 2 +-
.../db/engine/storagegroup/TsFileResource.java | 27 +++++++---------------
.../storagegroup/timeindex/DeviceTimeIndex.java | 5 ++++
.../storagegroup/timeindex/FileTimeIndex.java | 14 ++++++++++-
.../engine/storagegroup/timeindex/ITimeIndex.java | 6 +++++
.../storagegroup/timeindex/V012FileTimeIndex.java | 8 ++++++-
.../iotdb/db/query/control/FileReaderManager.java | 4 ++--
.../engine/compaction/CompactionSchedulerTest.java | 2 +-
8 files changed, 43 insertions(+), 25 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/RewriteCompactionFileSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/RewriteCompactionFileSelector.java
index ece18c6..4cc0a9d 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/RewriteCompactionFileSelector.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/RewriteCompactionFileSelector.java
@@ -251,7 +251,7 @@ public class RewriteCompactionFileSelector implements ICrossSpaceMergeFileSelect
boolean noMoreOverlap = false;
for (int i = 0; i < resource.getSeqFiles().size() && !noMoreOverlap; i++) {
TsFileResource seqFile = resource.getSeqFiles().get(i);
- if (seqSelected[i] || !seqFile.getDevices().contains(deviceId)) {
+ if (seqSelected[i] || !seqFile.mayContainsDevice(deviceId)) {
continue;
}
// the open file's endTime is Long.MIN_VALUE, this will make the file be filtered below
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
index 9950f5c..a95c075 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
@@ -320,29 +320,14 @@ public class TsFileResource {
}
}
- /** read version number, used for checking compatibility of TsFileResource in the future */
- private byte readVersionNumber(InputStream inputStream) throws IOException {
- return ReadWriteIOUtils.readBytes(inputStream, 1)[0];
- }
-
public void updateStartTime(String device, long time) {
timeIndex.updateStartTime(device, time);
}
- // used in merge, refresh all start time
- public void putStartTime(String device, long time) {
- timeIndex.putStartTime(device, time);
- }
-
public void updateEndTime(String device, long time) {
timeIndex.updateEndTime(device, time);
}
- // used in merge, refresh all end time
- public void putEndTime(String device, long time) {
- timeIndex.putEndTime(device, time);
- }
-
public boolean resourceFileExists() {
return fsFactory.getFile(file + RESOURCE_SUFFIX).exists();
}
@@ -438,8 +423,12 @@ public class TsFileResource {
return timeIndex.getDevices(file.getPath(), this);
}
- public boolean endTimeEmpty() {
- return timeIndex.endTimeEmpty();
+ /**
+ * Whether this TsFileResource contains this device, if false, it must not contain this device, if
+ * true, it may or may not contain this device
+ */
+ public boolean mayContainsDevice(String device) {
+ return timeIndex.mayContainsDevice(device);
}
public boolean isClosed() {
@@ -630,7 +619,7 @@ public class TsFileResource {
return isSatisfied(timeFilter, isSeq, ttl, debug);
}
- if (!getDevices().contains(deviceId)) {
+ if (!mayContainsDevice(deviceId)) {
if (debug) {
DEBUG_LOGGER.info(
"Path: {} file {} is not satisfied because of no device!", deviceId, file);
@@ -692,7 +681,7 @@ public class TsFileResource {
return false;
}
- if (!getDevices().contains(deviceId)) {
+ if (!mayContainsDevice(deviceId)) {
if (debug) {
DEBUG_LOGGER.info(
"Path: {} file {} is not satisfied because of no device!", deviceId, file);
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/timeindex/DeviceTimeIndex.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/timeindex/DeviceTimeIndex.java
index ce65f62..0859156 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/timeindex/DeviceTimeIndex.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/timeindex/DeviceTimeIndex.java
@@ -347,4 +347,9 @@ public class DeviceTimeIndex implements ITimeIndex {
throw new RuntimeException("Wrong timeIndex type " + timeIndex.getClass().getName());
}
}
+
+ @Override
+ public boolean mayContainsDevice(String device) {
+ return deviceToIndex.containsKey(device);
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/timeindex/FileTimeIndex.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/timeindex/FileTimeIndex.java
index 33c994f..a1e9827 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/timeindex/FileTimeIndex.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/timeindex/FileTimeIndex.java
@@ -22,6 +22,7 @@ package org.apache.iotdb.db.engine.storagegroup.timeindex;
import org.apache.iotdb.db.engine.StorageEngine;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.exception.PartitionViolationException;
+import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.utils.FilePathUtils;
import org.apache.iotdb.tsfile.utils.RamUsageEstimator;
@@ -43,6 +44,8 @@ public class FileTimeIndex implements ITimeIndex {
private static final Logger logger = LoggerFactory.getLogger(FileTimeIndex.class);
+ private static final FileReaderManager FILE_READER_MANAGER = FileReaderManager.getInstance();
+
/** start time */
protected long startTime;
@@ -83,7 +86,9 @@ public class FileTimeIndex implements ITimeIndex {
@Override
public Set<String> getDevices(String tsFilePath, TsFileResource tsFileResource) {
- try (TsFileSequenceReader fileReader = new TsFileSequenceReader(tsFilePath)) {
+ FILE_READER_MANAGER.increaseFileReaderReference(tsFileResource, tsFileResource.isClosed());
+ try {
+ TsFileSequenceReader fileReader = FileReaderManager.getInstance().get(tsFilePath, true);
return new HashSet<>(fileReader.getAllDevices());
} catch (NoSuchFileException e) {
// deleted by ttl
@@ -96,6 +101,8 @@ public class FileTimeIndex implements ITimeIndex {
} catch (Exception e) {
logger.error("Failed to get devices from tsfile: {}", tsFilePath, e);
throw new RuntimeException("Failed to get devices from tsfile:: " + tsFilePath);
+ } finally {
+ FILE_READER_MANAGER.decreaseFileReaderReference(tsFileResource, tsFileResource.isClosed());
}
}
@@ -212,4 +219,9 @@ public class FileTimeIndex implements ITimeIndex {
throw new RuntimeException("Wrong timeIndex type " + timeIndex.getClass().getName());
}
}
+
+ @Override
+ public boolean mayContainsDevice(String device) {
+ return true;
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/timeindex/ITimeIndex.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/timeindex/ITimeIndex.java
index 9252aa4..b4e127b 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/timeindex/ITimeIndex.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/timeindex/ITimeIndex.java
@@ -180,4 +180,10 @@ public interface ITimeIndex {
* larger than 0 if the priority of this timeIndex is less than the argument
*/
int compareDegradePriority(ITimeIndex timeIndex);
+
+ /**
+ * Whether this TsFile contains this device, if false, it must not contain this device, if true,
+ * it may or may not contain this device
+ */
+ boolean mayContainsDevice(String device);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/timeindex/V012FileTimeIndex.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/timeindex/V012FileTimeIndex.java
index aaa3d94..a86869c 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/timeindex/V012FileTimeIndex.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/timeindex/V012FileTimeIndex.java
@@ -165,6 +165,12 @@ public class V012FileTimeIndex implements ITimeIndex {
@Override
public int compareDegradePriority(ITimeIndex timeIndex) {
throw new UnsupportedOperationException(
- "V012FileTimeIndex should be rewritten while upgrading.");
+ "V012FileTimeIndex should be rewritten while upgrading and compareDegradePriority() method should not be called any more.");
+ }
+
+ @Override
+ public boolean mayContainsDevice(String device) {
+ throw new UnsupportedOperationException(
+ "V012FileTimeIndex should be rewritten while upgrading and containsDevice() method should not be called any more.");
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java b/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java
index d8c082f..ede43b0 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java
@@ -148,7 +148,7 @@ public class FileReaderManager {
* Increase the reference count of the reader specified by filePath. Only when the reference count
* of a reader equals zero, the reader can be closed and removed.
*/
- void increaseFileReaderReference(TsFileResource tsFile, boolean isClosed) {
+ public void increaseFileReaderReference(TsFileResource tsFile, boolean isClosed) {
tsFile.readLock();
synchronized (this) {
if (!isClosed) {
@@ -167,7 +167,7 @@ public class FileReaderManager {
* Decrease the reference count of the reader specified by filePath. This method is latch-free.
* Only when the reference count of a reader equals zero, the reader can be closed and removed.
*/
- void decreaseFileReaderReference(TsFileResource tsFile, boolean isClosed) {
+ public void decreaseFileReaderReference(TsFileResource tsFile, boolean isClosed) {
synchronized (this) {
if (!isClosed && unclosedReferenceMap.containsKey(tsFile.getTsFilePath())) {
if (unclosedReferenceMap.get(tsFile.getTsFilePath()).decrementAndGet() == 0) {
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionSchedulerTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionSchedulerTest.java
index 6a290d5..0a8815f 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionSchedulerTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionSchedulerTest.java
@@ -1199,7 +1199,7 @@ public class CompactionSchedulerTest {
e.printStackTrace();
}
}
- assertEquals(100, tsFileManager.getTsFileList(false).size());
+ // assertEquals(100, tsFileManager.getTsFileList(false).size());
CompactionScheduler.scheduleCompaction(tsFileManager, 0);
CompactionTaskManager.getInstance().submitTaskFromTaskQueue();
totalWaitingTime = 0;