You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2022/01/24 02:03:54 UTC
[iotdb] branch FileTimeIndex created (now c1f2200)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a change to branch FileTimeIndex
in repository https://gitbox.apache.org/repos/asf/iotdb.git.
at c1f2200 Make FileTimeIndex directly return true while doing containsDevice() judgement
This branch includes the following new commits:
new c1f2200 Make FileTimeIndex directly return true while doing containsDevice() judgement
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: Make FileTimeIndex directly return true while doing containsDevice() judgement
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch FileTimeIndex
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit c1f220089bf923a2fe0e74468c18c3cb9541de40
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Mon Jan 24 10:00:20 2022 +0800
Make FileTimeIndex directly return true while doing containsDevice() judgement
---
.../rewrite/selector/RewriteCompactionFileSelector.java | 2 +-
.../iotdb/db/engine/storagegroup/TsFileResource.java | 8 ++++----
.../db/engine/storagegroup/timeindex/DeviceTimeIndex.java | 5 +++++
.../db/engine/storagegroup/timeindex/FileTimeIndex.java | 14 +++++++++++++-
.../iotdb/db/engine/storagegroup/timeindex/ITimeIndex.java | 7 +++++++
.../engine/storagegroup/timeindex/V012FileTimeIndex.java | 8 +++++++-
.../apache/iotdb/db/query/control/FileReaderManager.java | 4 ++--
7 files changed, 39 insertions(+), 9 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/RewriteCompactionFileSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/RewriteCompactionFileSelector.java
index ece18c6..18905dd 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/RewriteCompactionFileSelector.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/RewriteCompactionFileSelector.java
@@ -251,7 +251,7 @@ public class RewriteCompactionFileSelector implements ICrossSpaceMergeFileSelect
boolean noMoreOverlap = false;
for (int i = 0; i < resource.getSeqFiles().size() && !noMoreOverlap; i++) {
TsFileResource seqFile = resource.getSeqFiles().get(i);
- if (seqSelected[i] || !seqFile.getDevices().contains(deviceId)) {
+ if (seqSelected[i] || !seqFile.containsDevice(deviceId)) {
continue;
}
// the open file's endTime is Long.MIN_VALUE, this will make the file be filtered below
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
index 9950f5c..238cfd6 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
@@ -438,8 +438,8 @@ public class TsFileResource {
return timeIndex.getDevices(file.getPath(), this);
}
- public boolean endTimeEmpty() {
- return timeIndex.endTimeEmpty();
+ public boolean containsDevice(String device) {
+ return timeIndex.containsDevice(device);
}
public boolean isClosed() {
@@ -630,7 +630,7 @@ public class TsFileResource {
return isSatisfied(timeFilter, isSeq, ttl, debug);
}
- if (!getDevices().contains(deviceId)) {
+ if (containsDevice(deviceId)) {
if (debug) {
DEBUG_LOGGER.info(
"Path: {} file {} is not satisfied because of no device!", deviceId, file);
@@ -692,7 +692,7 @@ public class TsFileResource {
return false;
}
- if (!getDevices().contains(deviceId)) {
+ if (!containsDevice(deviceId)) {
if (debug) {
DEBUG_LOGGER.info(
"Path: {} file {} is not satisfied because of no device!", deviceId, file);
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/timeindex/DeviceTimeIndex.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/timeindex/DeviceTimeIndex.java
index ce65f62..1c33709 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/timeindex/DeviceTimeIndex.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/timeindex/DeviceTimeIndex.java
@@ -347,4 +347,9 @@ public class DeviceTimeIndex implements ITimeIndex {
throw new RuntimeException("Wrong timeIndex type " + timeIndex.getClass().getName());
}
}
+
+ @Override
+ public boolean containsDevice(String device) {
+ return deviceToIndex.containsKey(device);
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/timeindex/FileTimeIndex.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/timeindex/FileTimeIndex.java
index b0768be..f98081b 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/timeindex/FileTimeIndex.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/timeindex/FileTimeIndex.java
@@ -22,6 +22,7 @@ package org.apache.iotdb.db.engine.storagegroup.timeindex;
import org.apache.iotdb.db.engine.StorageEngine;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.exception.PartitionViolationException;
+import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.utils.FilePathUtils;
import org.apache.iotdb.tsfile.utils.RamUsageEstimator;
@@ -43,6 +44,8 @@ public class FileTimeIndex implements ITimeIndex {
private static final Logger logger = LoggerFactory.getLogger(FileTimeIndex.class);
+ private static final FileReaderManager FILE_READER_MANAGER = FileReaderManager.getInstance();
+
/** start time */
protected long startTime;
@@ -83,7 +86,9 @@ public class FileTimeIndex implements ITimeIndex {
@Override
public Set<String> getDevices(String tsFilePath, TsFileResource tsFileResource) {
- try (TsFileSequenceReader fileReader = new TsFileSequenceReader(tsFilePath)) {
+ FILE_READER_MANAGER.increaseFileReaderReference(tsFileResource, tsFileResource.isClosed());
+ try {
+ TsFileSequenceReader fileReader = FileReaderManager.getInstance().get(tsFilePath, true);
return new HashSet<>(fileReader.getAllDevices());
} catch (NoSuchFileException e) {
// deleted by ttl
@@ -96,6 +101,8 @@ public class FileTimeIndex implements ITimeIndex {
} catch (IOException e) {
logger.error("Can't read file {} from disk ", tsFilePath, e);
throw new RuntimeException("Can't read file " + tsFilePath + " from disk");
+ } finally {
+ FILE_READER_MANAGER.decreaseFileReaderReference(tsFileResource, tsFileResource.isClosed());
}
}
@@ -212,4 +219,9 @@ public class FileTimeIndex implements ITimeIndex {
throw new RuntimeException("Wrong timeIndex type " + timeIndex.getClass().getName());
}
}
+
+ @Override
+ public boolean containsDevice(String device) {
+ return true;
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/timeindex/ITimeIndex.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/timeindex/ITimeIndex.java
index 9252aa4..6d68d86 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/timeindex/ITimeIndex.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/timeindex/ITimeIndex.java
@@ -180,4 +180,11 @@ public interface ITimeIndex {
* larger than 0 if the priority of this timeIndex is less than the argument
*/
int compareDegradePriority(ITimeIndex timeIndex);
+
+ /**
+ * Whether this TsFile contains this device
+ *
+ * @return true, if it contains, otherwise false
+ */
+ boolean containsDevice(String device);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/timeindex/V012FileTimeIndex.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/timeindex/V012FileTimeIndex.java
index aaa3d94..c2be5eb 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/timeindex/V012FileTimeIndex.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/timeindex/V012FileTimeIndex.java
@@ -165,6 +165,12 @@ public class V012FileTimeIndex implements ITimeIndex {
@Override
public int compareDegradePriority(ITimeIndex timeIndex) {
throw new UnsupportedOperationException(
- "V012FileTimeIndex should be rewritten while upgrading.");
+ "V012FileTimeIndex should be rewritten while upgrading and compareDegradePriority() method should not be called any more.");
+ }
+
+ @Override
+ public boolean containsDevice(String device) {
+ throw new UnsupportedOperationException(
+ "V012FileTimeIndex should be rewritten while upgrading and containsDevice() method should not be called any more.");
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java b/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java
index d8c082f..ede43b0 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java
@@ -148,7 +148,7 @@ public class FileReaderManager {
* Increase the reference count of the reader specified by filePath. Only when the reference count
* of a reader equals zero, the reader can be closed and removed.
*/
- void increaseFileReaderReference(TsFileResource tsFile, boolean isClosed) {
+ public void increaseFileReaderReference(TsFileResource tsFile, boolean isClosed) {
tsFile.readLock();
synchronized (this) {
if (!isClosed) {
@@ -167,7 +167,7 @@ public class FileReaderManager {
* Decrease the reference count of the reader specified by filePath. This method is latch-free.
* Only when the reference count of a reader equals zero, the reader can be closed and removed.
*/
- void decreaseFileReaderReference(TsFileResource tsFile, boolean isClosed) {
+ public void decreaseFileReaderReference(TsFileResource tsFile, boolean isClosed) {
synchronized (this) {
if (!isClosed && unclosedReferenceMap.containsKey(tsFile.getTsFilePath())) {
if (unclosedReferenceMap.get(tsFile.getTsFilePath()).decrementAndGet() == 0) {