You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by he...@apache.org on 2023/05/24 09:04:10 UTC
[iotdb] branch tiered_storage updated (92e7cfcf277 -> 024cc1b4c8d)
This is an automated email from the ASF dual-hosted git repository.
heiming pushed a change to branch tiered_storage
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 92e7cfcf277 refine code
new fd6d0953e5a disable compaction on the remote
new 024cc1b4c8d add cache recover
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../java/org/apache/iotdb/db/conf/directories/TierManager.java | 3 +++
.../main/java/org/apache/iotdb/db/engine/StorageEngine.java | 10 ++++++++++
.../compaction/selector/impl/SizeTieredCompactionSelector.java | 4 +++-
.../selector/utils/CrossSpaceCompactionCandidate.java | 2 ++
.../apache/iotdb/db/engine/storagegroup/TsFileResource.java | 4 ++++
5 files changed, 22 insertions(+), 1 deletion(-)
[iotdb] 02/02: add cache recover
Posted by he...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
heiming pushed a commit to branch tiered_storage
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 024cc1b4c8d003b971f2cfb3111b2e20b99f9c0b
Author: HeimingZ <zh...@qq.com>
AuthorDate: Wed May 24 16:42:56 2023 +0800
add cache recover
---
.../main/java/org/apache/iotdb/db/engine/StorageEngine.java | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
index 0021dcfa6f1..c122b79c444 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
@@ -63,6 +63,7 @@ import org.apache.iotdb.db.utils.UpgradeUtils;
import org.apache.iotdb.db.wal.WALManager;
import org.apache.iotdb.db.wal.exception.WALException;
import org.apache.iotdb.db.wal.recover.WALRecoverManager;
+import org.apache.iotdb.os.cache.CacheRecoverTask;
import org.apache.iotdb.rpc.RpcUtils;
import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.tsfile.exception.write.PageException;
@@ -273,6 +274,15 @@ public class StorageEngine implements IService {
futures.add(cachedThreadPool.submit(recoverDataRegionTask));
}
}
+ // submit object storage cache recover task
+ if (config.isEnableObjectStorage()) {
+ futures.add(
+ cachedThreadPool.submit(
+ () -> {
+ new CacheRecoverTask().run();
+ return null;
+ }));
+ }
}
/** get StorageGroup -> DataRegionIdList map from data/system directory. */
[iotdb] 01/02: disable compaction on the remote
Posted by he...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
heiming pushed a commit to branch tiered_storage
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit fd6d0953e5a338b7df1a41c3d86de26a304cb5d3
Author: HeimingZ <zh...@qq.com>
AuthorDate: Wed May 24 16:42:25 2023 +0800
disable compaction on the remote
---
.../main/java/org/apache/iotdb/db/conf/directories/TierManager.java | 3 +++
.../engine/compaction/selector/impl/SizeTieredCompactionSelector.java | 4 +++-
.../compaction/selector/utils/CrossSpaceCompactionCandidate.java | 2 ++
.../java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java | 4 ++++
4 files changed, 12 insertions(+), 1 deletion(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/directories/TierManager.java b/server/src/main/java/org/apache/iotdb/db/conf/directories/TierManager.java
index 734d7a5decf..81281a46ca0 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/directories/TierManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/directories/TierManager.java
@@ -101,6 +101,9 @@ public class TierManager {
for (int i = 0; i < tierDirs.length; ++i) {
for (int j = 0; j < tierDirs[i].length; ++j) {
if (tierDirs[i][j].equals(OBJECT_STORAGE_DIR)) {
+ if (i != tierDirs.length - 1) {
+ logger.error("Object Storage can only exist on the last tier.");
+ }
tierDirs[i][j] =
FSUtils.getOSDefaultPath(config.getObjectStorageBucket(), config.getDataNodeId());
} else if (FSUtils.isLocal(tierDirs[i][j])) {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/impl/SizeTieredCompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/impl/SizeTieredCompactionSelector.java
index 105e03637c4..c3dddf47402 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/impl/SizeTieredCompactionSelector.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/impl/SizeTieredCompactionSelector.java
@@ -114,7 +114,9 @@ public class SizeTieredCompactionSelector
selectedFileSize = 0L;
continue;
}
- if (currentFile.getStatus() != TsFileResourceStatus.NORMAL || currentFile.isMigrating()) {
+ if (currentFile.getStatus() != TsFileResourceStatus.NORMAL
+ || currentFile.onRemote()
+ || currentFile.isMigrating()) {
selectedFileList.clear();
selectedFileSize = 0L;
continue;
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/utils/CrossSpaceCompactionCandidate.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/utils/CrossSpaceCompactionCandidate.java
index 42a91acc792..b70cf0f0b0b 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/utils/CrossSpaceCompactionCandidate.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/utils/CrossSpaceCompactionCandidate.java
@@ -143,6 +143,7 @@ public class CrossSpaceCompactionCandidate {
List<TsFileResourceCandidate> ret = new ArrayList<>();
for (TsFileResource resource : unseqResources) {
if (resource.getStatus() != TsFileResourceStatus.NORMAL
+ || resource.onRemote()
|| resource.isMigrating()
|| !resource.getTsFile().exists()) {
break;
@@ -201,6 +202,7 @@ public class CrossSpaceCompactionCandidate {
// the status of file may be changed after the task is submitted to queue
this.isValidCandidate =
tsFileResource.getStatus() == TsFileResourceStatus.NORMAL
+ && !tsFileResource.onRemote()
&& !tsFileResource.isMigrating()
&& tsFileResource.getTsFile().exists();
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
index fe1b9c102d8..859d6b201ff 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
@@ -658,6 +658,10 @@ public class TsFileResource {
this.isMigrating = isMigrating;
}
+ public boolean onRemote() {
+ return !file.exists();
+ }
+
public void setStatus(TsFileResourceStatus status) {
switch (status) {
case NORMAL: