You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2023/05/08 16:43:27 UTC
[iotdb] branch tiered_storage updated: add draft code structure for migration part
This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch tiered_storage
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/tiered_storage by this push:
new 9f5b8f9764 add draft code structure for migration part
9f5b8f9764 is described below
commit 9f5b8f976487e80417939183a1850f4f68ce1928
Author: Jinrui.Zhang <xi...@gmail.com>
AuthorDate: Tue May 9 00:43:13 2023 +0800
add draft code structure for migration part
---
.../tieredstorage/config/TieredStorageConfig.java | 39 +++++++++++++
.../exception/TsFileMigrationException.java | 23 ++++++++
.../pipeline/TieredStorageTaskExecutor.java | 27 +++++++++
.../pipeline/TieredStorageTaskQueue.java | 27 +++++++++
.../pipeline/TieredStorageTaskScheduler.java | 56 +++++++++++++++++++
.../tieredstorage/task/TsFileMigrationTask.java | 64 ++++++++++++++++++++++
.../task/TsFileMigrationTaskType.java | 25 +++++++++
.../task/TsFileMigrationTriggerType.java | 25 +++++++++
8 files changed, 286 insertions(+)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/config/TieredStorageConfig.java b/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/config/TieredStorageConfig.java
new file mode 100644
index 0000000000..cde108f79d
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/config/TieredStorageConfig.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.tieredstorage.config;
+
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.engine.tieredstorage.task.TsFileMigrationTriggerType;
+
+/**
+ * // (jinrui)
+ * 该类为一个单例类，用来在运行时提供多级存储的相关配置（可能需要定义更多的类来实现它），其功能为：
+ * 1. 启动时，根据配置文件处理多级存储的配置信息
+ * 2. 将配置信息存储在该单例类的对象中，并提供对外的访问接口
+ * 3. 提供判断一个 TsFile 是否需要迁移的方法，并返回迁移的原因
+ */
+public class TieredStorageConfig {
+
+ // (jinrui) 传入参数可能需要更多信息，以辅助判断，尽可能地使用内存中的信息进行该判断，避免磁盘读取
+ // 如果在实现过程中需要从磁盘中读取信息，需要标注出来进行讨论
+ public TsFileMigrationTriggerType needMoveToNextTier(TsFileResource tsFileResource) {
+ return null;
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/exception/TsFileMigrationException.java b/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/exception/TsFileMigrationException.java
new file mode 100644
index 0000000000..e5c95908fc
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/exception/TsFileMigrationException.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.tieredstorage.exception;
+
+public class TsFileMigrationException extends Exception{
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/pipeline/TieredStorageTaskExecutor.java b/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/pipeline/TieredStorageTaskExecutor.java
new file mode 100644
index 0000000000..66711b811f
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/pipeline/TieredStorageTaskExecutor.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.tieredstorage.pipeline;
+
+/**
+ * // (jinrui)
+ * 设定一个并发执行的线程数，从 queue 中取出 task，并执行
+ */
+public class TieredStorageTaskExecutor {
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/pipeline/TieredStorageTaskQueue.java b/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/pipeline/TieredStorageTaskQueue.java
new file mode 100644
index 0000000000..8447fd021e
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/pipeline/TieredStorageTaskQueue.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.tieredstorage.pipeline;
+
+/**
+ * // (jinrui)
+ * 一个单例的、线程安全的队列，用来储存内存中的 MigrationTask 任务
+ */
+public class TieredStorageTaskQueue {
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/pipeline/TieredStorageTaskScheduler.java b/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/pipeline/TieredStorageTaskScheduler.java
new file mode 100644
index 0000000000..1590f053eb
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/pipeline/TieredStorageTaskScheduler.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.tieredstorage.pipeline;
+
+import org.apache.iotdb.db.engine.storagegroup.TsFileManager;
+
+/**
+ * // (jinrui)
+ * 取一个什么名字好呢？
+ * 负责周期性地检查是否需要提交 TsFileMigrationTask，初步计划每个 DataRegion 有一个这样的实例，定期执行；DataRegion 之间不混用
+ */
+public class TieredStorageTaskScheduler {
+
+ private String dataRegionId;
+ // (jinrui)
+ // 获取一个 TsFileManager 的引用，从而拿到对应的文件信息
+ // 相应的，系统重启的时候，在温数据区加载文件时，也要做相应的改动（读对应的 resource 文件）
+ private TsFileManager tsFileManager;
+
+ public TieredStorageTaskScheduler(String dataRegionId) {
+ this.dataRegionId = dataRegionId;
+ }
+
+ public void schedule() {
+
+ }
+
+ private void checkTsFileTTL() {
+
+ }
+
+ // (jinrui)
+ // 这里涉及到了一个问题，就是当磁盘超过阈值时，迁移文件时候怎么进行选择，有两个倾向
+ // 1. 从一个 DataRegion 迁移，直至预期空间小于阈值为止
+ // 2. 每个 DataRegion 迁移最老的 N 个文件，然后等待下一轮再进行检查
+ private void checkDiskThreshold() {
+
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/task/TsFileMigrationTask.java b/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/task/TsFileMigrationTask.java
new file mode 100644
index 0000000000..6d2194db57
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/task/TsFileMigrationTask.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.tieredstorage.task;
+
+import org.apache.iotdb.db.engine.tieredstorage.exception.TsFileMigrationException;
+
+import java.util.List;
+
+public abstract class TsFileMigrationTask {
+ // block size for each write when doing migration to avoid memory overflow
+ private static final int BLOCK_SIZE_PER_WRITE = 100 * 1024 * 1024;
+ private String sourceFilePath;
+ private String targetFilePath;
+ private TsFileMigrationTriggerType triggerType;
+ private TsFileMigrationTaskType taskType;
+
+ // (jinrui) 文件的输入都是从本地读，所以大概率是用同一个文件读取类即可，不过也可以抽象一个读取的接口
+
+ // (jinrui) 定义一个输出文件的接口，不同的 MigrationTask 提供不同的输出文件接口的实现，需要确认当前写 TsFile 是用的什么接口
+ // private XXFileWriter writer;
+
+ public TsFileMigrationTask(String sourceFilePath, String targetFilePath, TsFileMigrationTriggerType triggerType, TsFileMigrationTaskType taskType) {
+ this.sourceFilePath = sourceFilePath;
+ this.targetFilePath = targetFilePath;
+ this.triggerType = triggerType;
+ this.taskType = taskType;
+ }
+
+ /**
+ * // (jinrui)
+ * 该方法实现迁移一个 TsFile 的完整步骤：
+ * 1. 根据 getSourceFileList() 获取到所有要迁移的文件
+ * 2. 按照文件列表中的顺序，使用对应的 reader 和 writer 对文件进行读写
+ * 3. 如果迁移过程中遇到了异常，则执行 cleanupWhenException()
+ */
+ public void doMigration() throws TsFileMigrationException {
+
+ }
+
+ // (jinrui) 不同的迁移任务它涉及的文件不同，比如 local_to_local 不仅需要迁移 source file，还需要迁移对应的 mod、resource 文件
+ protected abstract List<String> getSourceFileList();
+
+ /**
+ * (jinrui) 迁移任务如果遇到异常，则需要进行相应的清理工作。可以考虑是否需要将该方式实现成一个统一的方法
+ */
+ protected abstract void cleanupWhenException() throws TsFileMigrationException;
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/task/TsFileMigrationTaskType.java b/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/task/TsFileMigrationTaskType.java
new file mode 100644
index 0000000000..3cb62093de
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/task/TsFileMigrationTaskType.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.tieredstorage.task;
+
+public enum TsFileMigrationTaskType {
+ FROM_LOCAL_TO_LOCAL,
+ FROM_LOCAL_TO_S3,
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/task/TsFileMigrationTriggerType.java b/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/task/TsFileMigrationTriggerType.java
new file mode 100644
index 0000000000..e393b2db43
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/task/TsFileMigrationTriggerType.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.tieredstorage.task;
+
+public enum TsFileMigrationTriggerType {
+ TTL,
+ DISK_FULL,
+}