You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2023/05/31 02:55:48 UTC
[iotdb] branch tiered_storage updated: fix UT
This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch tiered_storage
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/tiered_storage by this push:
new 33866ee7a27 fix UT
33866ee7a27 is described below
commit 33866ee7a27b90876a842abdc163f9f871ed707f
Author: Jinrui.Zhang <xi...@gmail.com>
AuthorDate: Tue May 30 20:07:54 2023 +0800
fix UT
---
.../util/TSFileConfigUtilCompletenessTest.java | 8 ++-
.../apache/iotdb/commons/conf/CommonConfig.java | 2 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 5 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 2 +-
.../iotdb/db/conf/directories/TierManager.java | 27 +++++-----
.../execute/recover/CompactionRecoverManager.java | 1 -
.../execute/recover/CompactionRecoverTask.java | 1 -
.../iotdb/db/engine/storagegroup/DataRegion.java | 2 +-
.../tieredstorage/config/TieredStorageConfig.java | 36 -------------
.../exception/TsFileMigrationException.java | 41 ---------------
.../pipeline/TieredStorageTaskExecutor.java | 23 --------
.../pipeline/TieredStorageTaskQueue.java | 23 --------
.../pipeline/TieredStorageTaskScheduler.java | 49 -----------------
.../tieredstorage/task/TsFileMigrationTask.java | 61 ----------------------
.../task/TsFileMigrationTaskType.java | 25 ---------
.../task/TsFileMigrationTriggerType.java | 25 ---------
.../java/org/apache/iotdb/db/service/DataNode.java | 9 ++--
.../iotdb/db/service/metrics/SystemMetrics.java | 1 -
.../db/engine/compaction/TsFileIdentifierUT.java | 16 +++---
.../recover/SizeTieredCompactionRecoverTest.java | 20 +++----
.../engine/migration/LocalMigrationTaskTest.java | 27 ----------
.../engine/migration/RemoteMigrationTaskTest.java | 27 ----------
.../db/engine/snapshot/IoTDBSnapshotTest.java | 2 +-
.../apache/iotdb/tsfile/utils/FilePathUtils.java | 4 ++
24 files changed, 55 insertions(+), 382 deletions(-)
diff --git a/iotdb-connector/flink-tsfile-connector/src/test/java/org/apache/iotdb/flink/util/TSFileConfigUtilCompletenessTest.java b/iotdb-connector/flink-tsfile-connector/src/test/java/org/apache/iotdb/flink/util/TSFileConfigUtilCompletenessTest.java
index c46c700f2c2..c209271f82c 100644
--- a/iotdb-connector/flink-tsfile-connector/src/test/java/org/apache/iotdb/flink/util/TSFileConfigUtilCompletenessTest.java
+++ b/iotdb-connector/flink-tsfile-connector/src/test/java/org/apache/iotdb/flink/util/TSFileConfigUtilCompletenessTest.java
@@ -78,7 +78,13 @@ public class TSFileConfigUtilCompletenessTest {
"setMaxTsBlockLineNumber",
"setMaxTsBlockSizeInBytes",
"setPatternMatchingThreshold",
- "setCustomizedProperties"
+ "setCustomizedProperties",
+ "setObjectStorageTsFileInput",
+ "setObjectStorageTsFileOutput",
+ "setHdfsFile",
+ "setHdfsTsFileInput",
+ "setHdfsTsFileOutput",
+ "setObjectStorageFile"
};
Set<String> addedSetters = new HashSet<>();
Collections.addAll(addedSetters, setters);
diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java b/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
index 1caa7365225..a44810612f2 100644
--- a/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
+++ b/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
@@ -92,7 +92,7 @@ public class CommonConfig {
* Notice: if this property is changed, previous created database which are not set TTL will also
* be affected. Unit: millisecond
*/
- private long[] tierTTLInMs = {2 * 24 * 60 * 60 * 1000L, Long.MAX_VALUE};
+ private long[] tierTTLInMs = {Long.MAX_VALUE};
/** Thrift socket and connection timeout between data node and config node. */
private int connectionTimeoutInMS = (int) TimeUnit.SECONDS.toMillis(20);
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 56335e03c2e..ca77ed0c9ae 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -300,8 +300,7 @@ public class IoTDBConfig {
/** Tiered data directories. It can be settled as dataDirs = {{"data1"}, {"data2", "data3"}}; */
private String[][] tierDataDirs = {
- {IoTDBConstant.DEFAULT_BASE_DIR + File.separator + IoTDBConstant.DATA_FOLDER_NAME},
- {IoTDBConstant.OBJECT_STORAGE_DIR}
+ {IoTDBConstant.DEFAULT_BASE_DIR + File.separator + IoTDBConstant.DATA_FOLDER_NAME}
};
private String loadTsFileDir =
@@ -530,7 +529,7 @@ public class IoTDBConfig {
private int candidateCompactionTaskQueueSize = 50;
/** whether to cache meta data(ChunkMetaData and TsFileMetaData) or not. */
- private boolean metaDataCacheEnable = false;
+ private boolean metaDataCacheEnable = true;
/** Memory allocated for bloomFilter cache in read process */
private long allocateMemoryForBloomFilterCache = allocateMemoryForRead / 1001;
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 3c67feff6c7..fd547979277 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -1083,7 +1083,7 @@ public class IoTDBDescriptor {
// author cache
loadAuthorCache(properties);
- // object storage
+ // migration
loadMigrationProps(properties);
// object storage
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/directories/TierManager.java b/server/src/main/java/org/apache/iotdb/db/conf/directories/TierManager.java
index db69bdb267d..4c69ad2b11c 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/directories/TierManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/directories/TierManager.java
@@ -70,13 +70,10 @@ public class TierManager {
/** total space of each tier, Long.MAX_VALUE when one tier contains remote storage */
private long[] tierDiskTotalSpace;
- private TierManager() {}
-
- public synchronized void resetFolders() {
- if (config.getDataNodeId() == -1) {
- return;
- }
-
+ private TierManager() {
+ initFolders();
+ }
+ public synchronized void initFolders() {
try {
String strategyName = Class.forName(config.getMultiDirStrategyClassName()).getSimpleName();
if (strategyName.equals(MaxDiskUsableSpaceFirstStrategy.class.getSimpleName())) {
@@ -91,11 +88,6 @@ public class TierManager {
"Can't find strategy {} for mult-directories.", config.getMultiDirStrategyClassName(), e);
}
- seqTiers.clear();
- unSeqTiers.clear();
- seqDir2TierLevel.clear();
- unSeqDir2TierLevel.clear();
-
config.updatePath();
String[][] tierDirs = config.getTierDataDirs();
for (int i = 0; i < tierDirs.length; ++i) {
@@ -172,6 +164,15 @@ public class TierManager {
tierDiskTotalSpace = getTierDiskSpace(DiskSpaceType.TOTAL);
}
+ public synchronized void resetFolders() {
+ seqTiers.clear();
+ unSeqTiers.clear();
+ seqDir2TierLevel.clear();
+ unSeqDir2TierLevel.clear();
+
+ initFolders();
+ }
+
private void mkDataDirs(List<String> folders) {
for (String folder : folders) {
File file = FSFactoryProducer.getFSFactory().getFile(folder);
@@ -250,7 +251,7 @@ public class TierManager {
return entry.getValue();
}
}
- throw new RuntimeException(String.format("%s is not a legal TsFile path", file));
+ return 0;
}
public long[] getTierDiskTotalSpace() {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/recover/CompactionRecoverManager.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/recover/CompactionRecoverManager.java
index 1ad685aac44..390ddf2b512 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/recover/CompactionRecoverManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/recover/CompactionRecoverManager.java
@@ -67,7 +67,6 @@ public class CompactionRecoverManager {
private void recoverCompaction(boolean isInnerSpace, boolean isLogSequence) {
List<String> dirs;
- // TODO(zhm) 当前远端存储介质不开启合并，无需恢复
if (isLogSequence) {
dirs = TierManager.getInstance().getAllLocalSequenceFileFolders();
} else {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/recover/CompactionRecoverTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/recover/CompactionRecoverTask.java
index feeeddb2f13..b75940a7970 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/recover/CompactionRecoverTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/recover/CompactionRecoverTask.java
@@ -248,7 +248,6 @@ public class CompactionRecoverTask {
* the file is not found, it will return null.
*/
private File getFileFromDataDirs(String filePath) {
- // TODO(zhm) 当前远端存储介质不开启合并，无需恢复
String[] dataDirs = IoTDBDescriptor.getInstance().getConfig().getLocalDataDirs();
for (String dataDir : dataDirs) {
File f = new File(dataDir, filePath);
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
index 7f3dfb11462..812299f07ac 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
@@ -762,7 +762,7 @@ public class DataRegion implements IDataRegionForQuery {
checkTsFileTime(f, currentTime);
ret.add(new TsFileResource(f));
}
-
+ upgradeFiles.sort(this::compareFileName);
List<TsFileResource> upgradeRet = new ArrayList<>();
for (File f : upgradeFiles) {
checkTsFileTime(f, currentTime);
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/config/TieredStorageConfig.java b/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/config/TieredStorageConfig.java
deleted file mode 100644
index 4db77421e3f..00000000000
--- a/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/config/TieredStorageConfig.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.engine.tieredstorage.config;
-
-import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
-import org.apache.iotdb.db.engine.tieredstorage.task.TsFileMigrationTriggerType;
-
-/**
- * // (jinrui) 该类为一个单例类，用来在运行时提供多级存储的相关配置（可能需要定义更多的类来实现它），其功能为： 1. 启动时，根据配置文件处理多级存储的配置信息 2.
- * 将配置信息存储在该单例类的对象中，并提供对外的访问接口 3. 提供判断一个 TsFile 是否需要迁移的方法，并返回迁移的原因
- */
-public class TieredStorageConfig {
-
- // (jinrui) 传入参数可能需要更多信息，以辅助判断，尽可能地使用内存中的信息进行该判断，避免磁盘读取
- // 如果在实现过程中需要从磁盘中读取信息，需要标注出来进行讨论
- public TsFileMigrationTriggerType needMoveToNextTier(TsFileResource tsFileResource) {
- return null;
- }
-}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/exception/TsFileMigrationException.java b/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/exception/TsFileMigrationException.java
deleted file mode 100644
index 88b7204106d..00000000000
--- a/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/exception/TsFileMigrationException.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.engine.tieredstorage.exception;
-
-public class TsFileMigrationException extends Exception {
- public TsFileMigrationException() {}
-
- public TsFileMigrationException(String message) {
- super(message);
- }
-
- public TsFileMigrationException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public TsFileMigrationException(Throwable cause) {
- super(cause);
- }
-
- public TsFileMigrationException(
- String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
- super(message, cause, enableSuppression, writableStackTrace);
- }
-}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/pipeline/TieredStorageTaskExecutor.java b/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/pipeline/TieredStorageTaskExecutor.java
deleted file mode 100644
index f720ab1debd..00000000000
--- a/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/pipeline/TieredStorageTaskExecutor.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.engine.tieredstorage.pipeline;
-
-/** // (jinrui) 设定一个并发执行的线程数，从 queue 中取出 task，并执行 */
-public class TieredStorageTaskExecutor {}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/pipeline/TieredStorageTaskQueue.java b/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/pipeline/TieredStorageTaskQueue.java
deleted file mode 100644
index 52c32605777..00000000000
--- a/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/pipeline/TieredStorageTaskQueue.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.engine.tieredstorage.pipeline;
-
-/** // (jinrui) 一个单例的、线程安全的队列，用来储存内存中的 MigrationTask 任务 */
-public class TieredStorageTaskQueue {}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/pipeline/TieredStorageTaskScheduler.java b/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/pipeline/TieredStorageTaskScheduler.java
deleted file mode 100644
index 8be573196f1..00000000000
--- a/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/pipeline/TieredStorageTaskScheduler.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.engine.tieredstorage.pipeline;
-
-import org.apache.iotdb.db.engine.storagegroup.TsFileManager;
-
-/**
- * // (jinrui) 取一个什么名字好呢？ 负责周期性地检查是否需要提交 TsFileMigrationTask，初步计划每个 DataRegion
- * 有一个这样的实例，定期执行；DataRegion 之间不混用
- */
-public class TieredStorageTaskScheduler {
-
- private String dataRegionId;
- // (jinrui)
- // 获取一个 TsFileManager 的引用，从而拿到对应的文件信息
- // 相应的，系统重启的时候，在温数据区加载文件时，也要做相应的改动（读对应的 resource 文件）
- private TsFileManager tsFileManager;
-
- public TieredStorageTaskScheduler(String dataRegionId) {
- this.dataRegionId = dataRegionId;
- }
-
- public void schedule() {}
-
- private void checkTsFileTTL() {}
-
- // (jinrui)
- // 这里涉及到了一个问题，就是当磁盘超过阈值时，迁移文件时候怎么进行选择，有两个倾向
- // 1. 从一个 DataRegion 迁移，直至预期空间小于阈值为止
- // 2. 每个 DataRegion 迁移最老的 N 个文件，然后等待下一轮再进行检查
- private void checkDiskThreshold() {}
-}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/task/TsFileMigrationTask.java b/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/task/TsFileMigrationTask.java
deleted file mode 100644
index b717aaaa442..00000000000
--- a/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/task/TsFileMigrationTask.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.engine.tieredstorage.task;
-
-import org.apache.iotdb.db.engine.tieredstorage.exception.TsFileMigrationException;
-
-import java.util.List;
-
-public abstract class TsFileMigrationTask {
- // block size for each write when doing migration to avoid memory overflow
- private static final int BLOCK_SIZE_PER_WRITE = 100 * 1024 * 1024;
- private String sourceFilePath;
- private String targetFilePath;
- private TsFileMigrationTriggerType triggerType;
- private TsFileMigrationTaskType taskType;
-
- // (jinrui) 文件的输入都是从本地读，所以大概率是用同一个文件读取类即可，不过也可以抽象一个读取的接口
-
- // (jinrui) 定义一个输出文件的接口，不同的 MigrationTask 提供不同的输出文件接口的实现，需要确认当前写 TsFile 是用的什么接口
- // private XXFileWriter writer;
-
- public TsFileMigrationTask(
- String sourceFilePath,
- String targetFilePath,
- TsFileMigrationTriggerType triggerType,
- TsFileMigrationTaskType taskType) {
- this.sourceFilePath = sourceFilePath;
- this.targetFilePath = targetFilePath;
- this.triggerType = triggerType;
- this.taskType = taskType;
- }
-
- /**
- * // (jinrui) 该方法实现迁移一个 TsFile 的完整步骤： 1. 根据 getSourceFileList() 获取到所有要迁移的文件 2. 按照文件列表中的顺序，使用对应的
- * reader 和 writer 对文件进行读写 3. 如果迁移过程中遇到了异常，则执行 cleanupWhenException()
- */
- public void doMigration() throws TsFileMigrationException {}
-
- // (jinrui) 不同的迁移任务它涉及的文件不同，比如 local_to_local 不仅需要迁移 source file，还需要迁移对应的 mod、resource 文件
- protected abstract List<String> getSourceFileList();
-
- /** (jinrui) 迁移任务如果遇到异常，则需要进行相应的清理工作。可以考虑是否需要将该方式实现成一个统一的方法 */
- protected abstract void cleanupWhenException() throws TsFileMigrationException;
-}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/task/TsFileMigrationTaskType.java b/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/task/TsFileMigrationTaskType.java
deleted file mode 100644
index 3cb62093de4..00000000000
--- a/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/task/TsFileMigrationTaskType.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.engine.tieredstorage.task;
-
-public enum TsFileMigrationTaskType {
- FROM_LOCAL_TO_LOCAL,
- FROM_LOCAL_TO_S3,
-}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/task/TsFileMigrationTriggerType.java b/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/task/TsFileMigrationTriggerType.java
deleted file mode 100644
index e393b2db432..00000000000
--- a/server/src/main/java/org/apache/iotdb/db/engine/tieredstorage/task/TsFileMigrationTriggerType.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.engine.tieredstorage.task;
-
-public enum TsFileMigrationTriggerType {
- TTL,
- DISK_FULL,
-}
diff --git a/server/src/main/java/org/apache/iotdb/db/service/DataNode.java b/server/src/main/java/org/apache/iotdb/db/service/DataNode.java
index e173013f900..a0cbe3823a6 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/DataNode.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/DataNode.java
@@ -171,7 +171,10 @@ public class DataNode implements DataNodeMBean {
// Send restart request of this DataNode
sendRestartRequestToConfigNode();
}
-
+ // TierManager need DataNodeId to do some operations so the reset method need to be invoked
+ // after DataNode adding
+ TierManager.getInstance().resetFolders();
+ configOSStorage(config.getDataNodeId());
// Active DataNode
active();
@@ -393,7 +396,6 @@ public class DataNode implements DataNodeMBean {
/* Store runtime configurations when register success */
int dataNodeID = dataNodeRegisterResp.getDataNodeId();
config.setDataNodeId(dataNodeID);
- configOSStorage(dataNodeID);
IoTDBStartCheck.getInstance()
.serializeClusterNameAndDataNodeId(config.getClusterName(), dataNodeID);
@@ -409,9 +411,7 @@ public class DataNode implements DataNodeMBean {
}
private void configOSStorage(int dataNodeID) {
- TierManager.getInstance().resetFolders();
FSFactoryProducer.setFileInputFactory(new HybridFileInputFactoryDecorator(dataNodeID));
- // recover OS cache
}
private void sendRestartRequestToConfigNode() throws StartupException {
@@ -459,7 +459,6 @@ public class DataNode implements DataNodeMBean {
/* Store runtime configurations when restart request is accepted */
storeRuntimeConfigurations(
dataNodeRestartResp.getConfigNodeList(), dataNodeRestartResp.getRuntimeConfiguration());
- configOSStorage(config.getDataNodeId());
logger.info("Restart request to cluster: {} is accepted.", config.getClusterName());
} else {
/* Throw exception when restart is rejected */
diff --git a/server/src/main/java/org/apache/iotdb/db/service/metrics/SystemMetrics.java b/server/src/main/java/org/apache/iotdb/db/service/metrics/SystemMetrics.java
index cbf545adaf1..b2bdc7e2dbd 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/metrics/SystemMetrics.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/metrics/SystemMetrics.java
@@ -187,7 +187,6 @@ public class SystemMetrics implements IMetricSet {
}
private void collectSystemDiskInfo(AbstractMetricService metricService) {
- // TODO(zhm) 远端存储介质不支持 file store
String[] dataDirs = IoTDBDescriptor.getInstance().getConfig().getLocalDataDirs();
for (String dataDir : dataDirs) {
Path path = Paths.get(dataDir);
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/TsFileIdentifierUT.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/TsFileIdentifierUT.java
index b05c34d0d27..5fa8328a2af 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/TsFileIdentifierUT.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/TsFileIdentifierUT.java
@@ -156,14 +156,18 @@ public class TsFileIdentifierUT {
@Test
public void testGetInfoFromFileFromMultiDirs() throws Exception {
+
IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
- String[] originDataDirs = config.getDataDirs();
+ String[][] originDataDirs = config.getTierDataDirs();
Class configClass = config.getClass();
- Field dataDirField = configClass.getDeclaredField("dataDirs");
- dataDirField.setAccessible(true);
- dataDirField.set(
+ Field dataDirsField = configClass.getDeclaredField("tierDataDirs");
+ dataDirsField.setAccessible(true);
+ dataDirsField.set(
config,
- new String[] {"target" + File.separator + "data1", "target" + File.separator + "data2"});
+ new String[][] {
+ {"target" + File.separator + "data1", "target" + File.separator + "data2"}
+ });
+
String filePath =
"sequence"
+ File.separator
@@ -187,7 +191,7 @@ public class TsFileIdentifierUT {
Assert.assertTrue(testFile.createNewFile());
Assert.assertTrue(Files.isSameFile(testFile.toPath(), info.getFileFromDataDirs().toPath()));
} finally {
- dataDirField.set(config, originDataDirs);
+ dataDirsField.set(config, originDataDirs);
Files.deleteIfExists(testFile.toPath());
FileUtils.deleteDirectory(new File("target" + File.separator + "data2"));
}
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverTest.java
index c6e62c33fcb..d195c6af061 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverTest.java
@@ -119,14 +119,14 @@ public class SizeTieredCompactionRecoverTest {
static final MeasurementSchema[] schemas = new MeasurementSchema[fullPaths.length];
static String logFilePath =
TestConstant.BASE_OUTPUT_PATH + File.separator + "test-compaction.compaction.log";
- static String[] originDataDirs = null;
- static String[] testDataDirs = new String[] {TestConstant.BASE_OUTPUT_PATH + "data"};
+ static String[][] originDataDirs = null;
+ static String[][] testDataDirs = new String[][] {{TestConstant.BASE_OUTPUT_PATH + "data"}};
static IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
@Before
public void setUp() throws Exception {
CompactionTaskManager.getInstance().start();
- originDataDirs = config.getDataDirs();
+ originDataDirs = config.getTierDataDirs();
setDataDirs(testDataDirs);
if (!new File(SEQ_FILE_DIR).exists()) {
Assert.assertTrue(new File(SEQ_FILE_DIR).mkdirs());
@@ -142,7 +142,7 @@ public class SizeTieredCompactionRecoverTest {
new CompactionConfigRestorer().restoreCompactionConfig();
CompactionTaskManager.getInstance().stop();
setDataDirs(originDataDirs);
- File dataDir = new File(testDataDirs[0]);
+ File dataDir = new File(testDataDirs[0][0]);
if (dataDir.exists()) {
FileUtils.forceDelete(dataDir);
}
@@ -166,9 +166,9 @@ public class SizeTieredCompactionRecoverTest {
}
}
- public void setDataDirs(String[] dataDirs) throws Exception {
+ public void setDataDirs(String[][] dataDirs) throws Exception {
Class configClass = config.getClass();
- Field dataDirsField = configClass.getDeclaredField("dataDirs");
+ Field dataDirsField = configClass.getDeclaredField("tierDataDirs");
dataDirsField.setAccessible(true);
dataDirsField.set(config, dataDirs);
}
@@ -470,7 +470,7 @@ public class SizeTieredCompactionRecoverTest {
FileUtils.moveDirectory(
new File(TestConstant.BASE_OUTPUT_PATH + File.separator + "data"),
new File(TestConstant.BASE_OUTPUT_PATH + File.separator + "data1"));
- setDataDirs(new String[] {TestConstant.BASE_OUTPUT_PATH + File.separator + "data1"});
+ setDataDirs(new String[][] {{TestConstant.BASE_OUTPUT_PATH + File.separator + "data1"}});
CompactionRecoverTask recoverTask =
new CompactionRecoverTask(
COMPACTION_TEST_SG, "0", tsFileManager, new File(logFilePath), true);
@@ -573,7 +573,7 @@ public class SizeTieredCompactionRecoverTest {
FileUtils.moveDirectory(
new File(TestConstant.BASE_OUTPUT_PATH + File.separator + "data"),
new File(TestConstant.BASE_OUTPUT_PATH + File.separator + "data1"));
- setDataDirs(new String[] {TestConstant.BASE_OUTPUT_PATH + File.separator + "data1"});
+ setDataDirs(new String[][] {{TestConstant.BASE_OUTPUT_PATH + File.separator + "data1"}});
CompactionRecoverTask recoverTask =
new CompactionRecoverTask(
COMPACTION_TEST_SG, "0", tsFileManager, new File(logFilePath), true);
@@ -667,7 +667,7 @@ public class SizeTieredCompactionRecoverTest {
FileUtils.moveDirectory(
new File(TestConstant.BASE_OUTPUT_PATH + File.separator + "data"),
new File(TestConstant.BASE_OUTPUT_PATH + File.separator + "data1"));
- setDataDirs(new String[] {TestConstant.BASE_OUTPUT_PATH + File.separator + "data1"});
+ setDataDirs(new String[][] {{TestConstant.BASE_OUTPUT_PATH + File.separator + "data1"}});
CompactionRecoverTask recoverTask =
new CompactionRecoverTask(
COMPACTION_TEST_SG, "0", tsFileManager, new File(logFilePath), true);
@@ -770,7 +770,7 @@ public class SizeTieredCompactionRecoverTest {
FileUtils.moveDirectory(
new File(TestConstant.BASE_OUTPUT_PATH + File.separator + "data"),
new File(TestConstant.BASE_OUTPUT_PATH + File.separator + "data1"));
- setDataDirs(new String[] {TestConstant.BASE_OUTPUT_PATH + File.separator + "data1"});
+ setDataDirs(new String[][] {{TestConstant.BASE_OUTPUT_PATH + File.separator + "data1"}});
CompactionRecoverTask recoverTask =
new CompactionRecoverTask(
COMPACTION_TEST_SG, "0", tsFileManager, new File(logFilePath), true);
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/migration/LocalMigrationTaskTest.java b/server/src/test/java/org/apache/iotdb/db/engine/migration/LocalMigrationTaskTest.java
deleted file mode 100644
index fac7ec3eb61..00000000000
--- a/server/src/test/java/org/apache/iotdb/db/engine/migration/LocalMigrationTaskTest.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.engine.migration;
-
-import org.junit.Test;
-
-public class LocalMigrationTaskTest {
-
- @Test
- public void migrate() {}
-}
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/migration/RemoteMigrationTaskTest.java b/server/src/test/java/org/apache/iotdb/db/engine/migration/RemoteMigrationTaskTest.java
deleted file mode 100644
index 0569924ae26..00000000000
--- a/server/src/test/java/org/apache/iotdb/db/engine/migration/RemoteMigrationTaskTest.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.engine.migration;
-
-import org.junit.Test;
-
-public class RemoteMigrationTaskTest {
-
- @Test
- public void migrate() {}
-}
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/snapshot/IoTDBSnapshotTest.java b/server/src/test/java/org/apache/iotdb/db/engine/snapshot/IoTDBSnapshotTest.java
index 1bb3b22913a..74e3119d281 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/snapshot/IoTDBSnapshotTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/snapshot/IoTDBSnapshotTest.java
@@ -66,7 +66,7 @@ public class IoTDBSnapshotTest {
List<TsFileResource> resources = new ArrayList<>();
for (int i = 0; i < 100; i++) {
String filePath =
- testDataDirs[i % 3]
+ testDataDirs[0][i % 3]
+ File.separator
+ "sequence"
+ File.separator
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/FilePathUtils.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/FilePathUtils.java
index d30e77f2632..df7ac44ef8e 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/FilePathUtils.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/FilePathUtils.java
@@ -67,6 +67,10 @@ public class FilePathUtils {
public static boolean isSequence(String tsFileAbsolutePath) {
String[] pathSegments = splitTsFilePath(tsFileAbsolutePath);
+ // If path is not a regular IoTDB TsFile path, then process it as an unsequence file
+ if (pathSegments.length < 5) {
+ return false;
+ }
return pathSegments[pathSegments.length - 5].equals("sequence");
}