You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2019/09/17 10:48:35 UTC
[incubator-iotdb] branch master updated: [IOTDB-219] Add config of
tsfile_storage_fs in iotdb-engine.properties (#395)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 0a4df04 [IOTDB-219] Add config of tsfile_storage_fs in iotdb-engine.properties (#395)
0a4df04 is described below
commit 0a4df04ee589c91d312dd5d5ba60ad1c92a10f5c
Author: Zesong Sun <15...@smail.nju.edu.cn>
AuthorDate: Tue Sep 17 18:48:31 2019 +0800
[IOTDB-219] Add config of tsfile_storage_fs in iotdb-engine.properties (#395)
* Add configuration ip and port
* Add config of tsfile_storage_fs in iotdb-engine.properties
---
.../4-Deployment and Management/2-Configuration.md | 27 ++++++++++++
.../4-Deployment and Management/2-Configuration.md | 27 ++++++++++++
.../resources/conf/iotdb-engine.properties | 10 ++++-
.../iotdb/db/auth/role/LocalFileRoleAccessor.java | 19 ++++----
.../iotdb/db/auth/user/LocalFileUserAccessor.java | 38 ++++++++--------
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 51 +++++++++++++++++++---
.../org/apache/iotdb/db/conf/IoTDBConfigCheck.java | 8 ++--
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 11 ++++-
.../iotdb/db/conf/adapter/CompressionRatio.java | 7 +--
.../org/apache/iotdb/db/engine/StorageEngine.java | 4 +-
.../apache/iotdb/db/engine/fileSystem/FSType.java | 24 ----------
.../{FileFactory.java => SystemFileFactory.java} | 16 ++++---
.../iotdb/db/engine/merge/recover/LogAnalyzer.java | 8 ++--
.../engine/storagegroup/StorageGroupProcessor.java | 6 +--
.../version/SimpleFileVersionController.java | 9 ++--
.../org/apache/iotdb/db/metadata/MManager.java | 9 ++--
.../iotdb/db/monitor/collector/FileSize.java | 5 ++-
.../java/org/apache/iotdb/db/service/Monitor.java | 5 ++-
.../iotdb/db/sync/receiver/SyncServiceImpl.java | 21 +++++----
.../apache/iotdb/db/tools/MemEst/MemEstTool.java | 5 ++-
.../iotdb/db/tools/TsFileResourcePrinter.java | 6 +--
.../java/org/apache/iotdb/db/tools/WalChecker.java | 6 ++-
.../org/apache/iotdb/db/writelog/io/LogWriter.java | 3 +-
.../db/writelog/node/ExclusiveWriteLogNode.java | 12 ++---
.../iotdb/db/writelog/recover/LogReplayer.java | 5 ++-
.../writelog/recover/TsFileRecoverPerformer.java | 4 +-
.../apache/iotdb/db/engine/merge/MergeTest.java | 8 ++--
.../engine/storagegroup/TsFileProcessorTest.java | 12 ++---
.../db/query/control/FileReaderManagerTest.java | 8 ++--
.../iotdb/db/writelog/recover/LogReplayerTest.java | 6 +--
.../db/writelog/recover/SeqTsFileRecoverTest.java | 4 +-
.../writelog/recover/UnseqTsFileRecoverTest.java | 4 +-
32 files changed, 243 insertions(+), 145 deletions(-)
diff --git a/docs/Documentation-CHN/UserGuide/4-Deployment and Management/2-Configuration.md b/docs/Documentation-CHN/UserGuide/4-Deployment and Management/2-Configuration.md
index d8816bc..7c101f4 100644
--- a/docs/Documentation-CHN/UserGuide/4-Deployment and Management/2-Configuration.md
+++ b/docs/Documentation-CHN/UserGuide/4-Deployment and Management/2-Configuration.md
@@ -292,3 +292,30 @@
|类型| Int32 |
|默认值|600 |
|改后生效方式|重启服务器生效|
+
+* tsfile\_storage\_fs
+
+|名字| tsfile\_storage\_fs |
+|:---:|:---|
+|描述| Tsfile和相关数据文件的存储文件系统。目前支持LOCAL（本地文件系统）和HDFS两种|
+|类型| String |
+|默认值|LOCAL |
+|改后生效方式|重启服务器生效|
+
+* hdfs\_ip
+
+|名字| hdfs\_ip |
+|:---:|:---|
+|描述| 在Tsfile和相关数据文件存储到HDFS的情况下用于配置HDFS的IP|
+|类型| String |
+|默认值|localhost |
+|改后生效方式|重启服务器生效|
+
+* hdfs\_port
+
+|名字| hdfs\_port |
+|:---:|:---|
+|描述| 在Tsfile和相关数据文件存储到HDFS的情况下用于配置HDFS的端口|
+|类型| String |
+|默认值|9000 |
+|改后生效方式|重启服务器生效|
diff --git a/docs/Documentation/UserGuide/4-Deployment and Management/2-Configuration.md b/docs/Documentation/UserGuide/4-Deployment and Management/2-Configuration.md
index 031c4d1..ec8494c 100644
--- a/docs/Documentation/UserGuide/4-Deployment and Management/2-Configuration.md
+++ b/docs/Documentation/UserGuide/4-Deployment and Management/2-Configuration.md
@@ -327,3 +327,30 @@ The detail of each variables are as follows:
|Type| Int32 |
|Default|600 |
|Effective|After restart system|
+
+* tsfile\_storage\_fs
+
+|Name| tsfile\_storage\_fs |
+|:---:|:---|
+|Description| The storage file system of Tsfile and related data files. Currently LOCAL file system and HDFS are supported.|
+|Type| String |
+|Default|LOCAL |
+|Effective|After restart system|
+
+* hdfs\_ip
+
+|Name| hdfs\_ip |
+|:---:|:---|
+|Description| IP of HDFS if Tsfile and related data files are stored in HDFS|
+|Type| String |
+|Default|localhost |
+|Effective|After restart system|
+
+* hdfs\_port
+
+|Name| hdfs\_port |
+|:---:|:---|
+|Description| Port of HDFS if Tsfile and related data files are stored in HDFS|
+|Type| String |
+|Default|9000 |
+|Effective|After restart system|
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index 42f527b..eacc84b 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -122,8 +122,14 @@ timestamp_precision=ms
# wal_dir=data/wal
-# Storage file system. Currently, Tsfile supports to be stored in LOCAL file system or HDFS.
-storage_fs=LOCAL
+# TSFile storage file system. Currently, Tsfile are supported to be stored in LOCAL file system or HDFS.
+tsfile_storage_fs=LOCAL
+
+# If using HDFS, hadoop ip can be configured
+hdfs_ip=localhost
+
+# If using HDFS, hadoop port can be configured
+hdfs_port=9000
####################
### Memory Control Configuration
diff --git a/server/src/main/java/org/apache/iotdb/db/auth/role/LocalFileRoleAccessor.java b/server/src/main/java/org/apache/iotdb/db/auth/role/LocalFileRoleAccessor.java
index 3bf3d56..253fe5f 100644
--- a/server/src/main/java/org/apache/iotdb/db/auth/role/LocalFileRoleAccessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/auth/role/LocalFileRoleAccessor.java
@@ -33,6 +33,7 @@ import java.util.Set;
import org.apache.iotdb.db.auth.entity.PathPrivilege;
import org.apache.iotdb.db.auth.entity.Role;
import org.apache.iotdb.db.conf.IoTDBConstant;
+import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
import org.apache.iotdb.db.utils.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -66,11 +67,11 @@ public class LocalFileRoleAccessor implements IRoleAccessor {
@Override
public Role loadRole(String rolename) throws IOException {
- File roleProfile = new File(
+ File roleProfile = SystemFileFactory.INSTANCE.getFile(
roleDirPath + File.separator + rolename + IoTDBConstant.PROFILE_SUFFIX);
if (!roleProfile.exists() || !roleProfile.isFile()) {
// System may crush before a newer file is written, so search for back-up file.
- File backProfile = new File(
+ File backProfile = SystemFileFactory.INSTANCE.getFile(
roleDirPath + File.separator + rolename + IoTDBConstant.PROFILE_SUFFIX + TEMP_SUFFIX);
if (backProfile.exists() && backProfile.isFile()) {
roleProfile = backProfile;
@@ -100,7 +101,7 @@ public class LocalFileRoleAccessor implements IRoleAccessor {
@Override
public void saveRole(Role role) throws IOException {
- File roleProfile = new File(
+ File roleProfile = SystemFileFactory.INSTANCE.getFile(
roleDirPath + File.separator + role.getName() + IoTDBConstant.PROFILE_SUFFIX + TEMP_SUFFIX);
try (BufferedOutputStream outputStream = new BufferedOutputStream(new FileOutputStream(roleProfile))) {
try {
@@ -121,16 +122,16 @@ public class LocalFileRoleAccessor implements IRoleAccessor {
}
}
- File oldFile = new File(
+ File oldFile = SystemFileFactory.INSTANCE.getFile(
roleDirPath + File.separator + role.getName() + IoTDBConstant.PROFILE_SUFFIX);
IOUtils.replaceFile(roleProfile, oldFile);
}
@Override
public boolean deleteRole(String rolename) throws IOException {
- File roleProfile = new File(
+ File roleProfile = SystemFileFactory.INSTANCE.getFile(
roleDirPath + File.separator + rolename + IoTDBConstant.PROFILE_SUFFIX);
- File backFile = new File(
+ File backFile = SystemFileFactory.INSTANCE.getFile(
roleDirPath + File.separator + rolename + IoTDBConstant.PROFILE_SUFFIX + TEMP_SUFFIX);
if (!roleProfile.exists() && !backFile.exists()) {
return false;
@@ -144,7 +145,7 @@ public class LocalFileRoleAccessor implements IRoleAccessor {
@Override
public List<String> listAllRoles() {
- File roleDir = new File(roleDirPath);
+ File roleDir = SystemFileFactory.INSTANCE.getFile(roleDirPath);
String[] names = roleDir
.list((dir, name) -> name.endsWith(IoTDBConstant.PROFILE_SUFFIX) || name
.endsWith(TEMP_SUFFIX));
@@ -163,9 +164,9 @@ public class LocalFileRoleAccessor implements IRoleAccessor {
@Override
public void reset() {
- if (new File(roleDirPath).mkdirs()) {
+ if (SystemFileFactory.INSTANCE.getFile(roleDirPath).mkdirs()) {
logger.info("role info dir {} is created", roleDirPath);
- } else if (!new File(roleDirPath).exists()) {
+ } else if (!SystemFileFactory.INSTANCE.getFile(roleDirPath).exists()) {
logger.error("role info dir {} can not be created", roleDirPath);
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/auth/user/LocalFileUserAccessor.java b/server/src/main/java/org/apache/iotdb/db/auth/user/LocalFileUserAccessor.java
index 8ee8856..c4fe1d0 100644
--- a/server/src/main/java/org/apache/iotdb/db/auth/user/LocalFileUserAccessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/auth/user/LocalFileUserAccessor.java
@@ -18,25 +18,21 @@
*/
package org.apache.iotdb.db.auth.user;
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.DataInputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
import org.apache.iotdb.db.auth.entity.PathPrivilege;
import org.apache.iotdb.db.auth.entity.User;
import org.apache.iotdb.db.conf.IoTDBConstant;
+import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
import org.apache.iotdb.db.utils.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.*;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
/**
* This class loads a user's information from the corresponding file.The user file is a sequential
* file. User file schema: Int32 username bytes size Utf-8 username bytes Int32 Password bytes
@@ -75,11 +71,11 @@ public class LocalFileUserAccessor implements IUserAccessor {
*/
@Override
public User loadUser(String username) throws IOException {
- File userProfile = new File(
+ File userProfile = SystemFileFactory.INSTANCE.getFile(
userDirPath + File.separator + username + IoTDBConstant.PROFILE_SUFFIX);
if (!userProfile.exists() || !userProfile.isFile()) {
// System may crush before a newer file is renamed.
- File newProfile = new File(
+ File newProfile = SystemFileFactory.INSTANCE.getFile(
userDirPath + File.separator + username + IoTDBConstant.PROFILE_SUFFIX + TEMP_SUFFIX);
if (newProfile.exists() && newProfile.isFile()) {
if(!newProfile.renameTo(userProfile)) {
@@ -124,7 +120,7 @@ public class LocalFileUserAccessor implements IUserAccessor {
*/
@Override
public void saveUser(User user) throws IOException {
- File userProfile = new File(
+ File userProfile = SystemFileFactory.INSTANCE.getFile(
userDirPath + File.separator + user.getName() + IoTDBConstant.PROFILE_SUFFIX + TEMP_SUFFIX);
try(BufferedOutputStream outputStream = new BufferedOutputStream(new FileOutputStream(userProfile))) {
try {
@@ -156,7 +152,7 @@ public class LocalFileUserAccessor implements IUserAccessor {
}
}
- File oldFile = new File(
+ File oldFile = SystemFileFactory.INSTANCE.getFile(
userDirPath + File.separator + user.getName() + IoTDBConstant.PROFILE_SUFFIX);
IOUtils.replaceFile(userProfile, oldFile);
}
@@ -170,9 +166,9 @@ public class LocalFileUserAccessor implements IUserAccessor {
*/
@Override
public boolean deleteUser(String username) throws IOException {
- File userProfile = new File(
+ File userProfile = SystemFileFactory.INSTANCE.getFile(
userDirPath + File.separator + username + IoTDBConstant.PROFILE_SUFFIX);
- File backFile = new File(
+ File backFile = SystemFileFactory.INSTANCE.getFile(
userDirPath + File.separator + username + IoTDBConstant.PROFILE_SUFFIX + TEMP_SUFFIX);
if (!userProfile.exists() && !backFile.exists()) {
return false;
@@ -186,7 +182,7 @@ public class LocalFileUserAccessor implements IUserAccessor {
@Override
public List<String> listAllUsers() {
- File userDir = new File(userDirPath);
+ File userDir = SystemFileFactory.INSTANCE.getFile(userDirPath);
String[] names = userDir
.list((dir, name) -> name.endsWith(IoTDBConstant.PROFILE_SUFFIX) || name
.endsWith(TEMP_SUFFIX));
@@ -205,9 +201,9 @@ public class LocalFileUserAccessor implements IUserAccessor {
@Override
public void reset() {
- if (new File(userDirPath).mkdirs()) {
+ if (SystemFileFactory.INSTANCE.getFile(userDirPath).mkdirs()) {
logger.info("user info dir {} is created", userDirPath);
- } else if (!new File(userDirPath).exists()) {
+ } else if (!SystemFileFactory.INSTANCE.getFile(userDirPath).exists()) {
logger.error("user info dir {} can not be created", userDirPath);
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 1b71c8e..34baf8f 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -327,9 +327,24 @@ public class IoTDBConfig {
private MergeFileStrategy mergeFileStrategy = MergeFileStrategy.MAX_SERIES_NUM;
/**
- * Default storage is in local file system
+ * Default system file storage is in local file system (unsupported)
*/
- private FSType storageFs = FSType.LOCAL;
+ private FSType systemFileStorageFs = FSType.LOCAL;
+
+ /**
+ * Default TSfile storage is in local file system
+ */
+ private FSType tsFileStorageFs = FSType.LOCAL;
+
+ /**
+ * Default HDFS ip is localhost
+ */
+ private String hdfsIp = "localhost";
+
+ /**
+ * Default HDFS port is 9000
+ */
+ private String hdfsPort = "9000";
public IoTDBConfig() {
// empty constructor
@@ -904,11 +919,35 @@ public class IoTDBConfig {
return null;
}
- public FSType getStorageFs() {
- return storageFs;
+ public FSType getSystemFileStorageFs() {
+ return systemFileStorageFs;
+ }
+
+ public void setSystemFileStorageFs(String systemFileStorageFs) {
+ this.systemFileStorageFs = FSType.valueOf(systemFileStorageFs);
+ }
+
+ public FSType getTsFileStorageFs() {
+ return tsFileStorageFs;
+ }
+
+ public void setTsFileStorageFs(String tsFileStorageFs) {
+ this.tsFileStorageFs = FSType.valueOf(tsFileStorageFs);
+ }
+
+ public String getHdfsIp() {
+ return hdfsIp;
+ }
+
+ public void setHdfsIp(String hdfsIp) {
+ this.hdfsIp = hdfsIp;
+ }
+
+ public String getHdfsPort() {
+ return hdfsPort;
}
- public void setStorageFs(String storageFs) {
- this.storageFs = FSType.valueOf(storageFs);
+ public void setHdfsPort(String hdfsPort) {
+ this.hdfsPort = hdfsPort;
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java
index 59270e2..ad42d18 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java
@@ -24,6 +24,8 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Properties;
+
+import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -53,7 +55,7 @@ public class IoTDBConfigCheck {
}
public void createDir(String filepath) {
- File dir = new File(filepath);
+ File dir = SystemFileFactory.INSTANCE.getFile(filepath);
if (!dir.exists()) {
dir.mkdirs();
logger.info(" {} dir has been created.", SCHEMA_DIR);
@@ -63,7 +65,7 @@ public class IoTDBConfigCheck {
public void checkFile(String filepath) {
// create file : read timestamp precision from engine.properties, create system_properties.txt
// use output stream to write timestamp precision to file.
- File file = new File(filepath + File.separator + PROPERTIES_FILE_NAME);
+ File file = SystemFileFactory.INSTANCE.getFile(filepath + File.separator + PROPERTIES_FILE_NAME);
try {
if (!file.exists()) {
file.createNewFile();
@@ -77,7 +79,7 @@ public class IoTDBConfigCheck {
logger.error("Can not create {}.", file.getAbsolutePath(), e);
}
// get existed properties from system_properties.txt
- File inputFile = new File(filepath + File.separator + PROPERTIES_FILE_NAME);
+ File inputFile = SystemFileFactory.INSTANCE.getFile(filepath + File.separator + PROPERTIES_FILE_NAME);
try (FileInputStream inputStream = new FileInputStream(inputFile.toString())) {
properties.load(new InputStreamReader(inputStream, TSFileConfig.STRING_ENCODING));
if (!properties.getProperty("timestamp_precision").equals(TIMESTAMP_PRECISION)) {
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index f1cf1dc..98321f3 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.conf;
import org.apache.iotdb.db.utils.FilePathUtils;
+import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -268,7 +269,15 @@ public class IoTDBDescriptor {
conf.setRpcMaxConcurrentClientNum(maxConcurrentClientNum);
- conf.setStorageFs(properties.getProperty("storage_fs"));
+ conf.setTsFileStorageFs(properties.getProperty("tsfile_storage_fs"));
+ conf.setHdfsIp(properties.getProperty("hdfs_ip"));
+ conf.setHdfsPort(properties.getProperty("hdfs_port"));
+
+ // At the same time, set TSFileConfig
+ TSFileConfig.setTSFileStorageFs(properties.getProperty("tsfile_storage_fs"));
+ TSFileConfig.setHdfsIp(properties.getProperty("hdfs_ip"));
+ TSFileConfig.setHdfsPort(properties.getProperty("hdfs_port"));
+
} catch (IOException e) {
logger.warn("Cannot load config file because, use default configuration", e);
} catch (Exception e) {
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/adapter/CompressionRatio.java b/server/src/main/java/org/apache/iotdb/db/conf/adapter/CompressionRatio.java
index c7819fe..86e2470 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/adapter/CompressionRatio.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/adapter/CompressionRatio.java
@@ -24,6 +24,7 @@ import java.util.Locale;
import org.apache.commons.io.FileUtils;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
import org.apache.iotdb.db.utils.FilePathUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -67,7 +68,7 @@ public class CompressionRatio {
private File directory;
private CompressionRatio() {
- directory = new File(
+ directory = SystemFileFactory.INSTANCE.getFile(
FilePathUtils.regularizePath(CONFIG.getSystemDir()) + COMPRESSION_RATIO_DIR);
restore();
}
@@ -79,11 +80,11 @@ public class CompressionRatio {
* @param currentCompressionRatio the compression ratio of the closing file.
*/
public synchronized void updateRatio(double currentCompressionRatio) throws IOException {
- File oldFile = new File(directory,
+ File oldFile = SystemFileFactory.INSTANCE.getFile(directory,
String.format(Locale.ENGLISH, RATIO_FILE_PATH_FORMAT, compressionRatioSum, calcTimes));
compressionRatioSum += currentCompressionRatio;
calcTimes++;
- File newFile = new File(directory,
+ File newFile = SystemFileFactory.INSTANCE.getFile(directory,
String.format(Locale.ENGLISH, RATIO_FILE_PATH_FORMAT, compressionRatioSum, calcTimes));
persist(oldFile, newFile);
if (LOGGER.isInfoEnabled()) {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
index 3e36516..fc11c68 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
@@ -18,7 +18,6 @@
*/
package org.apache.iotdb.db.engine;
-import java.io.File;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
@@ -28,6 +27,7 @@ import java.util.concurrent.ConcurrentHashMap;
import org.apache.commons.io.FileUtils;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
import org.apache.iotdb.db.engine.storagegroup.StorageGroupProcessor;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
@@ -78,7 +78,7 @@ public class StorageEngine implements IService {
systemDir = FilePathUtils.regularizePath(config.getSystemDir()) + "storage_groups";
// create systemDir
try {
- FileUtils.forceMkdir(new File(systemDir));
+ FileUtils.forceMkdir(SystemFileFactory.INSTANCE.getFile(systemDir));
} catch (IOException e) {
throw new StorageEngineFailureException("create system directory failed!");
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/fileSystem/FSType.java b/server/src/main/java/org/apache/iotdb/db/engine/fileSystem/FSType.java
deleted file mode 100644
index 4bbe2ec..0000000
--- a/server/src/main/java/org/apache/iotdb/db/engine/fileSystem/FSType.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.engine.fileSystem;
-
-public enum FSType {
- LOCAL, HDFS
-}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/fileSystem/FileFactory.java b/server/src/main/java/org/apache/iotdb/db/engine/fileSystem/SystemFileFactory.java
similarity index 73%
rename from server/src/main/java/org/apache/iotdb/db/engine/fileSystem/FileFactory.java
rename to server/src/main/java/org/apache/iotdb/db/engine/fileSystem/SystemFileFactory.java
index 32e8121..ce6ffea 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/fileSystem/FileFactory.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/fileSystem/SystemFileFactory.java
@@ -26,15 +26,16 @@ import org.apache.iotdb.tsfile.fileSystem.HDFSFile;
import java.io.File;
import java.net.URI;
-public enum FileFactory {
+public enum SystemFileFactory {
INSTANCE;
- private static FSType fsType = IoTDBDescriptor.getInstance().getConfig().getStorageFs();
+ private static FSType fsType = IoTDBDescriptor.getInstance().getConfig().getSystemFileStorageFs();
public File getFile(String pathname) {
if (fsType.equals(FSType.HDFS)) {
- return new HDFSFile(pathname);
+ throw new UnsupportedOperationException("Unsupported file system: " + fsType.name());
+ // return new HDFSFile(pathname);
} else {
return new File(pathname);
}
@@ -42,7 +43,8 @@ public enum FileFactory {
public File getFile(String parent, String child) {
if (fsType.equals(FSType.HDFS)) {
- return new HDFSFile(parent, child);
+ throw new UnsupportedOperationException("Unsupported file system: " + fsType.name());
+ // return new HDFSFile(parent, child);
} else {
return new File(parent, child);
}
@@ -50,7 +52,8 @@ public enum FileFactory {
public File getFile(File parent, String child) {
if (fsType.equals(FSType.HDFS)) {
- return new HDFSFile(parent, child);
+ throw new UnsupportedOperationException("Unsupported file system: " + fsType.name());
+ // return new HDFSFile(parent, child);
} else {
return new File(parent, child);
}
@@ -58,7 +61,8 @@ public enum FileFactory {
public File getFile(URI uri) {
if (fsType.equals(FSType.HDFS)) {
- return new HDFSFile(uri);
+ throw new UnsupportedOperationException("Unsupported file system: " + fsType.name());
+ // return new HDFSFile(uri);
} else {
return new File(uri);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/recover/LogAnalyzer.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/recover/LogAnalyzer.java
index b7d98fc..bde1971 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/merge/recover/LogAnalyzer.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/recover/LogAnalyzer.java
@@ -27,6 +27,8 @@ import java.io.FileReader;
import java.io.IOException;
import java.util.*;
import java.util.Map.Entry;
+
+import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
import org.apache.iotdb.db.engine.merge.manage.MergeResource;
import org.apache.iotdb.db.engine.merge.task.MergeTask;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
@@ -174,7 +176,7 @@ public class LogAnalyzer {
status = Status.MERGE_START;
for (TsFileResource seqFile : resource.getSeqFiles()) {
- File mergeFile = new File(seqFile.getFile().getPath() + MergeTask.MERGE_SUFFIX);
+ File mergeFile = SystemFileFactory.INSTANCE.getFile(seqFile.getFile().getPath() + MergeTask.MERGE_SUFFIX);
fileLastPositions.put(mergeFile, 0L);
}
@@ -194,7 +196,7 @@ public class LogAnalyzer {
} else if (!currLine.contains(STR_END)) {
// file position
String[] splits = currLine.split(" ");
- File file = new File(splits[0]);
+ File file = SystemFileFactory.INSTANCE.getFile(splits[0]);
Long position = Long.parseLong(splits[1]);
tempFileLastPositions.put(file, position);
} else {
@@ -231,7 +233,7 @@ public class LogAnalyzer {
}
if (!currLine.contains(STR_END)) {
String[] splits = currLine.split(" ");
- currFile = new File(splits[0]);
+ currFile = SystemFileFactory.INSTANCE.getFile(splits[0]);
Long lastPost = Long.parseLong(splits[1]);
fileLastPositions.put(currFile, lastPost);
} else {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index 8ceae5b..2cd8c87 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -73,7 +73,7 @@ import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
-import org.apache.iotdb.db.engine.fileSystem.FileFactory;
+import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
import org.apache.iotdb.tsfile.fileSystem.TSFileFactory;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.utils.Pair;
@@ -185,7 +185,7 @@ public class StorageGroupProcessor {
this.schema = constructSchema(storageGroupName);
try {
- storageGroupSysDir = FileFactory.INSTANCE.getFile(systemInfoDir, storageGroupName);
+ storageGroupSysDir = SystemFileFactory.INSTANCE.getFile(systemInfoDir, storageGroupName);
if (storageGroupSysDir.mkdirs()) {
logger.info("Storage Group system Directory {} doesn't exist, create it",
storageGroupSysDir.getPath());
@@ -215,7 +215,7 @@ public class StorageGroupProcessor {
recoverUnseqFiles(unseqTsFiles);
String taskName = storageGroupName + "-" + System.currentTimeMillis();
- File mergingMods = FileFactory.INSTANCE.getFile(storageGroupSysDir, MERGING_MODIFICAITON_FILE_NAME);
+ File mergingMods = SystemFileFactory.INSTANCE.getFile(storageGroupSysDir, MERGING_MODIFICAITON_FILE_NAME);
if (mergingMods.exists()) {
mergingModification = new ModificationFile(mergingMods.getPath());
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/version/SimpleFileVersionController.java b/server/src/main/java/org/apache/iotdb/db/engine/version/SimpleFileVersionController.java
index 53a93ee..80bd8ef 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/version/SimpleFileVersionController.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/version/SimpleFileVersionController.java
@@ -23,6 +23,7 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import org.apache.commons.io.FileUtils;
+import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -75,8 +76,8 @@ public class SimpleFileVersionController implements VersionController {
}
private void persist() throws IOException {
- File oldFile = new File(directoryPath, FILE_PREFIX + prevVersion);
- File newFile = new File(directoryPath, FILE_PREFIX + currVersion);
+ File oldFile = SystemFileFactory.INSTANCE.getFile(directoryPath, FILE_PREFIX + prevVersion);
+ File newFile = SystemFileFactory.INSTANCE.getFile(directoryPath, FILE_PREFIX + currVersion);
FileUtils.moveFile(oldFile, newFile);
logger.info("Version file updated, previous: {}, current: {}",
oldFile.getAbsolutePath(), newFile.getAbsolutePath());
@@ -84,7 +85,7 @@ public class SimpleFileVersionController implements VersionController {
}
private void restore() throws IOException {
- File directory = new File(directoryPath);
+ File directory = SystemFileFactory.INSTANCE.getFile(directoryPath);
File[] versionFiles = directory.listFiles((dir, name) -> name.startsWith(FILE_PREFIX));
File versionFile;
if (versionFiles != null && versionFiles.length > 0) {
@@ -105,7 +106,7 @@ public class SimpleFileVersionController implements VersionController {
}
}
} else {
- versionFile = new File(directory, FILE_PREFIX + "0");
+ versionFile = SystemFileFactory.INSTANCE.getFile(directory, FILE_PREFIX + "0");
prevVersion = 0;
new FileOutputStream(versionFile).close();
}
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
index 2da2f6c..24abcc5 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
@@ -24,6 +24,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.adapter.IoTDBConfigDynamicAdapter;
+import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
import org.apache.iotdb.db.exception.ConfigAdjusterException;
import org.apache.iotdb.db.exception.MetadataErrorException;
import org.apache.iotdb.db.exception.PathErrorException;
@@ -71,7 +72,7 @@ public class MManager {
schemaDir =
IoTDBDescriptor.getInstance().getConfig().getSystemDir() + File.separator + "schema";
- File systemFolder = new File(schemaDir);
+ File systemFolder = SystemFileFactory.INSTANCE.getFile(schemaDir);
if (!systemFolder.exists()) {
if (systemFolder.mkdirs()) {
logger.info("create system folder {}", systemFolder.getAbsolutePath());
@@ -124,7 +125,7 @@ public class MManager {
return;
}
lock.writeLock().lock();
- File logFile = new File(logFilePath);
+ File logFile = SystemFileFactory.INSTANCE.getFile(logFilePath);
try {
initFromLog(logFile);
@@ -232,8 +233,8 @@ public class MManager {
private BufferedWriter getLogWriter() throws IOException {
if (logWriter == null) {
- File logFile = new File(logFilePath);
- File metadataDir = new File(schemaDir);
+ File logFile = SystemFileFactory.INSTANCE.getFile(logFilePath);
+ File metadataDir = SystemFileFactory.INSTANCE.getFile(schemaDir);
if (!metadataDir.exists()) {
if (metadataDir.mkdirs()) {
logger.info("create schema folder {}.", metadataDir);
diff --git a/server/src/main/java/org/apache/iotdb/db/monitor/collector/FileSize.java b/server/src/main/java/org/apache/iotdb/db/monitor/collector/FileSize.java
index 01d4b52..e6cd84e 100644
--- a/server/src/main/java/org/apache/iotdb/db/monitor/collector/FileSize.java
+++ b/server/src/main/java/org/apache/iotdb/db/monitor/collector/FileSize.java
@@ -31,6 +31,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.StorageEngine;
+import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.monitor.IStatistic;
import org.apache.iotdb.db.monitor.MonitorConstants;
@@ -139,7 +140,7 @@ public class FileSize implements IStatistic {
if (kinds.equals(FileSizeConstants.SYS)) {
fileSizes.put(kinds, collectSeqFileSize(fileSizes, kinds));
} else {
- File file = new File(kinds.getPath());
+ File file = SystemFileFactory.INSTANCE.getFile(kinds.getPath());
if (file.exists()) {
try {
fileSizes.put(kinds, FileUtils.sizeOfDirectory(file));
@@ -162,7 +163,7 @@ public class FileSize implements IStatistic {
if (sequenceDir.contains("unsequence")) {
continue;
}
- File settledFile = new File(sequenceDir);
+ File settledFile = SystemFileFactory.INSTANCE.getFile(sequenceDir);
if (settledFile.exists()) {
try {
fileSize += FileUtils.sizeOfDirectory(settledFile);
diff --git a/server/src/main/java/org/apache/iotdb/db/service/Monitor.java b/server/src/main/java/org/apache/iotdb/db/service/Monitor.java
index 8193fd7..578630e 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/Monitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/Monitor.java
@@ -23,6 +23,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
import org.apache.iotdb.db.exception.StartupException;
import org.apache.iotdb.db.utils.OpenFileNumUtil;
import org.slf4j.Logger;
@@ -48,7 +49,7 @@ public class Monitor implements MonitorMBean, IService {
try {
long totalSize = 0;
for (String dataDir : config.getDataDirs()) {
- totalSize += FileUtils.sizeOfDirectory(new File(dataDir));
+ totalSize += FileUtils.sizeOfDirectory(SystemFileFactory.INSTANCE.getFile(dataDir));
}
return totalSize;
} catch (Exception e) {
@@ -78,7 +79,7 @@ public class Monitor implements MonitorMBean, IService {
@Override
public String getBaseDirectory() {
try {
- File file = new File(config.getBaseDir());
+ File file = SystemFileFactory.INSTANCE.getFile(config.getBaseDir());
return file.getAbsolutePath();
} catch (Exception e) {
logger.error("meet error while trying to get base dir.", e);
diff --git a/server/src/main/java/org/apache/iotdb/db/sync/receiver/SyncServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/sync/receiver/SyncServiceImpl.java
index 05c30a2..a2fa8c6 100644
--- a/server/src/main/java/org/apache/iotdb/db/sync/receiver/SyncServiceImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/sync/receiver/SyncServiceImpl.java
@@ -61,7 +61,6 @@ import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadataIndex;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
-import org.apache.iotdb.db.engine.fileSystem.FileFactory;
import org.apache.iotdb.tsfile.read.ReadOnlyTsFile;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Field;
@@ -145,7 +144,7 @@ public class SyncServiceImpl implements SyncService.Iface {
fileNodeStartTime.set(new HashMap<>());
fileNodeEndTime.set(new HashMap<>());
try {
- FileUtils.deleteDirectory(FileFactory.INSTANCE.getFile(syncDataPath));
+ FileUtils.deleteDirectory(new File(syncDataPath));
} catch (IOException e) {
logger.error("cannot delete directory {} ", syncFolderPath);
return false;
@@ -153,7 +152,7 @@ public class SyncServiceImpl implements SyncService.Iface {
for (String bufferWritePath : bufferWritePaths) {
bufferWritePath = FilePathUtils.regularizePath(bufferWritePath);
String backupPath = bufferWritePath + SYNC_SERVER + File.separator;
- File backupDirectory = FileFactory.INSTANCE.getFile(backupPath, this.uuid.get());
+ File backupDirectory = new File(backupPath, this.uuid.get());
if (backupDirectory.exists() && backupDirectory.list().length != 0) {
try {
FileUtils.deleteDirectory(backupDirectory);
@@ -202,7 +201,7 @@ public class SyncServiceImpl implements SyncService.Iface {
/** sync metadata, include storage group and timeseries **/
return Boolean.toString(loadMetadata());
} else if (status == SyncDataStatus.PROCESSING_STATUS) {
- File file = FileFactory.INSTANCE.getFile(schemaFromSenderPath.get());
+ File file = new File(schemaFromSenderPath.get());
if (!file.getParentFile().exists()) {
try {
file.getParentFile().mkdirs();
@@ -229,7 +228,7 @@ public class SyncServiceImpl implements SyncService.Iface {
}
md5OfReceiver = (new BigInteger(1, md.digest())).toString(16);
if (!md5.equals(md5OfReceiver)) {
- FileUtils.forceDelete(FileFactory.INSTANCE.getFile(schemaFromSenderPath.get()));
+ FileUtils.forceDelete(new File(schemaFromSenderPath.get()));
}
} catch (Exception e) {
logger.error("Receiver cannot generate md5 {}", schemaFromSenderPath.get(), e);
@@ -242,7 +241,7 @@ public class SyncServiceImpl implements SyncService.Iface {
* Load metadata from sender
*/
private boolean loadMetadata() {
- if (FileFactory.INSTANCE.getFile(schemaFromSenderPath.get()).exists()) {
+ if (new File(schemaFromSenderPath.get()).exists()) {
try (BufferedReader br = new BufferedReader(
new java.io.FileReader(schemaFromSenderPath.get()))) {
String metadataOperation;
@@ -327,7 +326,7 @@ public class SyncServiceImpl implements SyncService.Iface {
syncDataPath = FilePathUtils.regularizePath(syncDataPath);
filePath = syncDataPath + filePath;
if (status == SyncDataStatus.PROCESSING_STATUS) { // there are still data stream to add
- File file = FileFactory.INSTANCE.getFile(filePath);
+ File file = new File(filePath);
if (!file.getParentFile().exists()) {
try {
file.getParentFile().mkdirs();
@@ -359,7 +358,7 @@ public class SyncServiceImpl implements SyncService.Iface {
logger.info(String.format("Receiver has received %d files from sender", fileNum.get()));
} else {
- FileUtils.forceDelete(FileFactory.INSTANCE.getFile(filePath));
+ FileUtils.forceDelete(new File(filePath));
}
} catch (Exception e) {
logger.error("Receiver cannot generate md5 {}", filePath, e);
@@ -385,7 +384,7 @@ public class SyncServiceImpl implements SyncService.Iface {
* Get all tsfiles' info which are sent from sender, it is preparing for merging these data
*/
public void getFileNodeInfo() throws IOException {
- File dataFileRoot = FileFactory.INSTANCE.getFile(syncDataPath);
+ File dataFileRoot = new File(syncDataPath);
File[] files = dataFileRoot.listFiles();
int processedNum = 0;
for (File storageGroupPB : files) {
@@ -465,7 +464,7 @@ public class SyncServiceImpl implements SyncService.Iface {
String header = syncDataPath;
String relativePath = path.substring(header.length());
TsFileResource fileNode = new TsFileResource(
- FileFactory.INSTANCE.getFile(DirectoryManager.getInstance().getNextFolderIndexForSequenceFile() +
+ new File(DirectoryManager.getInstance().getNextFolderIndexForSequenceFile() +
File.separator + relativePath), startTimeMap, endTimeMap
);
// call interface of load external file
@@ -714,7 +713,7 @@ public class SyncServiceImpl implements SyncService.Iface {
fileNodeEndTime.remove();
schemaFromSenderPath.remove();
try {
- FileUtils.deleteDirectory(FileFactory.INSTANCE.getFile(syncFolderPath));
+ FileUtils.deleteDirectory(new File(syncFolderPath));
} catch (IOException e) {
logger.error("can not delete directory {}", syncFolderPath, e);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/tools/MemEst/MemEstTool.java b/server/src/main/java/org/apache/iotdb/db/tools/MemEst/MemEstTool.java
index 3f974b1..a5ecf63 100644
--- a/server/src/main/java/org/apache/iotdb/db/tools/MemEst/MemEstTool.java
+++ b/server/src/main/java/org/apache/iotdb/db/tools/MemEst/MemEstTool.java
@@ -29,11 +29,12 @@ import io.airlift.airline.ParseCommandUnrecognizedException;
import io.airlift.airline.ParseOptionConversionException;
import io.airlift.airline.ParseOptionMissingException;
import io.airlift.airline.ParseOptionMissingValueException;
-import java.io.File;
+
import java.io.IOException;
import java.util.List;
import org.apache.commons.io.FileUtils;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
public class MemEstTool {
@@ -69,7 +70,7 @@ public class MemEstTool {
err(Throwables.getRootCause(e));
status = 2;
}
- FileUtils.deleteDirectory(new File(IoTDBDescriptor.getInstance().getConfig().getBaseDir()));
+ FileUtils.deleteDirectory(SystemFileFactory.INSTANCE.getFile(IoTDBDescriptor.getInstance().getConfig().getBaseDir()));
System.exit(status);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/tools/TsFileResourcePrinter.java b/server/src/main/java/org/apache/iotdb/db/tools/TsFileResourcePrinter.java
index 8b8de72..f26268b 100644
--- a/server/src/main/java/org/apache/iotdb/db/tools/TsFileResourcePrinter.java
+++ b/server/src/main/java/org/apache/iotdb/db/tools/TsFileResourcePrinter.java
@@ -26,7 +26,7 @@ import java.util.Comparator;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.qp.constant.DatetimeUtils;
-import org.apache.iotdb.db.engine.fileSystem.FileFactory;
+import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
/**
* this tool can analyze the tsfile.resource files from a folder.
@@ -39,7 +39,7 @@ public class TsFileResourcePrinter {
if (args.length >= 1) {
folder = args[0];
}
- File folderFile = FileFactory.INSTANCE.getFile(folder);
+ File folderFile = SystemFileFactory.INSTANCE.getFile(folder);
File[] files = folderFile.listFiles(file -> file.getName().endsWith(".tsfile.resource"));
Arrays.sort(files, Comparator.comparingLong(x -> Long.valueOf(x.getName().split("-")[0])));
@@ -51,7 +51,7 @@ public class TsFileResourcePrinter {
public static void printResource(String filename) throws IOException {
filename = filename.substring(0, filename.length() - 9);
- TsFileResource resource = new TsFileResource(FileFactory.INSTANCE.getFile(filename));
+ TsFileResource resource = new TsFileResource(SystemFileFactory.INSTANCE.getFile(filename));
System.err.println(String.format("analyzing %s ...", filename));
resource.deSerialize();
diff --git a/server/src/main/java/org/apache/iotdb/db/tools/WalChecker.java b/server/src/main/java/org/apache/iotdb/db/tools/WalChecker.java
index d390403..9df9882 100644
--- a/server/src/main/java/org/apache/iotdb/db/tools/WalChecker.java
+++ b/server/src/main/java/org/apache/iotdb/db/tools/WalChecker.java
@@ -25,6 +25,8 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+
+import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
import org.apache.iotdb.db.exception.SysCheckException;
import org.apache.iotdb.db.writelog.io.SingleFileLogReader;
import org.slf4j.Logger;
@@ -52,7 +54,7 @@ public class WalChecker {
* @throws SysCheckException if the root wal dir does not exist.
*/
public List<File> doCheck() throws SysCheckException {
- File walFolderFile = new File(walFolder);
+ File walFolderFile = SystemFileFactory.INSTANCE.getFile(walFolder);
logger.info("Checking folder: {}", walFolderFile.getAbsolutePath());
if(!walFolderFile.exists() || !walFolderFile.isDirectory()) {
throw new SysCheckException(String.format("%s is not a directory", walFolder));
@@ -68,7 +70,7 @@ public class WalChecker {
for (int dirIndex = 0; dirIndex < storageWalFolders.length; dirIndex++) {
File storageWalFolder = storageWalFolders[dirIndex];
logger.info("Checking the No.{} directory {}", dirIndex, storageWalFolder.getName());
- File walFile = new File(storageWalFolder, WAL_FILE_NAME);
+ File walFile = SystemFileFactory.INSTANCE.getFile(storageWalFolder, WAL_FILE_NAME);
if (!checkFile(walFile)) {
failedFiles.add(walFile);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/writelog/io/LogWriter.java b/server/src/main/java/org/apache/iotdb/db/writelog/io/LogWriter.java
index a93cda1..40c923b 100644
--- a/server/src/main/java/org/apache/iotdb/db/writelog/io/LogWriter.java
+++ b/server/src/main/java/org/apache/iotdb/db/writelog/io/LogWriter.java
@@ -26,6 +26,7 @@ import java.nio.channels.FileChannel;
import java.util.zip.CRC32;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
/**
* LogWriter writes the binarized logs into a file using FileChannel together with check sums of
@@ -42,7 +43,7 @@ public class LogWriter implements ILogWriter {
private ByteBuffer checkSumBuffer = ByteBuffer.allocate(8);
public LogWriter(String logFilePath) {
- logFile = new File(logFilePath);
+ logFile = SystemFileFactory.INSTANCE.getFile(logFilePath);
}
public LogWriter(File logFile) {
diff --git a/server/src/main/java/org/apache/iotdb/db/writelog/node/ExclusiveWriteLogNode.java b/server/src/main/java/org/apache/iotdb/db/writelog/node/ExclusiveWriteLogNode.java
index 3ff3fa7..836eea8 100644
--- a/server/src/main/java/org/apache/iotdb/db/writelog/node/ExclusiveWriteLogNode.java
+++ b/server/src/main/java/org/apache/iotdb/db/writelog/node/ExclusiveWriteLogNode.java
@@ -26,7 +26,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.directories.DirectoryManager;
-import org.apache.iotdb.db.engine.StorageEngine;
+import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
import org.apache.iotdb.db.qp.physical.PhysicalPlan;
import org.apache.iotdb.db.writelog.io.ILogReader;
import org.apache.iotdb.db.writelog.io.ILogWriter;
@@ -70,7 +70,7 @@ public class ExclusiveWriteLogNode implements WriteLogNode, Comparable<Exclusive
this.identifier = identifier;
this.logDirectory =
DirectoryManager.getInstance().getWALFolder() + File.separator + this.identifier;
- if (new File(logDirectory).mkdirs()) {
+ if (SystemFileFactory.INSTANCE.getFile(logDirectory).mkdirs()) {
logger.info("create the WAL folder {}." + logDirectory);
}
}
@@ -143,7 +143,7 @@ public class ExclusiveWriteLogNode implements WriteLogNode, Comparable<Exclusive
public void notifyEndFlush() {
lock.writeLock().lock();
try {
- File logFile = new File(logDirectory, WAL_FILE_NAME + ++lastFlushedId);
+ File logFile = SystemFileFactory.INSTANCE.getFile(logDirectory, WAL_FILE_NAME + ++lastFlushedId);
discard(logFile);
} finally {
lock.writeLock().unlock();
@@ -166,7 +166,7 @@ public class ExclusiveWriteLogNode implements WriteLogNode, Comparable<Exclusive
try {
logBuffer.clear();
close();
- FileUtils.deleteDirectory(new File(logDirectory));
+ FileUtils.deleteDirectory(SystemFileFactory.INSTANCE.getFile(logDirectory));
} finally {
lock.writeLock().unlock();
}
@@ -174,7 +174,7 @@ public class ExclusiveWriteLogNode implements WriteLogNode, Comparable<Exclusive
@Override
public ILogReader getLogReader() {
- File[] logFiles = new File(logDirectory).listFiles();
+ File[] logFiles = SystemFileFactory.INSTANCE.getFile(logDirectory).listFiles();
Arrays.sort(logFiles,
Comparator.comparingInt(f -> Integer.parseInt(f.getName().replace(WAL_FILE_NAME, ""))));
return new MultiFileLogReader(logFiles);
@@ -238,7 +238,7 @@ public class ExclusiveWriteLogNode implements WriteLogNode, Comparable<Exclusive
private void nextFileWriter() {
fileId++;
- File newFile = new File(logDirectory, WAL_FILE_NAME + fileId);
+ File newFile = SystemFileFactory.INSTANCE.getFile(logDirectory, WAL_FILE_NAME + fileId);
if (newFile.getParentFile().mkdirs()) {
logger.info("create WAL parent folder {}.", newFile.getParent());
}
diff --git a/server/src/main/java/org/apache/iotdb/db/writelog/recover/LogReplayer.java b/server/src/main/java/org/apache/iotdb/db/writelog/recover/LogReplayer.java
index 42c5b58..164d058 100644
--- a/server/src/main/java/org/apache/iotdb/db/writelog/recover/LogReplayer.java
+++ b/server/src/main/java/org/apache/iotdb/db/writelog/recover/LogReplayer.java
@@ -19,11 +19,12 @@
package org.apache.iotdb.db.writelog.recover;
-import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+
+import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
import org.apache.iotdb.db.engine.memtable.IMemTable;
import org.apache.iotdb.db.engine.modification.Deletion;
import org.apache.iotdb.db.engine.modification.ModificationFile;
@@ -84,7 +85,7 @@ public class LogReplayer {
*/
public void replayLogs() throws ProcessorException {
WriteLogNode logNode = MultiFileLogNodeManager.getInstance().getNode(
- logNodePrefix + new File(insertFilePath).getName());
+ logNodePrefix + SystemFileFactory.INSTANCE.getFile(insertFilePath).getName());
ILogReader logReader = logNode.getLogReader();
try {
diff --git a/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java b/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
index 2fa3ec1..9052f7e 100644
--- a/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
+++ b/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
@@ -38,7 +38,7 @@ import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadata;
import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadataIndex;
import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData;
-import org.apache.iotdb.db.engine.fileSystem.FileFactory;
+import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
import org.apache.iotdb.tsfile.fileSystem.TSFileFactory;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.write.schema.Schema;
@@ -143,7 +143,7 @@ public class TsFileRecoverPerformer {
// clean logs
try {
MultiFileLogNodeManager.getInstance()
- .deleteNode(logNodePrefix + FileFactory.INSTANCE.getFile(insertFilePath).getName());
+ .deleteNode(logNodePrefix + SystemFileFactory.INSTANCE.getFile(insertFilePath).getName());
} catch (IOException e) {
throw new ProcessorException(e);
}
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java
index 853ceaf..59dad16 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java
@@ -40,7 +40,7 @@ import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
-import org.apache.iotdb.db.engine.fileSystem.FileFactory;
+import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
import org.apache.iotdb.tsfile.write.TsFileWriter;
import org.apache.iotdb.tsfile.write.record.TSRecord;
import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint;
@@ -115,18 +115,18 @@ abstract class MergeTest {
private void prepareFiles(int seqFileNum, int unseqFileNum) throws IOException, WriteProcessException {
for (int i = 0; i < seqFileNum; i++) {
- File file = FileFactory.INSTANCE.getFile(i + "seq.tsfile");
+ File file = SystemFileFactory.INSTANCE.getFile(i + "seq.tsfile");
TsFileResource tsFileResource = new TsFileResource(file);
seqResources.add(tsFileResource);
prepareFile(tsFileResource, i * ptNum, ptNum, 0);
}
for (int i = 0; i < unseqFileNum; i++) {
- File file = FileFactory.INSTANCE.getFile(i + "unseq.tsfile");
+ File file = SystemFileFactory.INSTANCE.getFile(i + "unseq.tsfile");
TsFileResource tsFileResource = new TsFileResource(file);
unseqResources.add(tsFileResource);
prepareFile(tsFileResource, i * ptNum, ptNum * (i + 1) / unseqFileNum, 10000);
}
- File file = FileFactory.INSTANCE.getFile(unseqFileNum + "unseq.tsfile");
+ File file = SystemFileFactory.INSTANCE.getFile(unseqFileNum + "unseq.tsfile");
TsFileResource tsFileResource = new TsFileResource(file);
unseqResources.add(tsFileResource);
prepareFile(tsFileResource, 0, ptNum * unseqFileNum, 20000);
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorTest.java b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorTest.java
index 407dd94..ff53332 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorTest.java
@@ -41,7 +41,7 @@ import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetaData;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.db.engine.fileSystem.FileFactory;
+import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.tsfile.write.record.TSRecord;
import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint;
@@ -76,7 +76,7 @@ public class TsFileProcessorTest {
@Test
public void testWriteAndFlush()
throws WriteProcessException, IOException, TsFileProcessorException {
- processor = new TsFileProcessor(storageGroup, FileFactory.INSTANCE.getFile(filePath),
+ processor = new TsFileProcessor(storageGroup, SystemFileFactory.INSTANCE.getFile(filePath),
SchemaUtils.constructSchema(deviceId), SysTimeVersionController.INSTANCE, x -> {
},
() -> true, true);
@@ -123,7 +123,7 @@ public class TsFileProcessorTest {
@Test
public void testWriteAndRestoreMetadata()
throws IOException {
- processor = new TsFileProcessor(storageGroup, FileFactory.INSTANCE.getFile(filePath),
+ processor = new TsFileProcessor(storageGroup, SystemFileFactory.INSTANCE.getFile(filePath),
SchemaUtils.constructSchema(deviceId), SysTimeVersionController.INSTANCE, x -> {
},
() -> true, true);
@@ -168,7 +168,7 @@ public class TsFileProcessorTest {
RestorableTsFileIOWriter tsFileIOWriter = processor.getWriter();
List<ChunkGroupMetaData> chunkGroupMetaDataList = tsFileIOWriter.getChunkGroupMetaDatas();
RestorableTsFileIOWriter restorableTsFileIOWriter = new RestorableTsFileIOWriter(
- FileFactory.INSTANCE.getFile(filePath));
+ SystemFileFactory.INSTANCE.getFile(filePath));
List<ChunkGroupMetaData> restoredChunkGroupMetaDataList = restorableTsFileIOWriter
.getChunkGroupMetaDatas();
assertEquals(chunkGroupMetaDataList.size(), restoredChunkGroupMetaDataList.size());
@@ -190,7 +190,7 @@ public class TsFileProcessorTest {
@Test
public void testMultiFlush()
throws WriteProcessException, IOException, TsFileProcessorException {
- processor = new TsFileProcessor(storageGroup, FileFactory.INSTANCE.getFile(filePath),
+ processor = new TsFileProcessor(storageGroup, SystemFileFactory.INSTANCE.getFile(filePath),
SchemaUtils.constructSchema(deviceId), SysTimeVersionController.INSTANCE, x -> {
},
() -> true, true);
@@ -226,7 +226,7 @@ public class TsFileProcessorTest {
@Test
public void testWriteAndClose()
throws WriteProcessException, IOException {
- processor = new TsFileProcessor(storageGroup, FileFactory.INSTANCE.getFile(filePath),
+ processor = new TsFileProcessor(storageGroup, SystemFileFactory.INSTANCE.getFile(filePath),
SchemaUtils.constructSchema(deviceId), SysTimeVersionController.INSTANCE,
unsealedTsFileProcessor -> {
TsFileResource resource = unsealedTsFileProcessor.getTsFileResource();
diff --git a/server/src/test/java/org/apache/iotdb/db/query/control/FileReaderManagerTest.java b/server/src/test/java/org/apache/iotdb/db/query/control/FileReaderManagerTest.java
index 0545df1..c810772 100644
--- a/server/src/test/java/org/apache/iotdb/db/query/control/FileReaderManagerTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/query/control/FileReaderManagerTest.java
@@ -26,7 +26,7 @@ import java.io.IOException;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
-import org.apache.iotdb.db.engine.fileSystem.FileFactory;
+import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -61,7 +61,7 @@ public class FileReaderManagerTest {
TsFileResource[] tsFileResources = new TsFileResource[MAX_FILE_SIZE + 1];
for (int i = 1; i <= MAX_FILE_SIZE; i++) {
- File file = FileFactory.INSTANCE.getFile(filePath + i);
+ File file = SystemFileFactory.INSTANCE.getFile(filePath + i);
file.createNewFile();
tsFileResources[i] = new TsFileResource(file);
}
@@ -116,7 +116,7 @@ public class FileReaderManagerTest {
t2.join();
for (int i = 1; i <= MAX_FILE_SIZE; i++) {
- TsFileResource tsFile = new TsFileResource(FileFactory.INSTANCE.getFile(filePath + i));
+ TsFileResource tsFile = new TsFileResource(SystemFileFactory.INSTANCE.getFile(filePath + i));
Assert.assertTrue(manager.contains(tsFile, false));
}
@@ -136,7 +136,7 @@ public class FileReaderManagerTest {
FileReaderManager.getInstance().closeAndRemoveAllOpenedReaders();
for (int i = 1; i < MAX_FILE_SIZE; i++) {
- File file = FileFactory.INSTANCE.getFile(filePath + i);
+ File file = SystemFileFactory.INSTANCE.getFile(filePath + i);
boolean result = !file.exists() || file.delete();
if (!result) {
fail();
diff --git a/server/src/test/java/org/apache/iotdb/db/writelog/recover/LogReplayerTest.java b/server/src/test/java/org/apache/iotdb/db/writelog/recover/LogReplayerTest.java
index 12bfbbc..dd46409 100644
--- a/server/src/test/java/org/apache/iotdb/db/writelog/recover/LogReplayerTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/writelog/recover/LogReplayerTest.java
@@ -43,7 +43,7 @@ import org.apache.iotdb.db.writelog.manager.MultiFileLogNodeManager;
import org.apache.iotdb.db.writelog.node.WriteLogNode;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
-import org.apache.iotdb.db.engine.fileSystem.FileFactory;
+import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.write.schema.Schema;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
@@ -54,8 +54,8 @@ public class LogReplayerTest {
@Test
public void test() throws IOException, ProcessorException {
String logNodePrefix = "testLogNode";
- File tsFile = FileFactory.INSTANCE.getFile("temp", "test.ts");
- File modF = FileFactory.INSTANCE.getFile("test.mod");
+ File tsFile = SystemFileFactory.INSTANCE.getFile("temp", "test.ts");
+ File modF = SystemFileFactory.INSTANCE.getFile("test.mod");
ModificationFile modFile = new ModificationFile(modF.getPath());
VersionController versionController = new VersionController() {
@Override
diff --git a/server/src/test/java/org/apache/iotdb/db/writelog/recover/SeqTsFileRecoverTest.java b/server/src/test/java/org/apache/iotdb/db/writelog/recover/SeqTsFileRecoverTest.java
index 67b0e2b..113e24d 100644
--- a/server/src/test/java/org/apache/iotdb/db/writelog/recover/SeqTsFileRecoverTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/writelog/recover/SeqTsFileRecoverTest.java
@@ -35,7 +35,7 @@ import org.apache.iotdb.db.writelog.node.WriteLogNode;
import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
-import org.apache.iotdb.db.engine.fileSystem.FileFactory;
+import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
import org.apache.iotdb.tsfile.read.ReadOnlyTsFile;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Field;
@@ -77,7 +77,7 @@ public class SeqTsFileRecoverTest {
@Before
public void setup() throws IOException, WriteProcessException {
- tsF = FileFactory.INSTANCE.getFile("temp", "test.ts");
+ tsF = SystemFileFactory.INSTANCE.getFile("temp", "test.ts");
tsF.getParentFile().mkdirs();
schema = new Schema();
diff --git a/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java b/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java
index 924081f..1358582 100644
--- a/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java
@@ -37,7 +37,7 @@ import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
-import org.apache.iotdb.db.engine.fileSystem.FileFactory;
+import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Chunk;
import org.apache.iotdb.tsfile.read.common.Path;
@@ -80,7 +80,7 @@ public class UnseqTsFileRecoverTest {
@Before
public void setup() throws IOException, WriteProcessException {
- tsF = FileFactory.INSTANCE.getFile("temp", "test.ts");
+ tsF = SystemFileFactory.INSTANCE.getFile("temp", "test.ts");
tsF.getParentFile().mkdirs();
schema = new Schema();