You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2019/09/06 12:21:23 UTC
[incubator-iotdb] branch master updated: [IOTDB-189] Support
compatibility of HDFS File and Java File (#365)
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 6c009d8 [IOTDB-189] Support compatibility of HDFS File and Java File (#365)
6c009d8 is described below
commit 6c009d862e0c8bd63d1ec863799ccb09cf83ec47
Author: Zesong Sun <15...@smail.nju.edu.cn>
AuthorDate: Fri Sep 6 20:21:18 2019 +0800
[IOTDB-189] Support compatibility of HDFS File and Java File (#365)
* [IOTDB-189] Support compatibility of HDFS file and java file
* Fix "input line is too long" in windows
* Use IoTDBFileFactory to create File and HdfsFile
---
.../src/assembly/resources/sbin/start-client.bat | 9 +-
client/src/assembly/resources/tools/export-csv.bat | 12 +-
client/src/assembly/resources/tools/import-csv.bat | 12 +-
.../org/apache/iotdb/client/AbstractScript.java | 7 +-
.../iotdb/tsfile/TsFileWriteWithRowBatch.java | 3 +-
server/pom.xml | 19 +++
.../resources/conf/iotdb-engine.properties | 4 +
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 16 ++-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 18 ++-
.../apache/iotdb/db/engine/fileSystem/FSType.java | 24 ++++
.../iotdb/db/engine/fileSystem/FileFactory.java | 65 +++++++++
.../iotdb/db/engine/fileSystem/HdfsFile.java | 156 +++++++++++++++++++++
.../db/engine/merge/manage/MergeResource.java | 25 ++--
.../engine/storagegroup/StorageGroupProcessor.java | 7 +-
.../db/engine/storagegroup/TsFileResource.java | 20 +--
.../iotdb/db/sync/receiver/SyncServiceImpl.java | 28 ++--
.../iotdb/db/tools/TsFileResourcePrinter.java | 12 +-
.../writelog/recover/TsFileRecoverPerformer.java | 5 +-
.../db/engine/memtable/MemTableFlushTaskTest.java | 4 +-
.../apache/iotdb/db/engine/merge/MergeTest.java | 7 +-
.../engine/storagegroup/TsFileProcessorTest.java | 12 +-
.../db/query/control/FileReaderManagerTest.java | 11 +-
.../iotdb/db/writelog/recover/LogReplayerTest.java | 5 +-
.../db/writelog/recover/SeqTsFileRecoverTest.java | 3 +-
.../writelog/recover/UnseqTsFileRecoverTest.java | 3 +-
.../write/writer/ForceAppendTsFileWriter.java | 2 -
.../iotdb/tsfile/write/writer/TsFileIOWriter.java | 6 +-
27 files changed, 367 insertions(+), 128 deletions(-)
diff --git a/client/src/assembly/resources/sbin/start-client.bat b/client/src/assembly/resources/sbin/start-client.bat
index 2642dfd..ba0d673 100755
--- a/client/src/assembly/resources/sbin/start-client.bat
+++ b/client/src/assembly/resources/sbin/start-client.bat
@@ -37,16 +37,9 @@ set JAVA_OPTS=-ea^
-DIOTDB_CLI_HOME=%IOTDB_CLI_HOME%
REM For each jar in the IOTDB_CLI_HOME lib directory call append to build the CLASSPATH variable.
-for %%i in ("%IOTDB_CLI_HOME%\lib\*.jar") do call :append "%%i"
-goto okClasspath
-
-:append
-set CLASSPATH=%CLASSPATH%;%1
-goto :eof
+set CLASSPATH=%IOTDB_CLI_HOME%\lib\*
REM -----------------------------------------------------------------------------
-:okClasspath
-
set PARAMETERS=%*
if "%PARAMETERS%" == "" set PARAMETERS=-h 127.0.0.1 -p 6667 -u root -pw root
diff --git a/client/src/assembly/resources/tools/export-csv.bat b/client/src/assembly/resources/tools/export-csv.bat
index f2432b8..2514bf6 100644
--- a/client/src/assembly/resources/tools/export-csv.bat
+++ b/client/src/assembly/resources/tools/export-csv.bat
@@ -37,19 +37,9 @@ set JAVA_OPTS=-ea^
-DIOTDB_CLI_HOME=%IOTDB_CLI_HOME%
@REM ***** CLASSPATH library setting *****
-@REM Ensure that any user defined CLASSPATH variables are not used on startup
-set CLASSPATH=""
-
-REM For each jar in the IOTDB_CLI_HOME lib directory call append to build the CLASSPATH variable.
-for %%i in ("%IOTDB_CLI_HOME%\lib\*.jar") do call :append "%%i"
-goto okClasspath
-
-:append
-set CLASSPATH=%CLASSPATH%;%1
-goto :eof
+set CLASSPATH=%IOTDB_CLI_HOME%\lib\*
REM -----------------------------------------------------------------------------
-:okClasspath
"%JAVA_HOME%\bin\java" -DIOTDB_CLI_HOME=%IOTDB_CLI_HOME% %JAVA_OPTS% -cp %CLASSPATH% %MAIN_CLASS% %*
diff --git a/client/src/assembly/resources/tools/import-csv.bat b/client/src/assembly/resources/tools/import-csv.bat
index ea514ea..c264bbf 100644
--- a/client/src/assembly/resources/tools/import-csv.bat
+++ b/client/src/assembly/resources/tools/import-csv.bat
@@ -37,19 +37,9 @@ set JAVA_OPTS=-ea^
-DIOTDB_CLI_HOME=%IOTDB_CLI_HOME%
@REM ***** CLASSPATH library setting *****
-@REM Ensure that any user defined CLASSPATH variables are not used on startup
-set CLASSPATH=""
-
-REM For each jar in the IOTDB_CLI_HOME lib directory call append to build the CLASSPATH variable.
-for %%i in ("%IOTDB_CLI_HOME%\lib\*.jar") do call :append "%%i"
-goto okClasspath
-
-:append
-set CLASSPATH=%CLASSPATH%;%1
-goto :eof
+set CLASSPATH=%IOTDB_CLI_HOME%\lib\*
REM -----------------------------------------------------------------------------
-:okClasspath
"%JAVA_HOME%\bin\java" -DIOTDB_CLI_HOME=%IOTDB_CLI_HOME% %JAVA_OPTS% -cp %CLASSPATH% %MAIN_CLASS% %*
diff --git a/client/src/test/java/org/apache/iotdb/client/AbstractScript.java b/client/src/test/java/org/apache/iotdb/client/AbstractScript.java
index 79c5daf..38e6df7 100644
--- a/client/src/test/java/org/apache/iotdb/client/AbstractScript.java
+++ b/client/src/test/java/org/apache/iotdb/client/AbstractScript.java
@@ -55,12 +55,7 @@ public abstract class AbstractScript {
throw new RuntimeException("user.dir " + userDir.getAbsolutePath() + " doesn't exist.");
}
File targetDir = new File(userDir, "target");
- File[] files = targetDir.listFiles(new FileFilter() {
- @Override
- public boolean accept(File pathname) {
- return pathname.isDirectory() && pathname.getName().startsWith("iotdb-client-");
- }
- });
+ File[] files = targetDir.listFiles(pathname -> pathname.isDirectory() && pathname.getName().startsWith("iotdb-client-"));
if(files.length != 1) {
throw new RuntimeException(
"Exactly one directory starting with 'iotdb-client-' should have been found, but was " + files.length);
diff --git a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileWriteWithRowBatch.java b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileWriteWithRowBatch.java
index 4bac373..bb9fc32 100644
--- a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileWriteWithRowBatch.java
+++ b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileWriteWithRowBatch.java
@@ -19,9 +19,10 @@
package org.apache.iotdb.tsfile;
-import java.io.File;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+
+import java.io.File;
import org.apache.iotdb.tsfile.write.TsFileWriter;
import org.apache.iotdb.tsfile.write.schema.Schema;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
diff --git a/server/pom.xml b/server/pom.xml
index 9055d28..1fab01f 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -85,6 +85,25 @@
<artifactId>powermock-api-mockito2</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
</dependencies>
<build>
<plugins>
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index 048acd5..42f527b 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -121,6 +121,10 @@ timestamp_precision=ms
# If its prefix is "/", then the path is absolute. Otherwise, it is relative.
# wal_dir=data/wal
+
+# Storage file system. Currently, Tsfile supports to be stored in LOCAL file system or HDFS.
+storage_fs=LOCAL
+
####################
### Memory Control Configuration
####################
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index d440e26..e464e5d 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -18,6 +18,7 @@
*/
package org.apache.iotdb.db.conf;
+import org.apache.iotdb.db.engine.fileSystem.FSType;
import java.io.File;
import java.time.ZoneId;
import java.util.ArrayList;
@@ -272,7 +273,7 @@ public class IoTDBConfig {
* Watermark method and parameters
*/
private String watermarkMethod = "GroupBasedLSBMethod(embed_row_cycle=5,embed_lsb_num=5)";
-
+
/**
* How much memory (in byte) can be used by a single merge task.
*/
@@ -324,6 +325,11 @@ public class IoTDBConfig {
private MergeFileStrategy mergeFileStrategy = MergeFileStrategy.MAX_SERIES_NUM;
+ /**
+ * Default storage is in local file system
+ */
+ private FSType storageFs = FSType.LOCAL;
+
public IoTDBConfig() {
// empty constructor
}
@@ -880,4 +886,12 @@ public class IoTDBConfig {
}
return null;
}
+
+ public FSType getStorageFs() {
+ return storageFs;
+ }
+
+ public void setStorageFs(String storageFs) {
+ this.storageFs = FSType.valueOf(storageFs);
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index f621eb0..f1cf1dc 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -18,17 +18,14 @@
*/
package org.apache.iotdb.db.conf;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.time.ZoneId;
-import java.util.Properties;
import org.apache.iotdb.db.utils.FilePathUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.*;
+import java.time.ZoneId;
+import java.util.Properties;
+
public class IoTDBDescriptor {
private static final Logger logger = LoggerFactory.getLogger(IoTDBDescriptor.class);
@@ -252,14 +249,14 @@ public class IoTDBDescriptor {
conf.setPerformanceStatMemoryInKB(Integer
.parseInt(properties.getProperty("performance_stat_memory_in_kb",
Integer.toString(conf.getPerformanceStatMemoryInKB())).trim()));
-
+
int maxConcurrentClientNum = Integer.parseInt(properties.
getProperty("rpc_max_concurrent_client_num",
Integer.toString(conf.getRpcMaxConcurrentClientNum()).trim()));
if (maxConcurrentClientNum <= 0) {
maxConcurrentClientNum = 65535;
}
-
+
conf.setEnableWatermark(Boolean.parseBoolean(properties.getProperty("watermark_module_opened",
Boolean.toString(conf.isEnableWatermark()).trim())));
conf.setWatermarkSecretKey(
@@ -270,7 +267,8 @@ public class IoTDBDescriptor {
properties.getProperty("watermark_method", conf.getWatermarkMethod()));
conf.setRpcMaxConcurrentClientNum(maxConcurrentClientNum);
-
+
+ conf.setStorageFs(properties.getProperty("storage_fs"));
} catch (IOException e) {
logger.warn("Cannot load config file because, use default configuration", e);
} catch (Exception e) {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/fileSystem/FSType.java b/server/src/main/java/org/apache/iotdb/db/engine/fileSystem/FSType.java
new file mode 100644
index 0000000..4bbe2ec
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/engine/fileSystem/FSType.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.fileSystem;
+
+public enum FSType {
+ LOCAL, HDFS
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/fileSystem/FileFactory.java b/server/src/main/java/org/apache/iotdb/db/engine/fileSystem/FileFactory.java
new file mode 100644
index 0000000..5240f8c
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/engine/fileSystem/FileFactory.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.fileSystem;
+
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+
+import java.io.File;
+import java.net.URI;
+
+public enum FileFactory {
+
+ INSTANCE;
+
+ private static FSType FSType = IoTDBDescriptor.getInstance().getConfig().getStorageFs();
+
+ public File getFile(String pathname) {
+ if (FSType.equals(FSType.HDFS)) {
+ return new HdfsFile(pathname);
+ } else {
+ return new File(pathname);
+ }
+ }
+
+ public File getFile(String parent, String child) {
+ if (FSType.equals(FSType.HDFS)) {
+ return new HdfsFile(parent, child);
+ } else {
+ return new File(parent, child);
+ }
+ }
+
+ public File getFile(File parent, String child) {
+ if (FSType.equals(FSType.HDFS)) {
+ return new HdfsFile(parent, child);
+ } else {
+ return new File(parent, child);
+ }
+ }
+
+ public File getFile(URI uri) {
+ if (FSType.equals(FSType.HDFS)) {
+ return new HdfsFile(uri);
+ } else {
+ return new File(uri);
+ }
+ }
+
+}
\ No newline at end of file
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/fileSystem/HdfsFile.java b/server/src/main/java/org/apache/iotdb/db/engine/fileSystem/HdfsFile.java
new file mode 100644
index 0000000..3378954
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/engine/fileSystem/HdfsFile.java
@@ -0,0 +1,156 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.fileSystem;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.*;
+import org.apache.iotdb.tsfile.write.TsFileWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.io.IOException;
+import java.net.URI;
+import java.util.ArrayList;
+
+public class HdfsFile extends File {
+
+ private Path hdfsPath;
+
+ private static final Logger logger = LoggerFactory.getLogger(TsFileWriter.class);
+
+
+ public HdfsFile(String pathname) {
+ super(pathname);
+ hdfsPath = new Path(pathname);
+ }
+
+ public HdfsFile(String parent, String child) {
+ super(parent, child);
+ }
+
+ public HdfsFile(File parent, String child) {
+ super(parent, child);
+ }
+
+ public HdfsFile(URI uri) {
+ super(uri);
+ }
+
+ @Override
+ public String getAbsolutePath() {
+ return hdfsPath.toUri().toString();
+ }
+
+ @Override
+ public long length() {
+ try {
+ FileSystem fs = hdfsPath.getFileSystem(new Configuration());
+ return fs.getFileStatus(hdfsPath).getLen();
+ } catch (IOException e) {
+ logger.error("Fail to get length of the file. ", e);
+ return 0;
+ }
+ }
+
+ @Override
+ public boolean exists() {
+ try {
+ FileSystem fs = hdfsPath.getFileSystem(new Configuration());
+ return fs.exists(hdfsPath);
+ } catch (IOException e) {
+ logger.error("Fail to check whether the file or directory exists. ", e);
+ return false;
+ }
+ }
+
+ @Override
+ public File[] listFiles() {
+ ArrayList<HdfsFile> files = new ArrayList<>();
+ try {
+ FileSystem fs = hdfsPath.getFileSystem(new Configuration());
+ RemoteIterator<LocatedFileStatus> iterator = fs.listFiles(hdfsPath, true);
+ while (iterator.hasNext()) {
+ LocatedFileStatus fileStatus = iterator.next();
+ Path fullPath = fileStatus.getPath();
+ files.add(new HdfsFile(fullPath.toUri()));
+ }
+ return files.toArray(new HdfsFile[files.size()]);
+ } catch (IOException e) {
+ logger.error("Fail to list files. ", e);
+ return null;
+ }
+ }
+
+ @Override
+ public File[] listFiles(FileFilter filter) {
+ ArrayList<HdfsFile> files = new ArrayList<>();
+ try {
+ PathFilter pathFilter = new GlobFilter(filter.toString()); // TODO
+ FileSystem fs = hdfsPath.getFileSystem(new Configuration());
+ RemoteIterator<LocatedFileStatus> iterator = fs.listFiles(hdfsPath, true);
+ while (iterator.hasNext()) {
+ LocatedFileStatus fileStatus = iterator.next();
+ Path fullPath = fileStatus.getPath();
+ if (pathFilter.accept(fullPath)) {
+ files.add(new HdfsFile(fullPath.toUri()));
+ }
+ }
+ return files.toArray(new HdfsFile[files.size()]);
+ } catch (IOException e) {
+ logger.error("Fail to list files. ", e);
+ return null;
+ }
+ }
+
+ @Override
+ public File getParentFile() {
+ return new HdfsFile(hdfsPath.getParent().toUri());
+ }
+
+ @Override
+ public boolean createNewFile() throws IOException {
+ FileSystem fs = hdfsPath.getFileSystem(new Configuration());
+ return fs.createNewFile(hdfsPath);
+ }
+
+ @Override
+ public boolean delete() {
+ try {
+ FileSystem fs = hdfsPath.getFileSystem(new Configuration());
+ return fs.delete(hdfsPath, true);
+ } catch (IOException e) {
+ logger.error("Fail to delete file. ", e);
+ return false;
+ }
+ }
+
+ @Override
+ public boolean mkdirs() {
+ try {
+ FileSystem fs = hdfsPath.getFileSystem(new Configuration());
+ return fs.mkdirs(hdfsPath);
+ } catch (IOException e) {
+ logger.error("Fail to create directory. ", e);
+ return false;
+ }
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeResource.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeResource.java
index 042c499..d8b6c4c 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeResource.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeResource.java
@@ -19,19 +19,6 @@
package org.apache.iotdb.db.engine.merge.manage;
-import static org.apache.iotdb.db.engine.merge.task.MergeTask.MERGE_SUFFIX;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.stream.Collectors;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.query.reader.IPointReader;
@@ -40,6 +27,7 @@ import org.apache.iotdb.db.utils.MergeUtils;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.db.engine.fileSystem.FileFactory;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Chunk;
import org.apache.iotdb.tsfile.read.common.Path;
@@ -51,6 +39,14 @@ import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.util.*;
+import java.util.Map.Entry;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.iotdb.db.engine.merge.task.MergeTask.MERGE_SUFFIX;
+
/**
* MergeResource manages files and caches of readers, writers, MeasurementSchemas and
* modifications to avoid unnecessary object creations and file openings.
@@ -102,7 +98,8 @@ public class MergeResource {
public RestorableTsFileIOWriter getMergeFileWriter(TsFileResource resource) throws IOException {
RestorableTsFileIOWriter writer = fileWriterCache.get(resource);
if (writer == null) {
- writer = new RestorableTsFileIOWriter(new File(resource.getFile().getPath() + MERGE_SUFFIX));
+ writer = new RestorableTsFileIOWriter(FileFactory.INSTANCE
+ .getFile(resource.getFile().getPath() + MERGE_SUFFIX));
fileWriterCache.put(resource, writer);
}
return writer;
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index 1e41c57..389de7e 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -74,6 +74,7 @@ import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.db.engine.fileSystem.FileFactory;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.tsfile.write.schema.Schema;
@@ -239,7 +240,7 @@ public class StorageGroupProcessor {
private List<TsFileResource> getAllFiles(List<String> folders) throws IOException {
List<File> tsFiles = new ArrayList<>();
for (String baseDir : folders) {
- File fileFolder = new File(baseDir, storageGroupName);
+ File fileFolder = FileFactory.INSTANCE.getFile(baseDir, storageGroupName);
if (!fileFolder.exists()) {
continue;
}
@@ -494,11 +495,11 @@ public class StorageGroupProcessor {
+ TSFILE_SUFFIX;
if (sequence) {
- return new TsFileProcessor(storageGroupName, new File(filePath),
+ return new TsFileProcessor(storageGroupName, FileFactory.INSTANCE.getFile(filePath),
schema, versionController, this::closeUnsealedTsFileProcessor,
this::updateLatestFlushTimeCallback, sequence);
} else {
- return new TsFileProcessor(storageGroupName, new File(filePath),
+ return new TsFileProcessor(storageGroupName, FileFactory.INSTANCE.getFile(filePath),
schema, versionController, this::closeUnsealedTsFileProcessor,
() -> true, sequence);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
index 3a7af7e..6dfdf57 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
@@ -18,14 +18,7 @@
*/
package org.apache.iotdb.db.engine.storagegroup;
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
+import java.io.*;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -37,6 +30,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.iotdb.db.engine.modification.ModificationFile;
import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
+import org.apache.iotdb.db.engine.fileSystem.FileFactory;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
public class TsFileResource {
@@ -124,8 +118,8 @@ public class TsFileResource {
ReadWriteIOUtils.write(entry.getValue(), outputStream);
}
}
- File src = new File(file + RESOURCE_SUFFIX + TEMP_SUFFIX);
- File dest = new File(file + RESOURCE_SUFFIX);
+ File src = FileFactory.INSTANCE.getFile(file + RESOURCE_SUFFIX + TEMP_SUFFIX);
+ File dest = FileFactory.INSTANCE.getFile(file + RESOURCE_SUFFIX);
dest.delete();
FileUtils.moveFile(src, dest);
}
@@ -167,7 +161,7 @@ public class TsFileResource {
}
public boolean fileExists() {
- return new File(file+RESOURCE_SUFFIX).exists();
+ return FileFactory.INSTANCE.getFile(file + RESOURCE_SUFFIX).exists();
}
public void forceUpdateEndTime(String device, long time) {
@@ -242,8 +236,8 @@ public class TsFileResource {
public void remove() {
file.delete();
- new File(file.getPath() + RESOURCE_SUFFIX).delete();
- new File(file.getPath() + ModificationFile.FILE_SUFFIX).delete();
+ FileFactory.INSTANCE.getFile(file.getPath() + RESOURCE_SUFFIX).delete();
+ FileFactory.INSTANCE.getFile(file.getPath() + ModificationFile.FILE_SUFFIX).delete();
}
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/sync/receiver/SyncServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/sync/receiver/SyncServiceImpl.java
index 1dcd4f5..05c30a2 100644
--- a/server/src/main/java/org/apache/iotdb/db/sync/receiver/SyncServiceImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/sync/receiver/SyncServiceImpl.java
@@ -18,12 +18,7 @@
*/
package org.apache.iotdb.db.sync.receiver;
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
+import java.io.*;
import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
@@ -66,6 +61,7 @@ import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadataIndex;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.db.engine.fileSystem.FileFactory;
import org.apache.iotdb.tsfile.read.ReadOnlyTsFile;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Field;
@@ -149,7 +145,7 @@ public class SyncServiceImpl implements SyncService.Iface {
fileNodeStartTime.set(new HashMap<>());
fileNodeEndTime.set(new HashMap<>());
try {
- FileUtils.deleteDirectory(new File(syncDataPath));
+ FileUtils.deleteDirectory(FileFactory.INSTANCE.getFile(syncDataPath));
} catch (IOException e) {
logger.error("cannot delete directory {} ", syncFolderPath);
return false;
@@ -157,7 +153,7 @@ public class SyncServiceImpl implements SyncService.Iface {
for (String bufferWritePath : bufferWritePaths) {
bufferWritePath = FilePathUtils.regularizePath(bufferWritePath);
String backupPath = bufferWritePath + SYNC_SERVER + File.separator;
- File backupDirectory = new File(backupPath, this.uuid.get());
+ File backupDirectory = FileFactory.INSTANCE.getFile(backupPath, this.uuid.get());
if (backupDirectory.exists() && backupDirectory.list().length != 0) {
try {
FileUtils.deleteDirectory(backupDirectory);
@@ -206,7 +202,7 @@ public class SyncServiceImpl implements SyncService.Iface {
/** sync metadata, include storage group and timeseries **/
return Boolean.toString(loadMetadata());
} else if (status == SyncDataStatus.PROCESSING_STATUS) {
- File file = new File(schemaFromSenderPath.get());
+ File file = FileFactory.INSTANCE.getFile(schemaFromSenderPath.get());
if (!file.getParentFile().exists()) {
try {
file.getParentFile().mkdirs();
@@ -233,7 +229,7 @@ public class SyncServiceImpl implements SyncService.Iface {
}
md5OfReceiver = (new BigInteger(1, md.digest())).toString(16);
if (!md5.equals(md5OfReceiver)) {
- FileUtils.forceDelete(new File(schemaFromSenderPath.get()));
+ FileUtils.forceDelete(FileFactory.INSTANCE.getFile(schemaFromSenderPath.get()));
}
} catch (Exception e) {
logger.error("Receiver cannot generate md5 {}", schemaFromSenderPath.get(), e);
@@ -246,7 +242,7 @@ public class SyncServiceImpl implements SyncService.Iface {
* Load metadata from sender
*/
private boolean loadMetadata() {
- if (new File(schemaFromSenderPath.get()).exists()) {
+ if (FileFactory.INSTANCE.getFile(schemaFromSenderPath.get()).exists()) {
try (BufferedReader br = new BufferedReader(
new java.io.FileReader(schemaFromSenderPath.get()))) {
String metadataOperation;
@@ -331,7 +327,7 @@ public class SyncServiceImpl implements SyncService.Iface {
syncDataPath = FilePathUtils.regularizePath(syncDataPath);
filePath = syncDataPath + filePath;
if (status == SyncDataStatus.PROCESSING_STATUS) { // there are still data stream to add
- File file = new File(filePath);
+ File file = FileFactory.INSTANCE.getFile(filePath);
if (!file.getParentFile().exists()) {
try {
file.getParentFile().mkdirs();
@@ -363,7 +359,7 @@ public class SyncServiceImpl implements SyncService.Iface {
logger.info(String.format("Receiver has received %d files from sender", fileNum.get()));
} else {
- FileUtils.forceDelete(new File(filePath));
+ FileUtils.forceDelete(FileFactory.INSTANCE.getFile(filePath));
}
} catch (Exception e) {
logger.error("Receiver cannot generate md5 {}", filePath, e);
@@ -389,7 +385,7 @@ public class SyncServiceImpl implements SyncService.Iface {
* Get all tsfiles' info which are sent from sender, it is preparing for merging these data
*/
public void getFileNodeInfo() throws IOException {
- File dataFileRoot = new File(syncDataPath);
+ File dataFileRoot = FileFactory.INSTANCE.getFile(syncDataPath);
File[] files = dataFileRoot.listFiles();
int processedNum = 0;
for (File storageGroupPB : files) {
@@ -469,7 +465,7 @@ public class SyncServiceImpl implements SyncService.Iface {
String header = syncDataPath;
String relativePath = path.substring(header.length());
TsFileResource fileNode = new TsFileResource(
- new File(DirectoryManager.getInstance().getNextFolderIndexForSequenceFile() +
+ FileFactory.INSTANCE.getFile(DirectoryManager.getInstance().getNextFolderIndexForSequenceFile() +
File.separator + relativePath), startTimeMap, endTimeMap
);
// call interface of load external file
@@ -718,7 +714,7 @@ public class SyncServiceImpl implements SyncService.Iface {
fileNodeEndTime.remove();
schemaFromSenderPath.remove();
try {
- FileUtils.deleteDirectory(new File(syncFolderPath));
+ FileUtils.deleteDirectory(FileFactory.INSTANCE.getFile(syncFolderPath));
} catch (IOException e) {
logger.error("can not delete directory {}", syncFolderPath, e);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/tools/TsFileResourcePrinter.java b/server/src/main/java/org/apache/iotdb/db/tools/TsFileResourcePrinter.java
index ed384ce..8b8de72 100644
--- a/server/src/main/java/org/apache/iotdb/db/tools/TsFileResourcePrinter.java
+++ b/server/src/main/java/org/apache/iotdb/db/tools/TsFileResourcePrinter.java
@@ -22,8 +22,11 @@ package org.apache.iotdb.db.tools;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
+import java.util.Comparator;
+
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.qp.constant.DatetimeUtils;
+import org.apache.iotdb.db.engine.fileSystem.FileFactory;
/**
* this tool can analyze the tsfile.resource files from a folder.
@@ -36,10 +39,9 @@ public class TsFileResourcePrinter {
if (args.length >= 1) {
folder = args[0];
}
- File folderFile = new File(folder);
- File[] files = folderFile.listFiles((dir, name) -> name.endsWith(".tsfile.resource"));
- Arrays.sort(files, (x, y) -> Long
- .compare(Long.valueOf(x.getName().split("-")[0]), Long.valueOf(y.getName().split("-")[0])));
+ File folderFile = FileFactory.INSTANCE.getFile(folder);
+ File[] files = folderFile.listFiles(file -> file.getName().endsWith(".tsfile.resource"));
+ Arrays.sort(files, Comparator.comparingLong(x -> Long.valueOf(x.getName().split("-")[0])));
for (File file : files) {
printResource(file.getAbsolutePath());
@@ -49,7 +51,7 @@ public class TsFileResourcePrinter {
public static void printResource(String filename) throws IOException {
filename = filename.substring(0, filename.length() - 9);
- TsFileResource resource = new TsFileResource(new File(filename));
+ TsFileResource resource = new TsFileResource(FileFactory.INSTANCE.getFile(filename));
System.err.println(String.format("analyzing %s ...", filename));
resource.deSerialize();
diff --git a/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java b/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
index cf8ee79..3039195 100644
--- a/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
+++ b/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
@@ -38,6 +38,7 @@ import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadata;
import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadataIndex;
import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData;
+import org.apache.iotdb.db.engine.fileSystem.FileFactory;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.write.schema.Schema;
import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter;
@@ -81,7 +82,7 @@ public class TsFileRecoverPerformer {
this.logReplayer = new LogReplayer(logNodePrefix, insertFilePath, tsFileResource.getModFile(),
versionController,
tsFileResource, schema, recoverMemTable, acceptUnseq);
- File insertFile = new File(insertFilePath);
+ File insertFile = FileFactory.INSTANCE.getFile(insertFilePath);
if (!insertFile.exists()) {
logger.error("TsFile {} is missing, will skip its recovery.", insertFilePath);
return;
@@ -141,7 +142,7 @@ public class TsFileRecoverPerformer {
// clean logs
try {
MultiFileLogNodeManager.getInstance()
- .deleteNode(logNodePrefix + new File(insertFilePath).getName());
+ .deleteNode(logNodePrefix + FileFactory.INSTANCE.getFile(insertFilePath).getName());
} catch (IOException e) {
throw new ProcessorException(e);
}
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskTest.java b/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskTest.java
index 09b34e8..5586144 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskTest.java
@@ -21,12 +21,12 @@ package org.apache.iotdb.db.engine.memtable;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
-import java.io.File;
import java.util.concurrent.ExecutionException;
import org.apache.iotdb.db.engine.MetadataManagerHelper;
import org.apache.iotdb.db.engine.flush.MemTableFlushTask;
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
+import org.apache.iotdb.db.engine.fileSystem.FileFactory;
import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter;
import org.junit.After;
import org.junit.Before;
@@ -45,7 +45,7 @@ public class MemTableFlushTaskTest {
public void setUp() throws Exception {
MetadataManagerHelper.initMetadata();
EnvironmentUtils.envSetUp();
- writer = new RestorableTsFileIOWriter(new File(filePath));
+ writer = new RestorableTsFileIOWriter(FileFactory.INSTANCE.getFile(filePath));
memTable = new PrimitiveMemTable();
}
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java
index cab3a59..853ceaf 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java
@@ -40,6 +40,7 @@ import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.db.engine.fileSystem.FileFactory;
import org.apache.iotdb.tsfile.write.TsFileWriter;
import org.apache.iotdb.tsfile.write.record.TSRecord;
import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint;
@@ -114,18 +115,18 @@ abstract class MergeTest {
private void prepareFiles(int seqFileNum, int unseqFileNum) throws IOException, WriteProcessException {
for (int i = 0; i < seqFileNum; i++) {
- File file = new File(i + "seq.tsfile");
+ File file = FileFactory.INSTANCE.getFile(i + "seq.tsfile");
TsFileResource tsFileResource = new TsFileResource(file);
seqResources.add(tsFileResource);
prepareFile(tsFileResource, i * ptNum, ptNum, 0);
}
for (int i = 0; i < unseqFileNum; i++) {
- File file = new File(i + "unseq.tsfile");
+ File file = FileFactory.INSTANCE.getFile(i + "unseq.tsfile");
TsFileResource tsFileResource = new TsFileResource(file);
unseqResources.add(tsFileResource);
prepareFile(tsFileResource, i * ptNum, ptNum * (i + 1) / unseqFileNum, 10000);
}
- File file = new File(unseqFileNum + "unseq.tsfile");
+ File file = FileFactory.INSTANCE.getFile(unseqFileNum + "unseq.tsfile");
TsFileResource tsFileResource = new TsFileResource(file);
unseqResources.add(tsFileResource);
prepareFile(tsFileResource, 0, ptNum * unseqFileNum, 20000);
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorTest.java b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorTest.java
index a61534a..407dd94 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorTest.java
@@ -22,7 +22,6 @@ import static junit.framework.TestCase.assertTrue;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
-import java.io.File;
import java.io.IOException;
import java.util.Collections;
import java.util.Iterator;
@@ -42,6 +41,7 @@ import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetaData;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.db.engine.fileSystem.FileFactory;
import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.tsfile.write.record.TSRecord;
import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint;
@@ -76,7 +76,7 @@ public class TsFileProcessorTest {
@Test
public void testWriteAndFlush()
throws WriteProcessException, IOException, TsFileProcessorException {
- processor = new TsFileProcessor(storageGroup, new File(filePath),
+ processor = new TsFileProcessor(storageGroup, FileFactory.INSTANCE.getFile(filePath),
SchemaUtils.constructSchema(deviceId), SysTimeVersionController.INSTANCE, x -> {
},
() -> true, true);
@@ -123,7 +123,7 @@ public class TsFileProcessorTest {
@Test
public void testWriteAndRestoreMetadata()
throws IOException {
- processor = new TsFileProcessor(storageGroup, new File(filePath),
+ processor = new TsFileProcessor(storageGroup, FileFactory.INSTANCE.getFile(filePath),
SchemaUtils.constructSchema(deviceId), SysTimeVersionController.INSTANCE, x -> {
},
() -> true, true);
@@ -168,7 +168,7 @@ public class TsFileProcessorTest {
RestorableTsFileIOWriter tsFileIOWriter = processor.getWriter();
List<ChunkGroupMetaData> chunkGroupMetaDataList = tsFileIOWriter.getChunkGroupMetaDatas();
RestorableTsFileIOWriter restorableTsFileIOWriter = new RestorableTsFileIOWriter(
- new File(filePath));
+ FileFactory.INSTANCE.getFile(filePath));
List<ChunkGroupMetaData> restoredChunkGroupMetaDataList = restorableTsFileIOWriter
.getChunkGroupMetaDatas();
assertEquals(chunkGroupMetaDataList.size(), restoredChunkGroupMetaDataList.size());
@@ -190,7 +190,7 @@ public class TsFileProcessorTest {
@Test
public void testMultiFlush()
throws WriteProcessException, IOException, TsFileProcessorException {
- processor = new TsFileProcessor(storageGroup, new File(filePath),
+ processor = new TsFileProcessor(storageGroup, FileFactory.INSTANCE.getFile(filePath),
SchemaUtils.constructSchema(deviceId), SysTimeVersionController.INSTANCE, x -> {
},
() -> true, true);
@@ -226,7 +226,7 @@ public class TsFileProcessorTest {
@Test
public void testWriteAndClose()
throws WriteProcessException, IOException {
- processor = new TsFileProcessor(storageGroup, new File(filePath),
+ processor = new TsFileProcessor(storageGroup, FileFactory.INSTANCE.getFile(filePath),
SchemaUtils.constructSchema(deviceId), SysTimeVersionController.INSTANCE,
unsealedTsFileProcessor -> {
TsFileResource resource = unsealedTsFileProcessor.getTsFileResource();
diff --git a/server/src/test/java/org/apache/iotdb/db/query/control/FileReaderManagerTest.java b/server/src/test/java/org/apache/iotdb/db/query/control/FileReaderManagerTest.java
index 683eb67..0545df1 100644
--- a/server/src/test/java/org/apache/iotdb/db/query/control/FileReaderManagerTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/query/control/FileReaderManagerTest.java
@@ -22,10 +22,11 @@ import static org.junit.Assert.fail;
import java.io.File;
import java.io.IOException;
-import java.nio.file.Files;
+
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.engine.fileSystem.FileFactory;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -60,7 +61,7 @@ public class FileReaderManagerTest {
TsFileResource[] tsFileResources = new TsFileResource[MAX_FILE_SIZE + 1];
for (int i = 1; i <= MAX_FILE_SIZE; i++) {
- File file = new File(filePath + i);
+ File file = FileFactory.INSTANCE.getFile(filePath + i);
file.createNewFile();
tsFileResources[i] = new TsFileResource(file);
}
@@ -115,7 +116,7 @@ public class FileReaderManagerTest {
t2.join();
for (int i = 1; i <= MAX_FILE_SIZE; i++) {
- TsFileResource tsFile = new TsFileResource(new File(filePath + i));
+ TsFileResource tsFile = new TsFileResource(FileFactory.INSTANCE.getFile(filePath + i));
Assert.assertTrue(manager.contains(tsFile, false));
}
@@ -135,8 +136,8 @@ public class FileReaderManagerTest {
FileReaderManager.getInstance().closeAndRemoveAllOpenedReaders();
for (int i = 1; i < MAX_FILE_SIZE; i++) {
- File file = new File(filePath + i);
- boolean result = Files.deleteIfExists(file.toPath());
+ File file = FileFactory.INSTANCE.getFile(filePath + i);
+ boolean result = !file.exists() || file.delete();
if (!result) {
fail();
}
diff --git a/server/src/test/java/org/apache/iotdb/db/writelog/recover/LogReplayerTest.java b/server/src/test/java/org/apache/iotdb/db/writelog/recover/LogReplayerTest.java
index b8c0613..12bfbbc 100644
--- a/server/src/test/java/org/apache/iotdb/db/writelog/recover/LogReplayerTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/writelog/recover/LogReplayerTest.java
@@ -43,6 +43,7 @@ import org.apache.iotdb.db.writelog.manager.MultiFileLogNodeManager;
import org.apache.iotdb.db.writelog.node.WriteLogNode;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.db.engine.fileSystem.FileFactory;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.write.schema.Schema;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
@@ -53,8 +54,8 @@ public class LogReplayerTest {
@Test
public void test() throws IOException, ProcessorException {
String logNodePrefix = "testLogNode";
- File tsFile = new File("temp", "test.ts");
- File modF = new File("test.mod");
+ File tsFile = FileFactory.INSTANCE.getFile("temp", "test.ts");
+ File modF = FileFactory.INSTANCE.getFile("test.mod");
ModificationFile modFile = new ModificationFile(modF.getPath());
VersionController versionController = new VersionController() {
@Override
diff --git a/server/src/test/java/org/apache/iotdb/db/writelog/recover/SeqTsFileRecoverTest.java b/server/src/test/java/org/apache/iotdb/db/writelog/recover/SeqTsFileRecoverTest.java
index 908337a..67b0e2b 100644
--- a/server/src/test/java/org/apache/iotdb/db/writelog/recover/SeqTsFileRecoverTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/writelog/recover/SeqTsFileRecoverTest.java
@@ -35,6 +35,7 @@ import org.apache.iotdb.db.writelog.node.WriteLogNode;
import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.db.engine.fileSystem.FileFactory;
import org.apache.iotdb.tsfile.read.ReadOnlyTsFile;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Field;
@@ -76,7 +77,7 @@ public class SeqTsFileRecoverTest {
@Before
public void setup() throws IOException, WriteProcessException {
- tsF = new File("temp", "test.ts");
+ tsF = FileFactory.INSTANCE.getFile("temp", "test.ts");
tsF.getParentFile().mkdirs();
schema = new Schema();
diff --git a/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java b/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java
index 8d19331..924081f 100644
--- a/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java
@@ -37,6 +37,7 @@ import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.db.engine.fileSystem.FileFactory;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Chunk;
import org.apache.iotdb.tsfile.read.common.Path;
@@ -79,7 +80,7 @@ public class UnseqTsFileRecoverTest {
@Before
public void setup() throws IOException, WriteProcessException {
- tsF = new File("temp", "test.ts");
+ tsF = FileFactory.INSTANCE.getFile("temp", "test.ts");
tsF.getParentFile().mkdirs();
schema = new Schema();
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriter.java
index 10c5f42..102c01a 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriter.java
@@ -20,9 +20,7 @@ package org.apache.iotdb.tsfile.write.writer;
import java.io.File;
import java.io.IOException;
-import java.util.HashMap;
import java.util.Map;
-import java.util.Map.Entry;
import org.apache.iotdb.tsfile.exception.write.TsFileNotCompleteException;
import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadata;
import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadataIndex;
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
index ff2a24c..26ff64b 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
@@ -230,7 +230,7 @@ public class TsFileIOWriter {
out.write(chunk.getData());
endChunk(chunkMetadata.getNumOfPoints());
}
-
+
/**
* end chunk and write some log.
*
@@ -376,12 +376,8 @@ public class TsFileIOWriter {
}
/**
-<<<<<<< HEAD
- * close the outputStream or file channel without writing FileMetadata.
-=======
* close the outputStream or file channel without writing FileMetadata. This is just used for
* Testing.
->>>>>>> master
*/
public void close() throws IOException {
canWrite = false;